Get up-to-date with main branch

Author: milancurcic

CMakeLists.txt

@@ -102,7 +102,7 @@ string(REGEX REPLACE "^ | $" "" LIBS "${LIBS}")
 
 # tests
 enable_testing()
-foreach(execid input1d_layer input3d_layer dense_layer conv2d_layer maxpool2d_layer dense_network)
+foreach(execid input1d_layer input3d_layer dense_layer conv2d_layer maxpool2d_layer dense_network conv2d_network)
   add_executable(test_${execid} test/test_${execid}.f90)
   target_link_libraries(test_${execid} neural ${LIBS})
   add_test(test_${execid} bin/test_${execid})
@@ -111,5 +111,4 @@ endforeach()
 foreach(execid mnist simple sine)
   add_executable(${execid} example/${execid}.f90)
   target_link_libraries(${execid} neural ${LIBS})
-  #add_test(example_${execid} bin/example_${execid})
 endforeach()

README.md

@@ -21,6 +21,14 @@ Read the paper [here](https://arxiv.org/abs/1902.06714).
 * Data-based parallelism
 * Several activation functions
 
+### Available layer types
+
+| Layer type | Constructor name | Rank of output array | Forward pass | Backward pass |
+|------------|------------------|----------------------|--------------|---------------|
+| Input | `input` | 1, 3 | n/a | n/a |
+| Dense (fully-connected) | `dense` | 1 | ✅ | ✅ |
+| Convolutional (2-d) | `conv2d` | 3 | ✅ | ❌ |
+
 ## Getting started
 
 Get the code:
@@ -172,6 +180,10 @@ Most Linux OSs have it out of the box.
 The dataset will be downloaded only the first time you run the example in any
 given directory.
 
+If you're using Windows OS or don't have curl for any other reason,
+download [mnist.tar.gz](https://github.com/modern-fortran/neural-fortran/files/8498876/mnist.tar.gz)
+directly and unpack in the directory in which you will run the example program.
+
 ## API documentation
 
 API documentation can be generated with [FORD](https://github.com/Fortran-FOSS-Programmers/ford/).

fpm.toml

@@ -1,5 +1,5 @@
 name = "neural-fortran"
-version = "0.3.0"
+version = "0.4.0"
 license = "MIT"
 author = "Milan Curcic"
 maintainer = "milancurcic@hey.com"

src/nf_conv2d_layer.f90

@@ -13,7 +13,7 @@ module nf_conv2d_layer
     integer :: width
     integer :: height
     integer :: channels
-    integer :: window_size
+    integer :: kernel_size
     integer :: filters
 
     real, allocatable :: biases(:) ! size(filters)
@@ -29,10 +29,11 @@ module nf_conv2d_layer
   end type conv2d_layer
 
   interface conv2d_layer
-    pure module function conv2d_layer_cons(window_size, filters, activation) result(res)
+    pure module function conv2d_layer_cons(filters, kernel_size, activation) &
+      result(res)
       !! `conv2d_layer` constructor function
-      integer, intent(in) :: window_size
       integer, intent(in) :: filters
+      integer, intent(in) :: kernel_size
       character(*), intent(in) :: activation
       type(conv2d_layer) :: res
     end function conv2d_layer_cons

src/nf_conv2d_layer_submodule.f90

@@ -6,13 +6,13 @@
 
 contains
 
-  pure module function conv2d_layer_cons(window_size, filters, activation) result(res)
+  pure module function conv2d_layer_cons(filters, kernel_size, activation) result(res)
     implicit none
-    integer, intent(in) :: window_size
     integer, intent(in) :: filters
+    integer, intent(in) :: kernel_size
     character(*), intent(in) :: activation
     type(conv2d_layer) :: res
-    res % window_size = window_size
+    res % kernel_size = kernel_size
     res % filters = filters
     call res % set_activation(activation)
   end function conv2d_layer_cons
@@ -24,21 +24,21 @@ module subroutine init(self, input_shape)
     integer, intent(in) :: input_shape(:)
 
     self % channels = input_shape(1)
-    self % width = input_shape(2) - self % window_size + 1
-    self % height = input_shape(3) - self % window_size + 1
+    self % width = input_shape(2) - self % kernel_size + 1
+    self % height = input_shape(3) - self % kernel_size + 1
 
     ! Output of shape filters x width x height
     allocate(self % output(self % filters, self % width, self % height))
     self % output = 0
 
     ! Kernel of shape filters x channels x width x height
     allocate(self % kernel(self % filters, self % channels, &
-                           self % window_size, self % window_size))
+                           self % kernel_size, self % kernel_size))
 
     ! Initialize the kernel with random values with a normal distribution.
     self % kernel = randn(self % filters, self % channels, &
-                          self % window_size, self % window_size) &
-                  / self % window_size**2 !TODO window_width * window_height
+                          self % kernel_size, self % kernel_size) &
+                  / self % kernel_size**2 !TODO kernel_width * kernel_height
 
     allocate(self % biases(self % filters))
     self % biases = 0
@@ -54,7 +54,6 @@ pure module subroutine forward(self, input)
     integer :: istart, iend
     integer :: jstart, jend
     integer :: i, j, n
-    integer :: ii, jj
     integer :: iws, iwe, jws, jwe
     integer :: half_window
 
@@ -64,34 +63,34 @@ pure module subroutine forward(self, input)
     input_height = size(input, dim=3)
 
     ! Half-window is 1 for window size 3; 2 for window size 5; etc.
-    half_window = self % window_size / 2
+    half_window = self % kernel_size / 2
 
     ! Determine the start and end indices for the width and height dimensions
     ! of the input that correspond to the center of each window.
-    istart = half_window + 1 ! TODO window_width
-    jstart = half_window + 1 ! TODO window_height
+    istart = half_window + 1 ! TODO kernel_width
+    jstart = half_window + 1 ! TODO kernel_height
     iend = input_width - istart + 1
     jend = input_height - jstart + 1
 
     convolution: do concurrent(i = istart:iend, j = jstart:jend)
 
       ! Start and end indices of the input data on the filter window
       ! iws and jws are also coincidentally the indices of the output matrix
-      iws = i - half_window ! TODO window_width
-      iwe = i + half_window ! TODO window_width
-      jws = j - half_window ! TODO window_height
-      jwe = j + half_window ! TODO window_height
+      iws = i - half_window ! TODO kernel_width
+      iwe = i + half_window ! TODO kernel_width
+      jws = j - half_window ! TODO kernel_height
+      jwe = j + half_window ! TODO kernel_height
 
-      ! This computes the inner tensor product, sum(w_ij * x_ij), for each filter,
-      ! and we add bias b_n to it.
+      ! This computes the inner tensor product, sum(w_ij * x_ij), for each
+      ! filter, and we add bias b_n to it.
       inner_product: do concurrent(n = 1:self % filters)
         self % output(n,iws,jws) = &
           sum(self % kernel(n,:,:,:) * input(:,iws:iwe,jws:jwe)) &
           + self % biases(n)
       end do inner_product
 
-      ! TODO We may need to store self % output before we activate it for the backward
-      ! TODO pass, just like we do for the dense layer.
+      ! TODO We may need to store self % output before we activate it for the
+      ! TODO backward pass, just like we do for the dense layer.
 
       ! Activate
       self % output(:,iws,jws) = self % activation(self % output(:,iws,jws))

src/nf_layer_constructors.f90

@@ -84,7 +84,7 @@ pure module function dense(layer_size, activation) result(res)
         !! Resulting layer instance
     end function dense
 
-    pure module function conv2d(window_size, filters, activation) result(res)
+    pure module function conv2d(filters, kernel_size, activation) result(res)
       !! 2-d convolutional layer constructor.
       !!
       !! This layer is for building 2-d convolutional network.
@@ -98,13 +98,13 @@ pure module function conv2d(window_size, filters, activation) result(res)
       !! ```
       !! use nf, only :: conv2d, layer
       !! type(layer) :: conv2d_layer
-      !! conv2d_layer = dense(window_size=3, filters=32)
-      !! conv2d_layer = dense(window_size=3, filters=32, activation='relu')
+      !! conv2d_layer = dense(filters=32, kernel_size=3)
+      !! conv2d_layer = dense(filters=32, kernel_size=3, activation='relu')
       !! ```
-      integer, intent(in) :: window_size
-        !! Width of the convolution window, commonly 3 or 5
       integer, intent(in) :: filters
         !! Number of filters in the output of the layer
+      integer, intent(in) :: kernel_size
+        !! Width of the convolution window, commonly 3 or 5
       character(*), intent(in), optional :: activation
         !! Activation function (default 'sigmoid')
       type(layer) :: res

src/nf_layer_constructors_submodule.f90

@@ -52,9 +52,9 @@ pure module function dense(layer_size, activation) result(res)
   end function dense
 
 
-  pure module function conv2d(window_size, filters, activation) result(res)
-    integer, intent(in) :: window_size
+  pure module function conv2d(filters, kernel_size, activation) result(res)
     integer, intent(in) :: filters
+    integer, intent(in) :: kernel_size
     character(*), intent(in), optional :: activation
     type(layer) :: res
 
@@ -68,7 +68,7 @@ pure module function conv2d(window_size, filters, activation) result(res)
 
     allocate( &
       res % p, &
-      source=conv2d_layer(window_size, filters, res % activation) &
+      source=conv2d_layer(filters, kernel_size, res % activation) &
     )
 
   end function conv2d

src/nf_network.f90

@@ -11,14 +11,22 @@ module nf_network
   public :: network
 
   type :: network
+
     type(layer), allocatable :: layers(:)
+
   contains
+
     procedure :: backward
-    procedure :: forward
     procedure :: output
     procedure :: print_info
     procedure :: train
     procedure :: update
+
+    procedure, private :: forward_1d
+    procedure, private :: forward_3d
+
+    generic :: forward => forward_1d, forward_3d
+
   end type network
 
   interface network
@@ -32,29 +40,50 @@ module function network_cons(layers) result(res)
     end function network_cons
   end interface network
 
-  interface
+  interface forward
 
-    pure module subroutine backward(self, output)
-      !! Apply one backward pass through the network.
+    pure module subroutine forward_1d(self, input)
+      !! Apply a forward pass through the network.
+      !!
       !! This changes the state of layers on the network.
       !! Typically used only internally from the `train` method,
       !! but can be invoked by the user when creating custom optimizers.
+      !!
+      !! This specific subroutine is for 1-d input data.
       class(network), intent(in out) :: self
         !! Network instance
-      real, intent(in) :: output(:)
-        !! Output data
-    end subroutine backward
+      real, intent(in) :: input(:)
+        !! 1-d input data
+    end subroutine forward_1d
 
-    pure module subroutine forward(self, input)
+    pure module subroutine forward_3d(self, input)
       !! Apply a forward pass through the network.
+      !!
       !! This changes the state of layers on the network.
       !! Typically used only internally from the `train` method,
       !! but can be invoked by the user when creating custom optimizers.
+      !!
+      !! This specific subroutine is for 3-d input data.
       class(network), intent(in out) :: self
         !! Network instance
-      real, intent(in) :: input(:)
-        !! Input data
-    end subroutine forward
+      real, intent(in) :: input(:,:,:)
+        !! 3-d input data
+    end subroutine forward_3d
+
+  end interface forward
+
+  interface
+
+    pure module subroutine backward(self, output)
+      !! Apply one backward pass through the network.
+      !! This changes the state of layers on the network.
+      !! Typically used only internally from the `train` method,
+      !! but can be invoked by the user when creating custom optimizers.
+      class(network), intent(in out) :: self
+        !! Network instance
+      real, intent(in) :: output(:)
+        !! Output data
+    end subroutine backward
 
     module function output(self, input) result(res)
       !! Return the output of the network given the input array.

src/nf_network_submodule.f90

@@ -2,6 +2,7 @@
 
   use nf_dense_layer, only: dense_layer
   use nf_input1d_layer, only: input1d_layer
+  use nf_input3d_layer, only: input3d_layer
   use nf_layer, only: layer
   use nf_loss, only: quadratic_derivative
   use nf_optimizers, only: sgd
@@ -80,7 +81,7 @@ pure module subroutine backward(self, output)
   end subroutine backward
 
 
-  pure module subroutine forward(self, input)
+  pure module subroutine forward_1d(self, input)
     class(network), intent(in out) :: self
     real, intent(in) :: input(:)
     integer :: n
@@ -94,7 +95,24 @@ pure module subroutine forward(self, input)
       call self % layers(n) % forward(self % layers(n - 1))
     end do
 
-  end subroutine forward
+  end subroutine forward_1d
+
+
+  pure module subroutine forward_3d(self, input)
+    class(network), intent(in out) :: self
+    real, intent(in) :: input(:,:,:)
+    integer :: n
+
+    ! Set the input array into the input layer
+    select type(input_layer => self % layers(1) % p); type is(input3d_layer)
+      call input_layer % set(input)
+    end select
+
+    do n = 2, size(self % layers)
+      call self % layers(n) % forward(self % layers(n - 1))
+    end do
+
+  end subroutine forward_3d
 
 
   module function output(self, input) result(res)

test/test_conv2d_layer.f90

@@ -7,12 +7,12 @@ program test_conv2d_layer
   implicit none
 
   type(layer) :: conv_layer, input_layer
-  integer, parameter :: window_size = 3, filters = 32
+  integer, parameter :: filters = 32, kernel_size=3
   real, allocatable :: sample_input(:,:,:), output(:,:,:)
   real, parameter :: tolerance = 1e-7
   logical :: ok = .true.
 
-  conv_layer = conv2d(window_size, filters)
+  conv_layer = conv2d(filters, kernel_size)
 
   if (.not. conv_layer % name == 'conv2d') then
     ok = .false.
@@ -52,7 +52,7 @@ program test_conv2d_layer
   sample_input = 0
 
   input_layer = input([1, 3, 3])
-  conv_layer = conv2d(window_size, filters)
+  conv_layer = conv2d(filters, kernel_size)
   call conv_layer % init(input_layer)
 
   select type(this_layer => input_layer % p); type is(input3d_layer)