Add node to connectivity between two layers.

Author: Oceania2018

src/TensorFlowNET.Core/APIs/keras.layers.cs

@@ -18,7 +18,7 @@ public static Embedding Embedding(int input_dim, int output_dim,
                     embeddings_initializer,
                     mask_zero);
 
-            public static InputLayer Input(int[] batch_shape = null,
+            public static Tensor[] Input(int[] batch_shape = null,
                 TF_DataType dtype = TF_DataType.DtInvalid,
                 string name = null,
                 bool sparse = false,
@@ -35,7 +35,9 @@ public static InputLayer Input(int[] batch_shape = null,
                     sparse: sparse,
                     input_tensor: tensor);
 
-                throw new NotImplementedException("");
+                var outputs = input_layer.inbound_nodes[0].output_tensors;
+
+                return outputs;
             }
         }
     }

src/TensorFlowNET.Core/Keras/Engine/Sequential.cs

@@ -33,6 +33,12 @@ public void add(Layer layer)
                       batch_shape: batch_shape,
                       dtype: dtype,
                       name: layer._name + "_input");
+
+                    // This will build the current layer
+                    // and create the node connecting the current layer
+                    // to the input layer we just created.
+                    layer.__call__(x);
+                    set_inputs = true;
                 }
             }
         }

src/TensorFlowNET.Core/Keras/Layers/Embedding.cs

@@ -9,6 +9,8 @@ public class Embedding : Layer
         private int input_dim;
         private int output_dim;
         private bool mask_zero;
+        public RefVariable embeddings;
+        public IInitializer embeddings_initializer;
 
         public Embedding(int input_dim, int output_dim,
             IInitializer embeddings_initializer = null,
@@ -18,10 +20,17 @@ public Embedding(int input_dim, int output_dim,
         {
             this.input_dim = input_dim;
             this.output_dim = output_dim;
-            if (embeddings_initializer == null)
-                embeddings_initializer = tf.uniform_initializer;
+            this.embeddings_initializer = embeddings_initializer == null ? tf.uniform_initializer : embeddings_initializer;
             this.mask_zero = mask_zero;
             supports_masking = mask_zero;
         }
+
+        protected override void build(TensorShape input_shape)
+        {
+            embeddings = add_weight(shape: new int[] { input_dim, output_dim },
+                initializer: embeddings_initializer,
+                name: "embeddings");
+            built = true;
+        }
     }
 }

src/TensorFlowNET.Core/Keras/Layers/InputLayer.cs

@@ -11,6 +11,7 @@ public class InputLayer : Layer
     {
         public bool sparse;
         public int? batch_size;
+        public bool is_placeholder;
 
         public InputLayer(int[] input_shape = null,
             int? batch_size = null,
@@ -24,7 +25,7 @@ public InputLayer(int[] input_shape = null,
             this.batch_size = batch_size;
             this.supports_masking = true;
 
-            if(input_tensor == null)
+            if (input_tensor == null)
             {
                 var batch_input_shape = new int[] { batch_size.HasValue ? batch_size.Value : -1, -1 };
 
@@ -39,7 +40,17 @@ public InputLayer(int[] input_shape = null,
                           dtype: dtype,
                           name: name);
                 }
+
+                is_placeholder = true;
+                _batch_input_shape = batch_input_shape;
             }
+
+            new Node(this,
+                inbound_layers: new Layer[0],
+                node_indices: new int[0],
+                tensor_indices: new int[0],
+                input_tensors: new Tensor[] { input_tensor },
+                output_tensors: new Tensor[] { input_tensor });
         }
     }
 }

src/TensorFlowNET.Core/Keras/Layers/Layer.cs

@@ -39,6 +39,12 @@ public class Layer : CheckpointableBase
         protected List<Operation> _updates;
         public int[] _batch_input_shape;
 
+        private List<Node> _inbound_nodes;
+        public List<Node> inbound_nodes => _inbound_nodes;
+
+        private List<Node> _outbound_nodes;
+        public List<Node> outbound_nodes => _outbound_nodes;
+
         public Layer(bool trainable = true, 
             string name = null, 
             TF_DataType dtype = TF_DataType.DtInvalid,
@@ -59,13 +65,15 @@ public Layer(bool trainable = true,
             _batch_input_shape = new int[] { -1, -1 };
 
             _dtype = dtype;
+
+            _inbound_nodes = new List<Node>();
         }
 
-        public Tensor __call__(Tensor inputs,
+        public Tensor __call__(Tensor[] inputs,
             Tensor training = null,
             VariableScope scope = null)
         {
-            var input_list = new Tensor[] { inputs };
+            var input_list = inputs;
             Tensor outputs = null;
 
             // We will attempt to build a TF graph if & only if all inputs are symbolic.
@@ -88,9 +96,9 @@ public Tensor __call__(Tensor inputs,
                     // Symbolic execution on symbolic tensors. We will attempt to build
                     // the corresponding TF subgraph inside `backend.get_graph()`
                     var graph = backend.get_graph();
-                    outputs = call(inputs, training: training);
-                    _handle_activity_regularization(inputs, outputs);
-                    _set_mask_metadata(inputs, outputs, null);
+                    outputs = call(inputs[0], training: training);
+                    _handle_activity_regularization(inputs[0], outputs);
+                    _set_mask_metadata(inputs[0], outputs, null);
                 }
             });
 
@@ -125,10 +133,10 @@ protected virtual string _name_scope()
             return null;
         }
 
-        protected void _maybe_build(Tensor inputs)
+        protected void _maybe_build(Tensor[] inputs)
         {
-            var input_list = new Tensor[] { inputs };
-            build(inputs.getShape());
+            var input_list = inputs;
+            build(input_list[0].getShape());
         }
 
         protected virtual void build(TensorShape input_shape)
@@ -143,10 +151,16 @@ protected virtual RefVariable add_weight(string name,
             bool? trainable = null,
             Func<string, int[], TF_DataType, IInitializer, bool, RefVariable> getter = null)
         {
+            if (dtype == TF_DataType.DtInvalid)
+                dtype = TF_DataType.TF_FLOAT;
+
+            if (trainable == null)
+                trainable = true;
+
             var variable = _add_variable_with_custom_getter(name,
                 shape,
                 dtype: dtype,
-                getter: getter,
+                getter: getter == null ? base_layer_utils.make_variable : getter,
                 overwrite: true,
                 initializer: initializer,
                 trainable: trainable.Value);

src/TensorFlowNET.Core/Keras/Layers/Node.cs

@@ -0,0 +1,71 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace Tensorflow.Keras.Layers
+{
+    /// <summary>
+    /// A `Node` describes the connectivity between two layers.
+    /// </summary>
+    public class Node
+    {
+        public InputLayer outbound_layer;
+        public Layer[] inbound_layers;
+        public int[] node_indices;
+        public int[] tensor_indices;
+        public Tensor[] input_tensors;
+        public Tensor[] output_tensors;
+        public int[][] input_shapes;
+        public int[][] output_shapes;
+
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <param name="outbound_layer">
+        /// the layer that takes
+        /// `input_tensors` and turns them into `output_tensors`
+        /// (the node gets created when the `call`
+        /// method of the layer was called).
+        /// </param>
+        /// <param name="inbound_layers">
+        /// a list of layers, the same length as `input_tensors`,
+        /// the layers from where `input_tensors` originate.
+        /// </param>
+        /// <param name="node_indices">
+        /// a list of integers, the same length as `inbound_layers`.
+        /// `node_indices[i]` is the origin node of `input_tensors[i]`
+        /// (necessary since each inbound layer might have several nodes,
+        /// e.g. if the layer is being shared with a different data stream).
+        /// </param>
+        /// <param name="tensor_indices"></param>
+        /// <param name="input_tensors">list of input tensors.</param>
+        /// <param name="output_tensors">list of output tensors.</param>
+        public Node(InputLayer outbound_layer, 
+            Layer[] inbound_layers, 
+            int[] node_indices,
+            int[] tensor_indices,
+            Tensor[] input_tensors,
+            Tensor[] output_tensors)
+        {
+            this.outbound_layer = outbound_layer;
+            this.inbound_layers = inbound_layers;
+            this.node_indices = node_indices;
+            this.tensor_indices = tensor_indices;
+            this.input_tensors = input_tensors;
+            this.output_tensors = output_tensors;
+
+            input_shapes = input_tensors.Select(x => x._shape_tuple()).ToArray();
+            output_shapes = output_tensors.Select(x => x._shape_tuple()).ToArray();
+
+            // Add nodes to all layers involved.
+            foreach (var layer in inbound_layers)
+            {
+                if (layer != null)
+                    layer.outbound_nodes.Add(this);
+            }
+
+            outbound_layer.inbound_nodes.Add(this);
+        }
+    }
+}

src/TensorFlowNET.Core/Keras/Engine/base_layer_utils.cs renamed to src/TensorFlowNET.Core/Keras/Utils/base_layer_utils.cs

@@ -2,10 +2,19 @@
 using System.Collections.Generic;
 using System.Text;
 
-namespace Tensorflow.Keras.Engine
+namespace Tensorflow.Keras.Utils
 {
     public class base_layer_utils
     {
+        public static RefVariable make_variable(string name,
+            int[] shape,
+            TF_DataType dtype = TF_DataType.TF_FLOAT,
+            IInitializer initializer = null,
+            bool trainable = false)
+        {
+            throw new NotImplementedException("");
+        }
+
         /// <summary>
         /// Makes a layer name (or arbitrary string) unique within a TensorFlow graph.
         /// </summary>

src/TensorFlowNET.Core/Layers/Layer.cs

@@ -52,7 +52,7 @@ public Tensor __call__(Tensor inputs,
 
             Python.with(scope_context_manager, scope2 => _current_scope = scope2);
             // Actually call layer
-            var outputs = base.__call__(inputs, training: training);
+            var outputs = base.__call__(new Tensor[] { inputs }, training: training);
 
             // Update global default collections.
             _add_elements_to_collection(_updates.ToArray(), new string[] { ops.GraphKeys.UPDATE_OPS });

src/TensorFlowNET.Core/TensorFlowNET.Core.csproj

@@ -43,7 +43,7 @@ Docs: https://tensorflownet.readthedocs.io</Description>
 
   <ItemGroup>
     <PackageReference Include="Google.Protobuf" Version="3.7.0" />
-    <PackageReference Include="NumSharp" Version="0.7.4" />
+    <PackageReference Include="NumSharp" Version="0.8.0" />
   </ItemGroup>
 
   <ItemGroup>

test/TensorFlowNET.Examples/TensorFlowNET.Examples.csproj

@@ -6,7 +6,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="NumSharp" Version="0.7.4" />
+    <PackageReference Include="NumSharp" Version="0.8.0" />
     <PackageReference Include="SharpZipLib" Version="1.1.0" />
     <PackageReference Include="TensorFlow.NET" Version="0.4.2" />
   </ItemGroup>