tf.keras.layers #355

Author: Oceania2018

src/TensorFlowNET.Core/Gradients/GradientTape.cs

@@ -107,19 +107,19 @@ public Tensor gradient(Tensor target, Tensor source)
 
         public Tensor gradient(Tensor target, ResourceVariable source)
         {
-            var results = gradient(target, new[] { source });
+            var results = gradient(target, new List<IVariableV1> { source });
 
             return results[0];
         }
 
         public (Tensor, Tensor) gradient(Tensor target, (ResourceVariable, ResourceVariable) sources)
         {
-            var results = gradient(target, new[] { sources.Item1, sources.Item2 });
+            var results = gradient(target, new List<IVariableV1> { sources.Item1, sources.Item2 });
 
             return (results[0], results[1]);
         }
 
-        public Tensor[] gradient(Tensor target, IEnumerable<IVariableV1> sources)
+        public Tensor[] gradient(Tensor target, List<IVariableV1> sources)
         {
             if (_recording)
             {

src/TensorFlowNET.Core/Gradients/Tape.ComputeGradient.cs

@@ -54,7 +54,16 @@ public Tensor[] ComputeGradient(long[] target_tensor_ids,
                     var id = trace.output_tensor_info[i].GetID();
                     if (!gradients.find(id, out var grad_it))
                     {
-                        throw new NotImplementedException("FunctionsAcceptingNoneForIndicesMap");
+                        if (FunctionsAcceptingNoneForIndicesMap().find(trace.op_type, out var func_name_it) &&
+                            func_name_it.find(i))
+                        {
+                            out_gradients.Add(null);
+                        }
+                        else
+                        {
+                            out_gradients.Add(null);
+                            zero_indices.Add(i);
+                        }
                     }
                     else
                     {
@@ -184,6 +193,15 @@ public Tensor[] ComputeGradient(long[] target_tensor_ids,
             return result.ToArray();
         }
 
+        UnorderedMap<string, UnorderedSet<int>> FunctionsAcceptingNoneForIndicesMap()
+        {
+            var m = new UnorderedMap<string, UnorderedSet<int>>();
+            m.Add("SoftmaxCrossEntropyWithLogits", new UnorderedSet<int>(new[] { 1 }));
+            m.Add("SparseSoftmaxCrossEntropyWithLogits", new UnorderedSet<int>(new[] { 1 }));
+            m.Add("FusedBatchNorm", new UnorderedSet<int>(new[] { 1, 2, 3, 4 }));
+            return m;
+        }
+
         UnorderedMapEnumerable<long, List<Tensor>> InitialGradients(long[] target_tensor_ids,
             UnorderedMap<long, TapeTensor> sources_that_are_targets,
             Tensor[] output_gradients,

src/TensorFlowNET.Core/Keras/Activations.cs renamed to src/TensorFlowNET.Core/Keras/Activations/Activations.Linear.cs

@@ -5,13 +5,11 @@
 
 namespace Tensorflow.Keras
 {
-    public delegate Tensor Activation(Tensor x);
-
-    public class Activations
+    public partial class Activations
     {
         /// <summary>
         /// Linear activation function (pass-through).
         /// </summary>
-        public Activation Linear = x => x;
+        public Activation Linear = (features, name) => features;
     }
 }

src/TensorFlowNET.Core/Keras/Activations/Activations.Relu.cs

@@ -0,0 +1,26 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Tensorflow.Operations;
+using static Tensorflow.Binding;
+
+namespace Tensorflow.Keras
+{
+    public partial class Activations
+    {
+        public Activation Relu = (features, name) =>
+        {
+            if (tf.executing_eagerly())
+            {
+                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
+                    "Relu", name,
+                    null,
+                    features);
+
+                return results[0];
+            }
+
+            throw new NotImplementedException("");
+        };
+    }
+}

src/TensorFlowNET.Core/Keras/Activations/Activations.cs

@@ -0,0 +1,9 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using static Tensorflow.Binding;
+
+namespace Tensorflow.Keras
+{
+    public delegate Tensor Activation(Tensor features, string name = null);
+}

src/TensorFlowNET.Core/Keras/ArgsDefinition/NodeArgs.cs

@@ -11,7 +11,7 @@ public class NodeArgs
         public Layer[] InboundLayers { get; set; }
         public int[] NodeIndices { get; set; }
         public int[] TensorIndices { get; set; }
-        public Tensor[] InputTensors { get; set; }
-        public Tensor[] Outputs { get; set; }
+        public Tensor InputTensors { get; set; }
+        public Tensor Outputs { get; set; }
     }
 }

src/TensorFlowNET.Core/Keras/Engine/Layer.Layers.cs

@@ -0,0 +1,29 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Tensorflow.Keras.ArgsDefinition;
+using Tensorflow.Keras.Layers;
+using static Tensorflow.Binding;
+
+namespace Tensorflow.Keras.Engine
+{
+    public partial class Layer
+    {
+        protected List<Layer> _layers = new List<Layer>();
+
+        protected Layer Dense(int units,
+            Activation activation = null,
+            TensorShape input_shape = null)
+        {
+            var layer = new Dense(new DenseArgs
+            {
+                Units = units,
+                Activation = activation ?? tf.keras.activations.Linear,
+                InputShape = input_shape
+            });
+
+            _layers.Add(layer);
+            return layer;
+        }
+    }
+}

src/TensorFlowNET.Core/Keras/Engine/Layer.cs

@@ -18,11 +18,9 @@ limitations under the License.
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
-using Tensorflow.Contexts;
 using Tensorflow.Keras.ArgsDefinition;
 using Tensorflow.Keras.Layers;
 using Tensorflow.Keras.Utils;
-using Tensorflow.Operations.Activation;
 using Tensorflow.Train;
 using static Tensorflow.Binding;
 
@@ -34,7 +32,7 @@ namespace Tensorflow.Keras.Engine
     /// as convolution, batch norm, etc. These operations require managing weights,
     /// losses, updates, and inter-layer connectivity.
     /// </summary>
-    public abstract class Layer : AutoTrackable
+    public abstract partial class Layer : AutoTrackable
     {
         /// <summary>
         /// Arguments initialize layer.
@@ -60,8 +58,19 @@ public abstract class Layer : AutoTrackable
         protected InputSpec inputSpec;
         public bool SupportsMasking { get; set; }
         protected List<IVariableV1> trainableWeights;
-        public List<IVariableV1> TrainableVariables => trainableWeights;
+        public List<IVariableV1> trainable_variables
+        { 
+            get 
+            {
+                if(trainableWeights.Count == 0)
+                    _layers.ForEach(x => trainableWeights.AddRange(x.trainableWeights));
+
+                return trainableWeights;
+            } 
+        } 
+
         protected List<IVariableV1> nonTrainableWeights;
+        public List<IVariableV1> non_trainable_variables => nonTrainableWeights;
 
         string name;
         public string Name => name;
@@ -112,20 +121,20 @@ public Layer(LayerArgs args)
         /// <param name="input"></param>
         /// <param name="is_training"></param>
         /// <returns></returns>
-        public Tensor[] Apply(Tensor[] inputs, bool is_training = false)
+        public Tensor Apply(Tensor inputs, bool is_training = false)
         {
-            var input = inputs[0];
-            Tensor[] outputs = null;
+            Tensor outputs = null;
 
             callContext = callContext ?? new ThreadLocal<CallContext>()
             {
                 Value = new CallContext()
             };
 
+            var eager = tf.executing_eagerly();
             using var ctxManager = CallContext.enter();
 
             string nameScope = "";
-            if (tf.executing_eagerly())
+            if (eager)
             {
                 nameScope = name;
             }
@@ -134,7 +143,7 @@ public Tensor[] Apply(Tensor[] inputs, bool is_training = false)
                 throw new NotImplementedException("");
             }
 
-            using var graph = tf.keras.backend.get_graph().as_default();
+            // using var graph = tf.keras.backend.get_graph().as_default();
 
             tf_with(ops.name_scope(nameScope), scope =>
             {
@@ -143,82 +152,44 @@ public Tensor[] Apply(Tensor[] inputs, bool is_training = false)
 
                 outputs = call(inputs, is_training: is_training);
 
-                (input, outputs) = _set_connectivity_metadata_(input, outputs);
-                _handle_activity_regularization(inputs[0], outputs);
-                _set_mask_metadata(inputs[0], outputs, null);
+                outputs = _set_connectivity_metadata_(inputs, outputs);
+                _handle_activity_regularization(inputs, outputs);
+                _set_mask_metadata(inputs, outputs, null);
             });
 
             return outputs;
         }
 
-        [Obsolete("User Apply()")]
-        public Tensor[] __call__(Tensor[] inputs,
-            Tensor training = null,
-            Tensor state = null,
-            VariableScope scope = null)
+        private Tensor _set_connectivity_metadata_(Tensor inputs, Tensor outputs)
         {
-            var input_list = inputs;
-            var input = inputs[0];
-            Tensor[] outputs = null;
-
-            // We will attempt to build a TF graph if & only if all inputs are symbolic.
-            // This is always the case in graph mode. It can also be the case in eager
-            // mode when all inputs can be traced back to `keras.Input()` (when building
-            // models using the functional API).
-            bool build_graph = tf_utils.are_all_symbolic_tensors(input_list);
-
-            if (build_graph)
-            {
-                // Only create Keras history if at least one tensor originates from a
-                // `keras.Input`. Otherwise this Layer may be being used outside the Keras
-                // framework.
-                // base_layer_utils.create_keras_history(inputs)
-            }
-
-            // with base_layer_utils.call_context(self):
-
-            // Handle Keras mask propagation from previous layer to current layer.
-            // with base_layer_utils.call_context(self):
-            // Check input assumptions set after layer building, e.g. input shape.
-            if (build_graph)
+            /*var returnOutputs = new List<Tensor>();
+            foreach(var x in outputs)
             {
-                // Symbolic execution on symbolic tensors. We will attempt to build
-                // the corresponding TF subgraph inside `backend.get_graph()`
-                var graph = tf.keras.backend.get_graph().as_default();
-                tf_with(ops.name_scope(_name_scope()), delegate
+                if (inputs.Contains(x))
                 {
-                    // Build layer if applicable (if the `build` method has been
-                    // overridden).
-                    MaybeBuild(inputs);
-
-                    outputs = call(inputs, 
-                        // training: training,
-                        state: state);
 
-                    (input, outputs) = _set_connectivity_metadata_(input, outputs);
-                    _handle_activity_regularization(inputs[0], outputs);
-                    _set_mask_metadata(inputs[0], outputs, null);
-                });
-            }
+                }
+                returnOutputs.Add(x);
+            }*/
 
-            return outputs;
-        }
+            new Node(this, new NodeArgs
+            {
+                Outputs = outputs
+            });
 
-        private (Tensor, Tensor[]) _set_connectivity_metadata_(Tensor inputs, Tensor[] outputs)
-        {
             //_add_inbound_node(input_tensors: inputs, output_tensors: outputs);
-            return (inputs, outputs);
+            return outputs;
         }
 
-        private void _handle_activity_regularization(Tensor inputs, Tensor[] outputs)
+        private void _handle_activity_regularization(Tensor inputs, Tensor outputs)
         {
             //if(_activity_regularizer != null)
             {
 
             }
         }
 
-        private void _set_mask_metadata(Tensor inputs, Tensor[] outputs, Tensor previous_mask)
+        private void _set_mask_metadata(Tensor inputs, Tensor outputs, Tensor previous_mask)
         {
 
         }
@@ -228,7 +199,7 @@ private Tensor compute_mask(Tensor inputs, Tensor mask = null)
             return null;
         }
 
-        protected virtual Tensor[] call(Tensor[] inputs, bool is_training = false, Tensor state = null)
+        protected virtual Tensor call(Tensor inputs, bool is_training = false, Tensor state = null)
         {
             throw new NotImplementedException("");
         }
@@ -238,15 +209,15 @@ protected virtual string _name_scope()
             return Name;
         }
 
-        protected void MaybeBuild(Tensor[] inputs)
+        protected void MaybeBuild(Tensor inputs)
         {
             // Check input assumptions set before layer building, e.g. input rank.
             if (built)
                 return;
             if (DType == TF_DataType.DtInvalid)
-                args.DType = inputs[0].dtype;
+                args.DType = inputs.dtype;
 
-            var input_shapes = inputs[0].TensorShape;
+            var input_shapes = inputs.TensorShape;
             build(input_shapes);
             built = true;
         }

src/TensorFlowNET.Core/Keras/Engine/Node.cs

@@ -35,13 +35,13 @@ public class Node
 
         public int[] node_indices;
         public int[] tensor_indices;
-        public Tensor[] input_tensors;
-        public Tensor[] Outputs => args.Outputs;
+        public Tensor input_tensors;
+        public Tensor Outputs => args.Outputs;
         public TensorShape[] input_shapes;
         public TensorShape[] output_shapes;
         List<Layer> kerasInputs;
 
-        public Node(InputLayer layer, NodeArgs args)
+        public Node(Layer layer, NodeArgs args)
         {
             this.args = args;