assign_sub, smart_cond

Author: Oceania2018

src/TensorFlowNET.Core/APIs/tf.layers.cs

@@ -100,6 +100,32 @@ public static Tensor batch_normalization(Tensor inputs,
 
                 return layer.apply(inputs, training: training);
             }
+
+            /// <summary>
+            /// Max pooling layer for 2D inputs (e.g. images).
+            /// </summary>
+            /// <param name="inputs">The tensor over which to pool. Must have rank 4.</param>
+            /// <param name="pool_size"></param>
+            /// <param name="strides"></param>
+            /// <param name="padding"></param>
+            /// <param name="data_format"></param>
+            /// <param name="name"></param>
+            /// <returns></returns>
+            public static Tensor max_pooling2d(Tensor inputs,
+                int[] pool_size,
+                int[] strides,
+                string padding = "valid",
+                string data_format = "channels_last",
+                string name = null)
+            {
+                var layer = new MaxPooling2D(pool_size: pool_size,
+                    strides: strides,
+                    padding: padding,
+                    data_format: data_format,
+                    name: name);
+
+                return layer.apply(inputs);
+            }
         }
     }
 }

src/TensorFlowNET.Core/APIs/tf.nn.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Text;
+using Tensorflow.Operations;
 using Tensorflow.Operations.Activation;
 
 namespace Tensorflow
@@ -27,19 +28,21 @@ public static Tensor embedding_lookup(RefVariable @params,
 
             public static IActivation relu => new relu();
 
-            public static (Tensor, Tensor, Tensor) fused_batch_norm(Tensor x,
-            RefVariable scale,
-            RefVariable offset,
-            Tensor mean = null,
-            Tensor variance = null,
-            float epsilon = 0.001f,
-            string data_format = "NHWC",
-            bool is_training = true,
-            string name = null) => nn_impl.fused_batch_norm(x, scale, offset, mean, variance,
-                epsilon: epsilon,
-                data_format: data_format,
-                is_training: is_training,
-                name: name);
+            public static Tensor[] fused_batch_norm(Tensor x,
+                RefVariable scale,
+                RefVariable offset,
+                Tensor mean = null,
+                Tensor variance = null,
+                float epsilon = 0.001f,
+                string data_format = "NHWC",
+                bool is_training = true,
+                string name = null) => nn_impl.fused_batch_norm(x, scale, offset, mean, variance,
+                    epsilon: epsilon,
+                    data_format: data_format,
+                    is_training: is_training,
+                    name: name);
+
+            public static Tensor max_pool() => gen_nn_ops.max_pool();
         }
     }
 }

src/TensorFlowNET.Core/Framework/smart_module.cs

@@ -6,9 +6,9 @@ namespace Tensorflow.Framework
 {
     public class smart_module
     {
-        public static object smart_cond(Tensor pred, 
-            Func<(Tensor, Tensor, Tensor)> true_fn = null, 
-            Func<(Tensor, Tensor, Tensor)> false_fn = null, 
+        public static Tensor[] smart_cond<T>(Tensor pred, 
+            Func<T[]> true_fn = null, 
+            Func<T[]> false_fn = null, 
             string name = null)
         {
             return control_flow_ops.cond(pred,
@@ -17,9 +17,12 @@ public static object smart_cond(Tensor pred,
                 name: name);
         }
 
-        public static bool smart_constant_value(Tensor pred)
+        public static bool? smart_constant_value(Tensor pred)
         {
             var pred_value = tensor_util.constant_value(pred);
+            if (pred_value is null)
+                return null;
+
             return pred_value;
         }
     }

src/TensorFlowNET.Core/Keras/Engine/Layer.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Text;
 using Tensorflow.Keras.Utils;
 
@@ -34,6 +35,7 @@ public class Layer : CheckpointableBase
         protected string _name;
         protected string _base_name;
         protected bool _compute_previous_mask;
+        protected List<Operation> _updates;
 
         public Layer(bool trainable = true, string name = null, TF_DataType dtype = TF_DataType.DtInvalid)
         {
@@ -45,6 +47,7 @@ public Layer(bool trainable = true, string name = null, TF_DataType dtype = TF_D
             _init_set_name(name);
             _trainable_weights = new List<RefVariable>();
             _compute_previous_mask = false;
+            _updates = new List<Operation>();
         }
 
         public Tensor __call__(Tensor inputs,
@@ -142,6 +145,12 @@ protected virtual RefVariable add_weight(string name,
             return variable;
         }
 
+        protected virtual void add_update(Tensor[] updates, bool inputs = false)
+        {
+            var updates_op = updates.Select(x => x.op).ToArray();
+            _updates.AddRange(updates_op);
+        }
+
         protected virtual void _init_set_name(string name)
         {
             string base_name = name;

src/TensorFlowNET.Core/Keras/Layers/BatchNormalization.cs

@@ -132,6 +132,7 @@ protected override Tensor call(Tensor inputs, Tensor training = null)
             if (fused)
             {
                 outputs = _fused_batch_norm(inputs, training: training);
+                return outputs;
             }
 
             throw new NotImplementedException("BatchNormalization call");
@@ -142,7 +143,7 @@ private Tensor _fused_batch_norm(Tensor inputs, Tensor training)
             var beta = this.beta;
             var gamma = this.gamma;
 
-            Func<(Tensor, Tensor, Tensor)> _fused_batch_norm_training = () =>
+            Func<Tensor[]> _fused_batch_norm_training = () =>
             {
                 return tf.nn.fused_batch_norm(
                   inputs,
@@ -152,7 +153,7 @@ private Tensor _fused_batch_norm(Tensor inputs, Tensor training)
                   data_format: _data_format);
             };
 
-            Func<(Tensor, Tensor, Tensor)> _fused_batch_norm_inference = () =>
+            Func<Tensor[]> _fused_batch_norm_inference = () =>
             {
                 return tf.nn.fused_batch_norm(
                   inputs,
@@ -165,9 +166,41 @@ private Tensor _fused_batch_norm(Tensor inputs, Tensor training)
                   data_format: _data_format);
             };
 
-            tf_utils.smart_cond(training, _fused_batch_norm_training, _fused_batch_norm_inference);
+            var results = tf_utils.smart_cond(training, _fused_batch_norm_training, _fused_batch_norm_inference);
+            var (output, mean, variance) = (results[0], results[1], results[2]);
+            var training_value = tf_utils.constant_value(training);
 
-            throw new NotImplementedException("_fused_batch_norm");
+            Tensor momentum_tensor;
+            if (training_value == null)
+            {
+                momentum_tensor = tf_utils.smart_cond(training,
+                    () => new float[] { momentum }, () => new float[] { 1.0f })[0];
+            }
+            else
+            {
+                momentum_tensor = ops.convert_to_tensor(momentum);
+            }
+                
+            if(training_value == null)
+            {
+                var mean_update = _assign_moving_average(moving_mean, mean, momentum_tensor);
+                var variance_update = _assign_moving_average(moving_variance, variance, momentum_tensor);
+                add_update(new Tensor[] { mean_update }, inputs: true);
+                add_update(new Tensor[] { variance_update }, inputs: true);
+            }
+
+            return output;
+        }
+
+        public Tensor _assign_moving_average(RefVariable variable, Tensor value, Tensor momentum)
+        {
+            return Python.with(ops.name_scope(null, "AssignMovingAvg", new { variable, value, momentum }), scope =>
+            {
+                // var cm = ops.colocate_with(variable);
+                var decay = ops.convert_to_tensor(1.0f - momentum, name: "decay");
+                var update_delta = (variable - math_ops.cast(value, variable.dtype)) * decay;
+                return state_ops.assign_sub(variable, update_delta, name: scope);
+            });
         }
     }
 }

src/TensorFlowNET.Core/Keras/Layers/MaxPooling2D.cs

@@ -0,0 +1,24 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using static Tensorflow.tf;
+
+namespace Tensorflow.Keras.Layers
+{
+    public class MaxPooling2D : Pooling2D
+    {
+        public MaxPooling2D(
+            int[] pool_size,
+            int[] strides,
+            string padding = "valid",
+            string data_format = null,
+            string name = null) : base(nn.max_pool, pool_size,
+                strides,
+                padding: padding,
+                data_format: data_format,
+                name: name)
+        {
+
+        }
+    }
+}

src/TensorFlowNET.Core/Keras/Layers/Pooling2D.cs

@@ -0,0 +1,33 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Tensorflow.Keras.Engine;
+using Tensorflow.Keras.Utils;
+
+namespace Tensorflow.Keras.Layers
+{
+    public class Pooling2D : Tensorflow.Layers.Layer
+    {
+        private Func<Tensor> pool_function;
+        private int[] pool_size;
+        private int[] strides;
+        private string padding;
+        private string data_format;
+        private InputSpec input_spec;
+
+        public Pooling2D(Func<Tensor> pool_function,
+            int[] pool_size,
+            int[] strides,
+            string padding = "valid",
+            string data_format = null,
+            string name = null) : base(name: name)
+        {
+            this.pool_function = pool_function;
+            this.pool_size = conv_utils.normalize_tuple(pool_size, 2, "pool_size");
+            this.strides = conv_utils.normalize_tuple(strides, 2, "strides");
+            this.padding = conv_utils.normalize_padding(padding);
+            this.data_format = conv_utils.normalize_data_format(data_format);
+            this.input_spec = new InputSpec(ndim: 4);
+        }
+    }
+}

src/TensorFlowNET.Core/Keras/Utils/conv_utils.cs

@@ -29,5 +29,20 @@ public static string convert_data_format(string data_format, int ndim)
             else
                 throw new ValueError($"Invalid data_format: {data_format}");
         }
+
+        public static int[] normalize_tuple(int[] value, int n, string name)
+        {
+            return value;
+        }
+
+        public static string normalize_padding(string value)
+        {
+            return value.ToLower();
+        }
+
+        public static string normalize_data_format(string value)
+        {
+            return value.ToLower();
+        }
     }
 }

src/TensorFlowNET.Core/Keras/Utils/tf_utils.cs

@@ -13,14 +13,19 @@ public static bool are_all_symbolic_tensors(Tensor[] tensors)
             return tensors.Select(x => is_symbolic_tensor(x)).Count() == tensors.Length;
         }
 
+        public static bool? constant_value(Tensor pred)
+        {
+            return smart_module.smart_constant_value(pred);
+        }
+
         public static bool is_symbolic_tensor(Tensor tensor)
         {
             return true;
         }
 
-        public static object smart_cond(Tensor pred, 
-            Func<(Tensor, Tensor, Tensor)> true_fn = null,
-            Func<(Tensor, Tensor, Tensor)> false_fn = null, 
+        public static Tensor[] smart_cond<T>(Tensor pred, 
+            Func<T[]> true_fn = null,
+            Func<T[]> false_fn = null, 
             string name = null)
         {
             return smart_module.smart_cond(pred,

src/TensorFlowNET.Core/Layers/Layer.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Text;
 using Tensorflow.Keras.Engine;
 
@@ -55,11 +56,23 @@ public Tensor __call__(Tensor inputs,
             var outputs = base.__call__(inputs, training: training);
 
             // Update global default collections.
-            //_add_elements_to_collection(updates, ops.GraphKeys.UPDATE_OPS);
+            _add_elements_to_collection(_updates.ToArray(), new string[] { ops.GraphKeys.UPDATE_OPS });
 
             return outputs;
         }
 
+        protected virtual void _add_elements_to_collection(Operation[] elements, string[] collection_list)
+        {
+            foreach(var name in collection_list)
+            {
+                var collection = ops.get_collection_ref(name) as List<object>;
+
+                foreach (var element in elements)
+                    if (!collection.Contains(element))
+                        collection.Add(element);
+            }
+        }
+
         protected virtual RefVariable add_weight(string name,
             int[] shape,
             TF_DataType dtype = TF_DataType.DtInvalid,