math.reduce_sum, tf.variables_initializer

Author: Oceania2018

src/TensorFlowNET.Core/APIs/tf.array.cs

@@ -17,7 +17,9 @@ limitations under the License.
 using NumSharp;
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
+using static Tensorflow.Binding;
 
 namespace Tensorflow
 {
@@ -76,7 +78,14 @@ public Tensor check_numerics(Tensor tensor, string message, string name = null)
         public Tensor concat(IList<Tensor> values, int axis, string name = "concat")
         {
             if (values.Count == 1)
-                throw new NotImplementedException("tf.concat length is 1");
+            {
+                return tf_with(ops.name_scope(name), scope =>
+                {
+                    var tensor = ops.convert_to_tensor(axis, name: "concat_dim", dtype: dtypes.int32);
+                    Debug.Assert(tensor.TensorShape.ndim == 0);
+                    return identity(values[0], name: scope);
+                });
+            }
 
             return gen_array_ops.concat_v2(values.ToArray(), axis, name: name);
         }
@@ -111,7 +120,7 @@ public Tensor fill<T>(Tensor dims, T value, string name = null)
         /// <param name="input"></param>
         /// <param name="name"></param>
         /// <returns></returns>
-        public static Tensor identity(Tensor input, string name = null)
+        public Tensor identity(Tensor input, string name = null)
             => array_ops.identity(input, name: name);
 
         /// <summary>
@@ -150,10 +159,10 @@ public Tensor transpose<T1>(T1 a, int[] perm = null, string name = "transpose",
         /// <param name="axis"></param>
         /// <param name="name"></param>
         /// <returns></returns>
-        public static Tensor reverse(Tensor tensor, int[] axis, string name = null)
+        public Tensor reverse(Tensor tensor, int[] axis, string name = null)
             => gen_array_ops.reverse(tensor, axis, name: name);
 
-        public static Tensor reverse(Tensor tensor, Tensor axis, string name = null)
+        public Tensor reverse(Tensor tensor, Tensor axis, string name = null)
             => gen_array_ops.reverse(tensor, axis, name: name);
 
         /// <summary>
@@ -277,5 +286,14 @@ public Tensor[] unstack(Tensor value, int? num = null, int axis = 0, string name
         /// <returns>A `Tensor` with all elements set to zero.</returns>
         public Tensor zeros_like(Tensor tensor, TF_DataType dtype = TF_DataType.DtInvalid, string name = null, bool optimize = true)
             => array_ops.zeros_like(tensor, dtype: dtype, name: name, optimize: optimize);
+
+        /// <summary>
+        /// Stops gradient computation.
+        /// </summary>
+        /// <param name="x"></param>
+        /// <param name="name"></param>
+        /// <returns></returns>
+        public Tensor stop_gradient(Tensor x, string name = null)
+            => gen_array_ops.stop_gradient(x, name: name);
     }
 }

src/TensorFlowNET.Core/APIs/tf.math.cs

@@ -434,11 +434,14 @@ public Tensor reduce_prod(Tensor input_tensor, int[] axis = null, bool keepdims
         public Tensor reduce_sum(Tensor input, int? axis = null, int? reduction_indices = null, 
             bool keepdims = false, string name = null)
         {
-            if(!axis.HasValue && reduction_indices.HasValue)
+            if (!axis.HasValue && reduction_indices.HasValue && !keepdims)
                 return math_ops.reduce_sum(input, reduction_indices.Value);
-            else if (axis.HasValue && !reduction_indices.HasValue)
+            else if (axis.HasValue && !reduction_indices.HasValue && !keepdims)
                 return math_ops.reduce_sum(input, axis.Value);
-            return math_ops.reduce_sum(input, keepdims: keepdims, name: name);
+            else if (axis.HasValue && !reduction_indices.HasValue && keepdims)
+                return math_ops.reduce_sum(input, keepdims: keepdims, axis: axis.Value, name: name);
+            else
+                return math_ops.reduce_sum(input, keepdims: keepdims, name: name);
         }
 
         public Tensor reduce_sum(Tensor input, TensorShape axis, int? reduction_indices = null, 
@@ -471,6 +474,9 @@ public Tensor sum(Tensor input, int axis, bool keep_dims = false, string name =
         public Tensor reduce_mean(Tensor input_tensor, int[] axis = null, bool keepdims = false, string name = null, int? reduction_indices = null)
             => math_ops.reduce_mean(input_tensor, axis: axis, keepdims: keepdims, name: name, reduction_indices: reduction_indices);
 
+        public Tensor reduce_mean(Tensor[] input_tensors, int axis, bool keepdims = false, string name = null)
+            => math_ops.reduce_mean(input_tensors, axis: axis, keepdims: keepdims, name: name);
+
         public Tensor round(Tensor x, string name = null)
             => gen_math_ops.round(x, name: name);
 

src/TensorFlowNET.Core/APIs/tf.random.cs

@@ -65,5 +65,10 @@ public Tensor random_shuffle(Tensor value, int? seed = null, string name = null)
 
         public void set_random_seed(int seed)
             => ops.get_default_graph().seed = seed;
+
+        public Tensor multinomial(Tensor logits, int num_samples, int? seed = null,
+            string name = null, TF_DataType output_dtype = TF_DataType.DtInvalid)
+            => random_ops.multinomial(logits, num_samples, seed: seed, 
+                name: name, output_dtype: output_dtype);
     }
 }

src/TensorFlowNET.Core/APIs/tf.train.cs

@@ -15,6 +15,7 @@ limitations under the License.
 ******************************************************************************/
 
 using System.Collections.Generic;
+using Tensorflow.Keras.Optimizers;
 using Tensorflow.Train;
 
 namespace Tensorflow
@@ -73,6 +74,26 @@ public string latest_checkpoint(string checkpoint_dir, string latest_filename =
 
             public CheckpointState get_checkpoint_state(string checkpoint_dir, string latest_filename = null)
                 => checkpoint_management.get_checkpoint_state(checkpoint_dir, latest_filename: latest_filename);
+
+            public Tensor polynomial_decay(float learning_rate,
+                RefVariable global_step,
+                float decay_steps,
+                float end_learning_rate = 0.0001f,
+                float power = 1.0f,
+                bool cycle = false,
+                string name = null)
+            {
+                var decayed = new PolynomialDecay(learning_rate,
+                    decay_steps,
+                    end_learning_rate: end_learning_rate,
+                    power: power,
+                    cycle: cycle,
+                    name: name);
+
+                var decayed_lr = decayed.__call__(global_step);
+
+                return decayed_lr;
+            }
         }
     }
 }

src/TensorFlowNET.Core/APIs/tf.variable.cs

@@ -27,6 +27,15 @@ public VariableV1[] global_variables(string scope = null)
                 .ToArray();
         }
 
+        /// <summary>
+        /// Returns an Op that initializes a list of variables.
+        /// </summary>
+        /// <param name="var_list">List of `Variable` objects to initialize.</param>
+        /// <param name="name">Optional name for the returned operation.</param>
+        /// <returns>An Op that run the initializers of all the specified variables.</returns>
+        public Operation variables_initializer(VariableV1[] var_list, string name = "init")
+            => variables.variables_initializer(var_list, name: name);
+
         public Operation global_variables_initializer()
         {
             var g = variables.global_variables();

src/TensorFlowNET.Core/Binding.Util.cs

@@ -115,6 +115,7 @@ public static IEnumerable<int> range(int start, int end)
             return instance;
         }
 
+        [DebuggerStepThrough]
         [DebuggerNonUserCode()] // with "Just My Code" enabled this lets the debugger break at the origin of the exception
         public static void tf_with(IObjectLife py, Action<IObjectLife> action)
         {
@@ -273,7 +274,10 @@ public static double sum(IEnumerable enumerable)
             return sum;
         }
 
-        public static double sum(IEnumerable<int> enumerable)
+        public static float sum(IEnumerable<float> enumerable)
+            => enumerable.Sum();
+
+        public static int sum(IEnumerable<int> enumerable)
             => enumerable.Sum();
 
         public static double sum<TKey, TValue>(Dictionary<TKey, TValue> values)

src/TensorFlowNET.Core/Keras/Optimizers/LearningRateSchedule.cs

@@ -0,0 +1,16 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Tensorflow.Keras.Optimizers
+{
+    public class LearningRateSchedule
+    {
+        public LearningRateSchedule()
+        {
+
+        }
+    }
+}

src/TensorFlowNET.Core/Keras/Optimizers/PolynomialDecay.cs

@@ -0,0 +1,62 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using static Tensorflow.Binding;
+
+namespace Tensorflow.Keras.Optimizers
+{
+    /// <summary>
+    /// A LearningRateSchedule that uses a polynomial decay schedule.
+    /// </summary>
+    public class PolynomialDecay : LearningRateSchedule
+    {
+        float initial_learning_rate;
+        float decay_steps;
+        float end_learning_rate;
+        float power;
+        bool cycle;
+        string name;
+
+        public PolynomialDecay(float initial_learning_rate,
+            float decay_steps,
+            float end_learning_rate = 0.0001f,
+            float power = 1.0f,
+            bool cycle = false,
+            string name = null) : base()
+        {
+            this.initial_learning_rate = initial_learning_rate;
+            this.decay_steps = decay_steps;
+            this.end_learning_rate = end_learning_rate;
+            this.power = power;
+            this.cycle = cycle;
+            this.name = name;
+        }
+
+        public Tensor __call__(RefVariable step)
+        {
+            tf_with(ops.name_scope(name ?? "PolynomialDecay"), scope =>
+            {
+                name = scope;
+                var initial_learning_rate_tensor = ops.convert_to_tensor(initial_learning_rate, name: "initial_learning_rate");
+                var dtype = initial_learning_rate_tensor.dtype;
+                var end_learning_rate_tensor = math_ops.cast(end_learning_rate, dtype);
+                var power_tensor = math_ops.cast(power, dtype);
+
+                var global_step_recomp = math_ops.cast(step, dtype);
+                var decay_steps_recomp = math_ops.cast(decay_steps, dtype);
+
+                if(cycle)
+                {
+                    throw new NotImplementedException("PolynomialDecay cycle");
+                }
+                else
+                {
+
+                }
+            });
+            throw new NotImplementedException("");
+        }
+    }
+}

src/TensorFlowNET.Core/Operations/Initializers/GlorotUniform.cs

@@ -19,8 +19,7 @@ namespace Tensorflow.Operations.Initializers
     public class GlorotUniform : VarianceScaling
     {
         public GlorotUniform(float scale = 1.0f,
-            string mode = "fan_avg",
-            string distribution = "uniform",
+            string mode = "FAN_AVG",
             int? seed = null,
             TF_DataType dtype = TF_DataType.TF_FLOAT) : base(factor: scale, 
                 mode: mode, 
@@ -36,7 +35,6 @@ public object get_config()
             {
                 scale = _scale,
                 mode = _mode,
-                distribution = _distribution,
                 seed = _seed,
                 dtype = _dtype
             };

src/TensorFlowNET.Core/Operations/Initializers/VarianceScaling.cs

@@ -30,6 +30,7 @@ public class VarianceScaling : IInitializer
         protected string _distribution;
         protected int? _seed;
         protected TF_DataType _dtype;
+        protected bool _uniform;
 
         public VarianceScaling(float factor = 2.0f,
             string mode = "FAN_IN",
@@ -49,31 +50,31 @@ public VarianceScaling(float factor = 2.0f,
             _mode = mode;
             _seed = seed;
             _dtype = dtype;
+            _uniform = uniform;
         }
 
         public Tensor call(TensorShape shape, TF_DataType dtype, bool? verify_shape = null)
         {
+            float n = 0;
             var (fan_in, fan_out) = _compute_fans(shape);
-            if (_mode == "fan_in")
-                _scale /= Math.Max(1, fan_in);
-            else if (_mode == "fan_out")
-                _scale /= Math.Max(1, fan_out);
-            else
-                _scale /= Math.Max(1, (fan_in + fan_out) / 2);
+            if (_mode == "FAN_IN")
+                n = fan_in;
+            else if (_mode == "FAN_OUT")
+                n = fan_out;
+            else if(_mode == "FAN_AVG")
+                n = (fan_in + fan_out) / 2.0f;
 
-            if (_distribution == "normal" || _distribution == "truncated_normal")
-            {
-                float stddev = (float)Math.Sqrt(_scale) / .87962566103423978f;
-                return random_ops.truncated_normal(shape, mean: 0.0f, stddev: stddev, dtype: dtype, seed: _seed);
-            }
-            else if (_distribution == "untruncated_normal")
+            if(_uniform)
             {
-                throw new NotImplementedException("truncated_normal");
+                var limit = Convert.ToSingle(Math.Sqrt(3.0f * _scale / n));
+                return random_ops.random_uniform(shape, -limit, limit,
+                                                 dtype, seed: _seed);
             }
             else
             {
-                var limit = Math.Sqrt(3.0f * _scale);
-                return random_ops.random_uniform(shape, (float)-limit, (float)limit, dtype, seed: _seed);
+                var trunc_stddev = Convert.ToSingle(Math.Sqrt(1.3f * _scale / n));
+                return random_ops.truncated_normal(shape, 0.0f, trunc_stddev, dtype,
+                                                   seed: _seed);
             }
         }
 
@@ -106,6 +107,7 @@ public virtual object get_config()
                 mode = _mode,
                 distribution = _distribution,
                 seed = _seed,
+                uniform = _uniform,
                 dtype = _dtype
             };
         }