Merge branch 'master' into tf.keras-0.3.image-classification

Author: Oceania2018

README.md

@@ -56,30 +56,32 @@ PM> Install-Package SciSharp.TensorFlow.Redist-Windows-GPU
 
 Import TF.NET and Keras API in your project.
 
-```cs
+```csharp
 using static Tensorflow.Binding;
 using static Tensorflow.KerasApi;
+using Tensorflow;
+using NumSharp;
 ```
 
 Linear Regression in `Eager` mode:
 
-```c#
+```csharp
 // Parameters        
 var training_steps = 1000;
 var learning_rate = 0.01f;
 var display_step = 100;
 
 // Sample data
-var train_X = np.array(3.3f, 4.4f, 5.5f, 6.71f, 6.93f, 4.168f, 9.779f, 6.182f, 7.59f, 2.167f,
+var X = np.array(3.3f, 4.4f, 5.5f, 6.71f, 6.93f, 4.168f, 9.779f, 6.182f, 7.59f, 2.167f,
              7.042f, 10.791f, 5.313f, 7.997f, 5.654f, 9.27f, 3.1f);
-var train_Y = np.array(1.7f, 2.76f, 2.09f, 3.19f, 1.694f, 1.573f, 3.366f, 2.596f, 2.53f, 1.221f,
+var Y = np.array(1.7f, 2.76f, 2.09f, 3.19f, 1.694f, 1.573f, 3.366f, 2.596f, 2.53f, 1.221f,
              2.827f, 3.465f, 1.65f, 2.904f, 2.42f, 2.94f, 1.3f);
-var n_samples = train_X.shape[0];
+var n_samples = X.shape[0];
 
 // We can set a fixed init value in order to demo
 var W = tf.Variable(-0.06f, name: "weight");
 var b = tf.Variable(-0.73f, name: "bias");
-var optimizer = tf.optimizers.SGD(learning_rate);
+var optimizer = keras.optimizers.SGD(learning_rate);
 
 // Run training for the given number of steps.
 foreach (var step in range(1, training_steps + 1))
@@ -112,46 +114,40 @@ Run this example in [Jupyter Notebook](https://github.com/SciSharp/SciSharpCube)
 Toy version of `ResNet` in `Keras` functional API:
 
 ```csharp
+var layers = new LayersApi();
 // input layer
 var inputs = keras.Input(shape: (32, 32, 3), name: "img");
-
 // convolutional layer
 var x = layers.Conv2D(32, 3, activation: "relu").Apply(inputs);
 x = layers.Conv2D(64, 3, activation: "relu").Apply(x);
 var block_1_output = layers.MaxPooling2D(3).Apply(x);
-
 x = layers.Conv2D(64, 3, activation: "relu", padding: "same").Apply(block_1_output);
 x = layers.Conv2D(64, 3, activation: "relu", padding: "same").Apply(x);
-var block_2_output = layers.add(x, block_1_output);
-
+var block_2_output = layers.Add().Apply(new Tensors(x, block_1_output));
 x = layers.Conv2D(64, 3, activation: "relu", padding: "same").Apply(block_2_output);
 x = layers.Conv2D(64, 3, activation: "relu", padding: "same").Apply(x);
-var block_3_output = layers.add(x, block_2_output);
-
+var block_3_output = layers.Add().Apply(new Tensors(x, block_2_output));
 x = layers.Conv2D(64, 3, activation: "relu").Apply(block_3_output);
 x = layers.GlobalAveragePooling2D().Apply(x);
 x = layers.Dense(256, activation: "relu").Apply(x);
 x = layers.Dropout(0.5f).Apply(x);
-
 // output layer
 var outputs = layers.Dense(10).Apply(x);
-
 // build keras model
-model = keras.Model(inputs, outputs, name: "toy_resnet");
+var model = keras.Model(inputs, outputs, name: "toy_resnet");
 model.summary();
-
 // compile keras model in tensorflow static graph
 model.compile(optimizer: keras.optimizers.RMSprop(1e-3f),
-	loss: keras.losses.CategoricalCrossentropy(from_logits: true),
-	metrics: new[] { "acc" });
-
+    loss: keras.losses.CategoricalCrossentropy(from_logits: true),
+    metrics: new[] { "acc" });
 // prepare dataset
 var ((x_train, y_train), (x_test, y_test)) = keras.datasets.cifar10.load_data();
-
+x_train = x_train / 255.0f;
+y_train = np_utils.to_categorical(y_train, 10);
 // training
-model.fit(x_train[new Slice(0, 1000)], y_train[new Slice(0, 1000)], 
-          batch_size: 64, 
-          epochs: 10, 
+model.fit(x_train[new Slice(0, 2000)], y_train[new Slice(0, 2000)],
+          batch_size: 64,
+          epochs: 10,
           validation_split: 0.2f);
 ```
 
@@ -260,4 +256,4 @@ WeChat Sponsor 微信打赏:
 
 TensorFlow.NET is a part of [SciSharp STACK](https://scisharp.github.io/SciSharp/)
 <br>
-<a href="http://scisharpstack.org"><img src="https://github.com/SciSharp/SciSharp/blob/master/art/scisharp-stack.png" width="391" height="100" /></a>
+<a href="http://scisharpstack.org"><img src="https://github.com/SciSharp/SciSharp/blob/master/art/scisharp-stack.png" width="391" height="100" /></a>

src/TensorFlowNET.Core/Operations/array_ops.cs

@@ -506,6 +506,27 @@ public static Tensor where(Tensor condition, object x = null, object y = null, s
             }
         }
 
+
+        public static Tensor where_v2(Tensor condition, object x = null, object y = null, string name = null)
+        {
+            if (x == null && y == null)
+            {
+                return tf_with(ops.name_scope(name, "Where", new { condition }), scope =>
+                {
+                    name = scope;
+                    condition = ops.convert_to_tensor(condition, preferred_dtype: dtypes.@bool, name: "condition");
+                    return gen_array_ops.where(condition: condition, name: name);
+                });
+            }
+            else if (x != null && y != null)
+            {
+                return gen_array_ops.select_v2(condition, x, y, name);
+            }
+            else
+            {
+                throw new ValueError("x and y must both be non-None or both be None.");
+            }
+        }
         /// <summary>
         /// Returns the shape of a tensor.
         /// </summary>

src/TensorFlowNET.Core/Operations/gen_array_ops.cs

@@ -423,6 +423,21 @@ public static Tensor select<Tx, Ty>(Tensor condition, Tx x, Ty y, string name =
             var _op = tf.OpDefLib._apply_op_helper("Select", name, new { condition, t = x, e = y });
             return _op.outputs[0];
         }
+        public static Tensor select_v2<Tx, Ty>(Tensor condition, Tx x, Ty y, string name = null)
+        {
+            if (tf.Context.executing_eagerly())
+            {
+                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
+                    "SelectV2", name,
+                    null,
+                    condition, x, y);
+
+                return results[0];
+            }
+
+            var _op = tf.OpDefLib._apply_op_helper("SelectV2", name, new { condition, t = x, e = y });
+            return _op.outputs[0];
+        }
 
         public static Tensor scatter_nd(Tensor indices, Tensor updates, Tensor[] shape, string name = null)
         {

src/TensorFlowNET.Core/Operations/gen_math_ops.cs

@@ -714,7 +714,23 @@ public static Tensor log(Tensor x, string name = null)
 
             return _op.outputs[0];
         }
+        public static Tensor softplus(Tensor features, string name = null)
+        {
+            if (tf.Context.executing_eagerly())
+            {
+                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
+                    "Softplus", name,
+                    null,
+                    features);
 
+                return results[0];
+            }
+
+            var _op = tf.OpDefLib._apply_op_helper("Softplus", name, args: new { features });
+
+            return _op.outputs[0];
+        }
+        
         public static Tensor cast(Tensor x, TF_DataType DstT, bool Truncate = false, string name = null)
             => tf.Context.RunInAutoMode(()
                 => tf.OpDefLib._apply_op_helper("Cast", name, args: new { x, DstT, Truncate }).output, ()
@@ -1068,6 +1084,15 @@ public static Tensor minimum<T1, T2>(T1 x, T2 y, string name = null)
 
         public static Tensor _abs(Tensor x, string name = null)
         {
+            if (tf.Context.executing_eagerly())
+            {
+                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
+                    "Abs", name,
+                     null,
+                     x);
+
+                return results[0];
+            }
             var _op = tf.OpDefLib._apply_op_helper("Abs", name, args: new { x });
 
             return _op.output;
@@ -1202,6 +1227,15 @@ public static Tensor round(Tensor x, string name = "Round")
         /// <returns></returns>
         public static Tensor rsqrt(Tensor x, string name = null)
         {
+            if (tf.Context.executing_eagerly())
+            {
+                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
+                    "Rsqrt", name,
+                    null,
+                    x);
+
+                return results[0];
+            }
             var _op = tf.OpDefLib._apply_op_helper("Rsqrt", name, new { x });
 
             return _op.outputs[0];

src/TensorFlowNET.Core/Operations/nn_impl.py.cs

@@ -31,15 +31,15 @@ public class nn_impl
         /// <returns></returns>
         public static Tensor l2_normalize(Tensor x,
             int axis = 0,
-            float epsilon = 1e-12f,
+            Tensor epsilon =null,
             string name = null)
         {
             return tf_with(ops.name_scope(name, "l2_normalize", new { x }), scope =>
             {
                 x = ops.convert_to_tensor(x, name: "x");
                 var sq = math_ops.square(x);
                 var square_sum = math_ops.reduce_sum(sq, axis, keepdims: true);
-                var x_inv_norm = math_ops.rsqrt(math_ops.maximum(square_sum, epsilon));
+                var x_inv_norm = math_ops.rsqrt(math_ops.maximum(square_sum, epsilon == null ? tf.Variable(1e-12f) : epsilon));
                 return math_ops.multiply(x, x_inv_norm, name: name);
             });
         }

src/TensorFlowNET.Keras/Losses/CategoricalCrossentropy.cs

@@ -9,18 +9,19 @@ namespace Tensorflow.Keras.Losses
     public class CategoricalCrossentropy : LossFunctionWrapper, ILossFunc
     {
         float label_smoothing;
-
-        public CategoricalCrossentropy(bool from_logits = false,
+        public CategoricalCrossentropy(
+            bool from_logits = false,
             float label_smoothing = 0,
-            string reduction = ReductionV2.AUTO,
-            string name = "categorical_crossentropy") :
-            base(reduction: reduction, 
-                name: name,
-                from_logits: from_logits)
+            string reduction = null,
+            string name = null) :
+            base(reduction: reduction,
+                 name: name == null ? "categorical_crossentropy" : name, 
+                 from_logits: from_logits)
         {
             this.label_smoothing = label_smoothing;
         }
 
+
         public override Tensor Apply(Tensor y_true, Tensor y_pred, bool from_logits = false, int axis = -1)
         {
             // Try to adjust the shape so that rank of labels = rank of logits - 1.

src/TensorFlowNET.Keras/Losses/CosineSimilarity.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using static Tensorflow.Binding;
+using static Tensorflow.KerasApi;
+
+namespace Tensorflow.Keras.Losses
+{
+    public class CosineSimilarity : LossFunctionWrapper, ILossFunc
+    {
+        protected int axis=-1;
+        public CosineSimilarity(
+            string reduction = null,
+            int axis=-1,
+            string name = null) :
+            base(reduction: reduction, name: name == null ? "cosine_similarity" : name)
+        {
+            this.axis = axis;
+        }
+
+        public override Tensor Apply(Tensor y_true = null, Tensor y_pred =null, bool from_logits = false, int axis = -1)
+        {
+            Tensor y_true_normalize = nn_impl.l2_normalize(y_true, axis : this.axis);
+            Tensor y_pred_normalize = nn_impl.l2_normalize(y_pred, axis: this.axis);
+            return -math_ops.reduce_sum(y_true_normalize * y_pred_normalize, axis : this.axis);
+        }
+    }
+}

src/TensorFlowNET.Keras/Losses/Huber.cs

@@ -0,0 +1,36 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using static Tensorflow.Binding;
+using static Tensorflow.KerasApi;
+
+namespace Tensorflow.Keras.Losses
+{
+    public class Huber : LossFunctionWrapper, ILossFunc
+    {
+        protected Tensor delta = tf.Variable(1.0) ;
+        public Huber (
+            string reduction = null,
+            Tensor delta = null,
+            string name = null) :
+            base(reduction: reduction, name: name == null ? "huber" : name)
+        {
+            this.delta = delta==null? this.delta: delta;
+            
+        }
+
+        public override Tensor Apply(Tensor y_true = null, Tensor y_pred =null, bool from_logits = false, int axis = -1)
+        {
+            Tensor y_pred_cast = math_ops.cast(y_pred, dtype: TF_DataType.TF_FLOAT);
+            Tensor y_true_cast = math_ops.cast(y_true, dtype: TF_DataType.TF_FLOAT);
+            Tensor delta = math_ops.cast(this.delta, dtype: TF_DataType.TF_FLOAT);
+            Tensor error = math_ops.subtract(y_pred_cast, y_true_cast);
+            Tensor abs_error = math_ops.abs(error);
+            Tensor half = ops.convert_to_tensor(0.5, dtype: abs_error.dtype);
+            return gen_math_ops.mean(array_ops.where_v2(abs_error <= delta, 
+                                                        half * math_ops.pow(error, 2), 
+                                                        half * math_ops.pow(delta, 2) + delta * (abs_error - delta)),
+                                     axis : -1);
+        }
+    }
+}

src/TensorFlowNET.Keras/Losses/ILossFunc.cs

@@ -2,7 +2,8 @@
 {
     public interface ILossFunc
     {
-        string Reduction { get; }
-        Tensor Call(Tensor y_true, Tensor y_pred);
+        public string Reduction { get; }
+        public string Name { get; }
+        Tensor Call(Tensor y_true, Tensor y_pred, Tensor sample_weight = null);
     }
 }

src/TensorFlowNET.Keras/Losses/LogCosh.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Tensorflow.Operations;
+using static Tensorflow.Binding;
+using static Tensorflow.KerasApi;
+
+namespace Tensorflow.Keras.Losses
+{
+    public class LogCosh : LossFunctionWrapper, ILossFunc
+    {
+        public LogCosh(
+            string reduction = null,
+            string name = null) :
+            base(reduction: reduction, name: name == null ? "huber" : name){ }
+
+        public override Tensor Apply(Tensor y_true = null, Tensor y_pred =null, bool from_logits = false, int axis = -1)
+        {
+            Tensor y_pred_dispatch = ops.convert_to_tensor(y_pred);
+            Tensor y_true_cast = gen_math_ops.cast(y_true, y_pred_dispatch.dtype);
+            Tensor x = y_pred_dispatch - y_true_cast;
+       
+            return gen_math_ops.mean(x + gen_math_ops.softplus(-2.0 * x) - math_ops.cast(math_ops.log(tf.Variable(2.0)), x.dtype),axis: -1);
+            
+
+        }
+    }
+}