fix Operation.get_attr for all properties.

add Convolution class
add convert_data_format
add as_numpy_dtype

Author: Oceania2018

TensorFlow.NET.sln

@@ -11,6 +11,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TensorFlowNET.Core", "src\T
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TensorFlowNET.Visualization", "src\TensorFlowNET.Visualization\TensorFlowNET.Visualization.csproj", "{0254BFF9-453C-4FE0-9609-3644559A79CE}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "NumSharp.Core", "..\NumSharp\src\NumSharp.Core\NumSharp.Core.csproj", "{3EEAFB06-BEF0-4261-BAAB-630EABD25290}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -33,6 +35,10 @@ Global
 		{0254BFF9-453C-4FE0-9609-3644559A79CE}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{0254BFF9-453C-4FE0-9609-3644559A79CE}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{0254BFF9-453C-4FE0-9609-3644559A79CE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{3EEAFB06-BEF0-4261-BAAB-630EABD25290}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{3EEAFB06-BEF0-4261-BAAB-630EABD25290}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3EEAFB06-BEF0-4261-BAAB-630EABD25290}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{3EEAFB06-BEF0-4261-BAAB-630EABD25290}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/TensorFlowNET.Core/Graphs/Graph.cs

@@ -21,6 +21,7 @@ public partial class Graph : IPython, IDisposable
         public int _version;
         private int _next_id_counter;
         private List<String> _unfetchable_ops = new List<string>();
+        private List<Tensor> _unfeedable_tensors = new List<Tensor>();
 
         public string _name_stack = "";
         public string _graph_key;
@@ -366,6 +367,11 @@ public object get_collection_ref(string name)
             return _collections[name];
         }
 
+        public void prevent_feeding(Tensor tensor)
+        {
+            _unfeedable_tensors.Add(tensor);
+        }
+
         public void Dispose()
         {
             c_api.TF_DeleteGraph(_handle);

src/TensorFlowNET.Core/Keras/Engine/InputSpec.cs

@@ -10,11 +10,16 @@ namespace Tensorflow.Keras.Engine
     public class InputSpec
     {
         public int ndim;
+        Dictionary<int, int> axes;
 
         public InputSpec(TF_DataType dtype = TF_DataType.DtInvalid, 
-            int? ndim = null)
+            int? ndim = null,
+            Dictionary<int, int> axes = null)
         {
             this.ndim = ndim.Value;
+            if (axes == null)
+                axes = new Dictionary<int, int>();
+            this.axes = axes;
         }
     }
 }

src/TensorFlowNET.Core/Keras/Engine/Layer.cs

@@ -66,7 +66,7 @@ protected virtual void build(TensorShape input_shape)
 
         }
 
-        protected virtual void add_weight(string name,
+        protected virtual RefVariable add_weight(string name,
             int[] shape,
             TF_DataType dtype = TF_DataType.DtInvalid,
             IInitializer initializer = null,
@@ -82,6 +82,8 @@ protected virtual void add_weight(string name,
                 trainable: trainable.Value);
             backend.track_variable(variable);
             _trainable_weights.Add(variable);
+
+            return variable;
         }
     }
 }

src/TensorFlowNET.Core/Keras/Layers/Conv.cs

@@ -2,6 +2,8 @@
 using System.Collections.Generic;
 using System.Text;
 using Tensorflow.Keras.Engine;
+using Tensorflow.Keras.Utils;
+using Tensorflow.Operations;
 using Tensorflow.Operations.Activation;
 
 namespace Tensorflow.Keras.Layers
@@ -19,6 +21,9 @@ public class Conv : Tensorflow.Layers.Layer
         protected bool use_bias;
         protected IInitializer kernel_initializer;
         protected IInitializer bias_initializer;
+        protected RefVariable kernel;
+        protected RefVariable bias;
+        protected Convolution _convolution_op;
 
         public Conv(int rank, 
             int filters,
@@ -53,11 +58,37 @@ protected override void build(TensorShape input_shape)
             int channel_axis = data_format == "channels_first" ? 1 : -1;
             int input_dim = input_shape.Dimensions[input_shape.NDim - 1];
             var kernel_shape = new int[] { kernel_size[0], kernel_size[1], input_dim, filters };
-            add_weight(name: "kernel", 
+            kernel = add_weight(name: "kernel",
                 shape: kernel_shape,
                 initializer: kernel_initializer,
                 trainable: true,
                 dtype: _dtype);
+            if (use_bias)
+                bias = add_weight(name: "bias",
+                    shape: new int[] { filters },
+                    initializer: bias_initializer,
+                    trainable: true,
+                    dtype: _dtype);
+
+            var axes = new Dictionary<int, int>();
+            axes.Add(-1, input_dim);
+            input_spec = new InputSpec(ndim: rank + 2, axes: axes);
+
+            string op_padding;
+            if (padding == "causal")
+                op_padding = "valid";
+            else
+                op_padding = padding;
+
+            var df = conv_utils.convert_data_format(data_format, rank + 2);
+            _convolution_op = nn_ops.Convolution(input_shape,
+                kernel.shape,
+                op_padding.ToUpper(),
+                strides,
+                dilation_rate,
+                data_format: df);
+
+            built = true;
         }
     }
 }

src/TensorFlowNET.Core/Keras/Utils/conv_utils.cs

@@ -0,0 +1,33 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow.Keras.Utils
+{
+    public class conv_utils
+    {
+        public static string convert_data_format(string data_format, int ndim)
+        {
+            if (data_format == "channels_last")
+                if (ndim == 3)
+                    return "NWC";
+                else if (ndim == 4)
+                    return "NHWC";
+                else if (ndim == 5)
+                    return "NDHWC";
+                else
+                    throw new ValueError($"Input rank not supported: {ndim}");
+            else if (data_format == "channels_first")
+                if (ndim == 3)
+                    return "NCW";
+                else if (ndim == 4)
+                    return "NCHW";
+                else if (ndim == 5)
+                    return "NCDHW";
+                else
+                    throw new ValueError($"Input rank not supported: {ndim}");
+            else
+                throw new ValueError($"Invalid data_format: {data_format}");
+        }
+    }
+}

src/TensorFlowNET.Core/Layers/Layer.cs

@@ -68,7 +68,7 @@ public Tensor __call__(Tensor inputs,
             throw new NotImplementedException("");
         }
 
-        protected virtual void add_weight(string name,
+        protected virtual RefVariable add_weight(string name,
             int[] shape,
             TF_DataType dtype = TF_DataType.DtInvalid,
             IInitializer initializer = null,
@@ -93,14 +93,14 @@ protected virtual void add_weight(string name,
 
             _set_scope();
             var reuse = built || (_reuse != null && _reuse.Value);
-            Python.with(tf.variable_scope(_scope, 
+            return Python.with(tf.variable_scope(_scope, 
                 reuse: reuse, 
                 auxiliary_name_scope: false), scope =>
             {
                 _current_scope = scope;
-                Python.with(ops.name_scope(_name_scope()), delegate
+                return Python.with(ops.name_scope(_name_scope()), delegate
                 {
-                    base.add_weight(name,
+                    var variable = base.add_weight(name,
                         shape,
                         dtype: dtype,
                         initializer: initializer,
@@ -113,6 +113,12 @@ protected virtual void add_weight(string name,
                                 initializer: initializer1,
                                 trainable: trainable1);
                         });
+
+                    if(init_graph != null)
+                    {
+                        var trainable_variables = variables.trainable_variables();
+                    }
+                    return variable;
                 });
             });
         }

src/TensorFlowNET.Core/Operations/NnOps/Convolution.cs

@@ -0,0 +1,66 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace Tensorflow.Operations
+{
+    public class Convolution
+    {
+        public TensorShape input_shape;
+        public TensorShape filter_shape;
+        public string data_format;
+        public int[] strides;
+        public string name;
+        public _WithSpaceToBatch conv_op;
+
+        public Convolution(TensorShape input_shape,
+            TensorShape filter_shape,
+            string padding,
+            int[] strides,
+            int[] dilation_rate,
+            string name = null,
+            string data_format = null)
+        {
+            var num_total_dims = filter_shape.NDim;
+            var num_spatial_dims = num_total_dims - 2;
+            int input_channels_dim;
+            int[] spatial_dims;
+            if (string.IsNullOrEmpty(data_format) || !data_format.StartsWith("NC"))
+            {
+                input_channels_dim = input_shape.Dimensions[num_spatial_dims + 1];
+                spatial_dims = Enumerable.Range(1, num_spatial_dims).ToArray();
+            }
+            else
+            {
+                input_channels_dim = input_shape.Dimensions[1];
+                spatial_dims = Enumerable.Range(2, num_spatial_dims).ToArray();
+            }
+
+            this.input_shape = input_shape;
+            this.filter_shape = filter_shape;
+            this.data_format = data_format;
+            this.strides = strides;
+            this.name = name;
+
+            conv_op = new _WithSpaceToBatch(
+                input_shape,
+                dilation_rate: dilation_rate,
+                padding: padding,
+                build_op: _build_op,
+                filter_shape: filter_shape,
+                spatial_dims: spatial_dims,
+                data_format: data_format);
+        }
+
+        public _NonAtrousConvolution _build_op(int _, string padding)
+        {
+            return new _NonAtrousConvolution(input_shape,
+                filter_shape: filter_shape,
+                padding: padding,
+                data_format: data_format,
+                strides: strides,
+                name: name);
+        }
+    }
+}

src/TensorFlowNET.Core/Operations/NnOps/_NonAtrousConvolution.cs

@@ -0,0 +1,56 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace Tensorflow.Operations
+{
+    public class _NonAtrousConvolution
+    {
+        public string padding;
+        public string name;
+        public int[] strides;
+        public string data_format;
+        private Func<object, Tensor> conv_op;
+
+        public _NonAtrousConvolution(TensorShape input_shape,
+            TensorShape filter_shape,
+            string padding,
+            string data_format,
+            int[] strides,
+            string name)
+        {
+            this.padding = padding;
+            this.name = name;
+            var conv_dims = input_shape.NDim - 2;
+            if (conv_dims == 1)
+            {
+                throw new NotImplementedException("_NonAtrousConvolution conv_dims 1");
+            }
+            else if (conv_dims == 2)
+            {
+                var list = strides.ToList();
+
+                if (string.IsNullOrEmpty(data_format) || data_format == "NHWC")
+                {
+                    data_format = "NHWC";
+                    list.Insert(0, 1);
+                    list.Add(1);
+                }
+                else if (data_format == "NCHW")
+                    list.InsertRange(0, new int[] { 1, 1 });
+                else
+                    throw new ValueError("data_format must be \"NHWC\" or \"NCHW\".");
+
+                strides = list.ToArray();
+                this.strides = strides;
+                this.data_format = data_format;
+                conv_op = gen_nn_ops.conv2d;
+            }
+            else if (conv_dims == 3)
+            {
+                throw new NotImplementedException("_NonAtrousConvolution conv_dims 3");
+            }
+        }
+    }
+}

src/TensorFlowNET.Core/Operations/NnOps/_WithSpaceToBatch.cs

@@ -0,0 +1,55 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace Tensorflow.Operations
+{
+    public class _WithSpaceToBatch
+    {
+        private _NonAtrousConvolution call;
+
+        public _WithSpaceToBatch(TensorShape input_shape,
+            int[] dilation_rate,
+            string padding,
+            Func<int, string, _NonAtrousConvolution> build_op,
+            TensorShape filter_shape = null,
+            int[] spatial_dims = null,
+            string data_format = null)
+        {
+            var dilation_rate_tensor = ops.convert_to_tensor(dilation_rate, TF_DataType.TF_INT32, name: "dilation_rate");
+            var rate_shape = dilation_rate_tensor.getShape();
+            var num_spatial_dims = rate_shape.Dimensions[0];
+            int starting_spatial_dim = -1;
+            if (!string.IsNullOrEmpty(data_format) && data_format.StartsWith("NC"))
+                starting_spatial_dim = 2;
+            else
+                starting_spatial_dim = 1;
+
+            if (spatial_dims == null)
+                throw new NotImplementedException("_WithSpaceToBatch spatial_dims");
+
+            var orig_spatial_dims = spatial_dims;
+            spatial_dims = spatial_dims.OrderBy(x => x).ToArray();
+            if (!Enumerable.SequenceEqual(spatial_dims, orig_spatial_dims) || spatial_dims.Any(x => x < 1))
+                throw new ValueError("spatial_dims must be a montonically increasing sequence of positive integers");
+
+            int expected_input_rank = -1;
+            if (!string.IsNullOrEmpty(data_format) && data_format.StartsWith("NC"))
+                expected_input_rank = spatial_dims.Last();
+            else
+                expected_input_rank = spatial_dims.Last() + 1;
+
+            var const_rate = tensor_util.constant_value(dilation_rate_tensor);
+            var rate_or_const_rate = dilation_rate;
+            if(!(const_rate is null))
+            {
+                if (const_rate.Data<int>().Count(x => x == 1) == const_rate.size)
+                {
+                    call = build_op(num_spatial_dims, padding);
+                    return;
+                }
+            }
+        }
+    }
+}