fix neural_net.trainable_variables #595.

Author: Oceania2018

src/TensorFlowNET.Core/APIs/tf.init.cs

@@ -20,6 +20,8 @@ namespace Tensorflow
 {
     public partial class tensorflow
     {
+        public InitializersImpl initializers { get; } = new InitializersImpl();
+
         public IInitializer constant_initializer<T>(T value, TF_DataType dtype = TF_DataType.TF_FLOAT, bool verify_shape = false) 
             => new Constant<T>(value, dtype: dtype, verify_shape: verify_shape);
         public IInitializer zeros_initializer => new Zeros();
@@ -82,5 +84,20 @@ public IInitializer variance_scaling_initializer(float factor = 1.0f,
                 uniform: uniform,
                 seed: seed,
                 dtype: dtype);
+
+        public class InitializersImpl
+        {
+            public IInitializer random_normal_initializer(float mean = 0.0f,
+                float stddev = 1.0f,
+                int? seed = null,
+                TF_DataType dtype = TF_DataType.TF_FLOAT) => new RandomNormal(mean: mean,
+                    stddev: stddev,
+                    seed: seed,
+                    dtype: dtype);
+
+            public IInitializer zeros_initializer(TensorShape shape = null,
+                TF_DataType dtype = TF_DataType.TF_FLOAT) => new Zeros(shape: shape, 
+                    dtype: dtype);
+        }
     }
 }

src/TensorFlowNET.Core/APIs/tf.optimizers.cs

@@ -27,6 +27,18 @@ public partial class tensorflow
         public class KerasOptimizers
         {
             public SGD SGD(float learning_rate) => new SGD(learning_rate);
+
+            public Adam Adam(float learning_rate = 0.001f,
+                float beta_1 = 0.9f,
+                float beta_2 = 0.999f,
+                float epsilon = 1e-7f,
+                bool amsgrad = false,
+                string name = "Adam") => new Adam(learning_rate: learning_rate,
+                    beta_1: beta_1,
+                    beta_2: beta_2,
+                    epsilon: epsilon,
+                    amsgrad: amsgrad,
+                    name: name);
         }
     }
 }

src/TensorFlowNET.Core/Eager/EagerOperation.cs

@@ -51,28 +51,14 @@ public override Tensor[] outputs
 
         public override object get_attr(string attr_name)
         {
-            object value = null;
-            byte isList = 0;
-            var attrType = c_api.TFE_OpNameGetAttrType(tf.Context.Handle, Name, attr_name, ref isList, tf.Status.Handle);
-            switch (attrType)
-            {
-                case TF_AttrType.TF_ATTR_BOOL:
-                    value = get_attr_bool(attr_name);
-                    break;
-                default:
-                    break;
-            }
-
-            return value;
-        }
-
-        public bool get_attr_bool(string attr_name)
-        {
+            // var attrType = c_api.TFE_OpNameGetAttrType(tf.Context.Handle, Name, attr_name, ref isList, tf.Status.Handle);
             for (int i = 0; i < Attrs.Length; i = i + 2)
+            {
                 if (Attrs[i].Equals(attr_name))
-                    return Attrs[i + 1].Equals("1");
+                    return Attrs[i + 1];
+            }
 
-            throw new ValueError($"Can't find attr: {attr_name}");
+            return null;
         }
 
         public override string ToString()

src/TensorFlowNET.Core/Eager/EagerRunner.TFE_FastPathExecute.cs

@@ -344,6 +344,11 @@ bool SetOpAttrList(Context ctx, SafeOpHandle op,
                 c_api.TFE_OpSetAttrTypeList(op, key, values2, values2.Length);
                 attr_list_sizes[key] = values2.Length;
             }
+            else if (type == TF_AttrType.TF_ATTR_INT && values is int[] values4)
+            {
+                c_api.TFE_OpSetAttrIntList(op, key, values4.Select(x => Convert.ToInt64(x)).ToArray(), values4.Length);
+                attr_list_sizes[key] = values4.Length;
+            }
             else
             {
                 throw new NotImplementedException("");

src/TensorFlowNET.Core/Eager/c_api.eager.cs

@@ -209,6 +209,9 @@ public static void TFE_Execute(SafeOpHandle op, SafeTensorHandleHandle[] retvals
         [DllImport(TensorFlowLibName)]
         public static extern void TFE_OpSetAttrTypeList(SafeOpHandle op, string attr_name, TF_DataType[] values, int num_values);
 
+        [DllImport(TensorFlowLibName)]
+        public static extern void TFE_OpSetAttrIntList(SafeOpHandle op, string attr_name, long[] values, int num_values);
+
         [DllImport(TensorFlowLibName)]
         public static extern void TFE_OpSetAttrValueProto(SafeOpHandle op, string attr_name, IMessage[] proto, int proto_len, SafeStatusHandle status);
 

src/TensorFlowNET.Core/Gradients/GradientTape.cs

@@ -119,7 +119,7 @@ public Tensor gradient(Tensor target, ResourceVariable source)
             return (results[0], results[1]);
         }
 
-        public Tensor[] gradient(Tensor target, List<IVariableV1> sources)
+        public Tensor[] gradient(Tensor target, IEnumerable<IVariableV1> sources)
         {
             if (_recording)
             {

src/TensorFlowNET.Core/Gradients/nn_grad.cs

@@ -128,12 +128,12 @@ public static Tensor[] _SparseSoftmaxCrossEntropyWithLogitsGrad(Operation op, Te
         [RegisterGradient("Conv2D")]
         public static Tensor[] _Conv2DGrad(Operation op, Tensor[] grads)
         {
-            var dilations = (op.get_attr("dilations") as AttrValue.Types.ListValue).I.Select(x => Convert.ToInt32(x)).ToArray();
-            var strides = (op.get_attr("strides") as AttrValue.Types.ListValue).I.Select(x => Convert.ToInt32(x)).ToArray();
-            var padding = op.get_attr("padding");
-            var explicit_paddings = (op.get_attr("explicit_paddings") as AttrValue.Types.ListValue).I.Select(x => Convert.ToInt32(x)).ToArray();
-            var use_cudnn_on_gpu = op.get_attr("use_cudnn_on_gpu");
-            var data_format = op.get_attr("data_format");
+            var dilations = op.get_attr<int[]>("dilations");
+            var strides = op.get_attr<int[]>("strides");
+            var padding = op.get_attr<string>("padding");
+            var explicit_paddings = op.get_attr<int[]>("explicit_paddings");
+            var use_cudnn_on_gpu = op.get_attr<bool>("use_cudnn_on_gpu");
+            var data_format = op.get_attr<string>("data_format");
             var shape = gen_array_ops.shape_n(new Tensor[] { op.inputs[0], op.inputs[1] });
 
             return new Tensor[]
@@ -287,8 +287,8 @@ public static Tensor[] _MaxPoolGrad(Operation op, Tensor[] grads)
                   op.inputs[0],
                   op.outputs[0],
                   grad,
-                  (op.get_attr("ksize") as AttrValue.Types.ListValue).I.Select(x => Convert.ToInt32(x)).ToArray(),
-                  (op.get_attr("strides") as AttrValue.Types.ListValue).I.Select(x => Convert.ToInt32(x)).ToArray(),
+                  op.get_attr("ksize") as int[],
+                  op.get_attr("strides") as int[],
                   padding: op.get_attr("padding").ToString(),
                   data_format: op.get_attr("data_format").ToString())
             };

src/TensorFlowNET.Core/Keras/Optimizers/Adam.cs

@@ -0,0 +1,91 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Tensorflow.Eager;
+
+namespace Tensorflow.Keras.Optimizers
+{
+    /// <summary>
+    /// Optimizer that implements the Adam algorithm.
+    /// Adam optimization is a stochastic gradient descent method that is based on
+    /// adaptive estimation of first-order and second-order moments.
+    /// </summary>
+    public class Adam : OptimizerV2
+    {
+        protected override string _name => "Adam";
+        float epsilon = 1e-7f;
+        bool amsgrad = false;
+
+        public Adam(float learning_rate = 0.001f,
+                float beta_1 = 0.9f,
+                float beta_2 = 0.999f,
+                float epsilon = 1e-7f,
+                bool amsgrad = false,
+                string name = "Adam")
+        {
+            _set_hyper("learning_rate", learning_rate);
+            // _set_hyper("decay", _initial_decay);
+            _set_hyper("beta_1", beta_1);
+            _set_hyper("beta_2", beta_2);
+            this.epsilon = epsilon;
+            this.amsgrad = amsgrad;
+        }
+
+        protected override void _create_slots(IVariableV1[] var_list)
+        {
+            foreach(var var in var_list)
+                add_slot(var, "m");
+            foreach (var var in var_list)
+                add_slot(var, "v");
+            if (amsgrad)
+                foreach (var var in var_list)
+                    add_slot(var, "vhat");
+        }
+
+        protected override void _prepare_local(DeviceDType device_dtype, Dictionary<DeviceDType, Dictionary<string, Tensor>> apply_state)
+        {
+            base._prepare_local(device_dtype, apply_state);
+            var var_dtype = device_dtype.DType;
+            var var_device = device_dtype.Device;
+            var local_step = math_ops.cast(iterations + 1, var_dtype);
+            var beta_1_t = array_ops.identity(_get_hyper("beta_1", var_dtype));
+            var beta_2_t = array_ops.identity(_get_hyper("beta_2", var_dtype));
+            var beta_1_power = math_ops.pow(beta_1_t, local_step);
+            var beta_2_power = math_ops.pow(beta_2_t, local_step);
+            var lr = apply_state[device_dtype]["lr_t"] * (math_ops.sqrt(1 - beta_2_power) / (1 - beta_1_power));
+            // update state
+            apply_state[device_dtype]["lr"] = lr;
+            apply_state[device_dtype]["epsilon"] = ops.convert_to_tensor(epsilon);
+            apply_state[device_dtype]["beta_1_t"] = beta_1_t;
+            apply_state[device_dtype]["beta_1_power"] = beta_1_power;
+            apply_state[device_dtype]["one_minus_beta_1_t"] = 1 - beta_1_t;
+            apply_state[device_dtype]["beta_2_t"] = beta_2_t;
+            apply_state[device_dtype]["beta_2_power"] = beta_2_power;
+            apply_state[device_dtype]["one_minus_beta_2_t"] = 1 - beta_2_t;
+        }
+
+        protected override Operation _resource_apply_dense(IVariableV1 var, Tensor grad, Dictionary<DeviceDType, Dictionary<string, Tensor>> apply_state)
+        {
+            var (var_device, var_dtype) = (var.Device, var.dtype.as_base_dtype());
+            var coefficients = apply_state.FirstOrDefault(x => x.Key.Device == var_device && x.Key.DType == var_dtype).Value ?? _fallback_apply_state(var_device, var_dtype);
+            var m = get_slot(var, "m");
+            var v = get_slot(var, "v");
+
+            if (!amsgrad)
+                return gen_training_ops.resource_apply_adam(var.Handle,
+                    m.Handle,
+                    v.Handle,
+                    coefficients["beta_1_power"],
+                    coefficients["beta_2_power"],
+                    coefficients["lr_t"],
+                    coefficients["beta_1_t"],
+                    coefficients["beta_2_t"],
+                    coefficients["epsilon"],
+                    grad,
+                    use_locking: _use_locking);
+            else
+                throw new NotImplementedException("");
+        }
+    }
+}

src/TensorFlowNET.Core/Keras/Optimizers/OptimizerV2.cs

@@ -18,22 +18,25 @@ public class OptimizerV2 : Trackable, IOptimizer
         protected bool _hypers_created;
         protected virtual string _name { get; }
 
-        ResourceVariable _iterations;
-        List<ResourceVariable> _weight;
+        IVariableV1 _iterations;
+        protected ResourceVariable iterations => _iterations as ResourceVariable;
+        List<IVariableV1> _weights;
         Dictionary<string, float> _hyper;
-        Dictionary<string, ResourceVariable> _hyper_variables;
+        Dictionary<string, IVariableV1> _hyper_variables;
         protected bool _momentum;
         protected float _initial_decay = 0.0f;
         protected bool _use_locking = true;
 
-        Dictionary<DeviceDType, Dictionary<string, Tensor>> apply_state;
+        Dictionary<string, Dictionary<string, IVariableV1>> _slots;
+        List<string> _slot_names;
 
         public OptimizerV2() : base()
         {
-            _weight = new List<ResourceVariable>();
+            _weights = new List<IVariableV1>();
             _hyper = new Dictionary<string, float>();
-            _hyper_variables = new Dictionary<string, ResourceVariable>();
-            apply_state = new Dictionary<DeviceDType, Dictionary<string, Tensor>>();
+            _hyper_variables = new Dictionary<string, IVariableV1>();
+            _slots = new Dictionary<string, Dictionary<string, IVariableV1>>();
+            _slot_names = new List<string>();
         }
 
         public void apply_gradients((Tensor, ResourceVariable) grads_and_vars,
@@ -61,7 +64,7 @@ public void apply_gradients(IEnumerable<(Tensor, ResourceVariable)> grads_and_va
                 if (grads_and_vars == null || grads_and_vars.Count() == 0)
                     return control_flow_ops.no_op();
 
-                apply_state = _prepare(var_list);
+                var apply_state = _prepare(var_list);
                 if(experimental_aggregate_gradients)
                 {
                     // var reduced_grads = _aggregate_gradients(grads_and_vars);
@@ -72,13 +75,13 @@ public void apply_gradients(IEnumerable<(Tensor, ResourceVariable)> grads_and_va
             });
         }
 
-        void apply_grad_to_update_var(ResourceVariable var, EagerTensor grad)
+        void apply_grad_to_update_var(ResourceVariable var, Tensor grad, Dictionary<DeviceDType, Dictionary<string, Tensor>> apply_state)
         {
             _resource_apply_dense(var, grad, apply_state);
         }
 
         protected virtual Operation _resource_apply_dense(IVariableV1 var, 
-            EagerTensor grad, 
+            Tensor grad, 
             Dictionary<DeviceDType, Dictionary<string, Tensor>> _apply_state)
         {
             throw new NotImplementedException("_resource_apply_dense");
@@ -94,7 +97,7 @@ void _distributed_apply(IEnumerable<(Tensor, ResourceVariable)> grads_and_vars,
                 {
                     tf_with(ops.name_scope("update"), delegate
                     {
-                        apply_grad_to_update_var(var, grad as EagerTensor);
+                        apply_grad_to_update_var(var, grad, _apply_state);
                     });
                 }
 
@@ -107,6 +110,12 @@ Tensor[] _aggregate_gradients(IEnumerable<(Tensor, ResourceVariable)> grads_and_
             return grads_and_vars.Select(x => x.Item1).ToArray();
         }
 
+        protected IVariableV1 get_slot(IVariableV1 var, string slot_name)
+        {
+            var slot_dict = _slots[var.UniqueId];
+            return slot_dict[slot_name];
+        }
+
         Dictionary<DeviceDType, Dictionary<string, Tensor>> _prepare(IVariableV1[] var_list)
         {
             var _apply_state = new Dictionary<DeviceDType, Dictionary<string, Tensor>>();
@@ -125,6 +134,11 @@ Dictionary<DeviceDType, Dictionary<string, Tensor>> _prepare(IVariableV1[] var_l
             return _apply_state;
         }
 
+        protected Dictionary<string, Tensor> _fallback_apply_state(string var_device, TF_DataType var_dtype)
+        {
+            throw new NotImplementedException("");
+        }
+
         protected virtual void _prepare_local(DeviceDType device_dtype, 
             Dictionary<DeviceDType, Dictionary<string, Tensor>> _apply_state)
         {
@@ -145,7 +159,7 @@ Tensor _decayed_lr(TF_DataType var_dtype)
             return lr_t;
         }
 
-        protected ResourceVariable _get_hyper(string name, TF_DataType dtype = TF_DataType.DtInvalid)
+        protected Tensor _get_hyper(string name, TF_DataType dtype = TF_DataType.DtInvalid)
         {
             var value = _hyper_variables[name];
             return math_ops.cast(value, dtype);
@@ -160,7 +174,7 @@ void _create_all_weights(IVariableV1[] var_list)
                     dtype: TF_DataType.TF_INT64, 
                     trainable: false, 
                     aggregation: VariableAggregation.OnlyFirstReplica);
-                _weight.Add(_iterations);
+                _weights.Add(_iterations);
             }
 
             _create_hypers();
@@ -190,7 +204,7 @@ void _create_hypers()
             _hypers_created = true;
         }
 
-        void _create_slots(IVariableV1[] var_list)
+        protected virtual void _create_slots(IVariableV1[] var_list)
         {
             if(_momentum)
             {
@@ -199,6 +213,35 @@ void _create_slots(IVariableV1[] var_list)
             }
         }
 
+        protected IVariableV1 add_slot(IVariableV1 var, string slot_name, IInitializer initializer = null)
+        {
+            if (initializer == null)
+                initializer = tf.zeros_initializer;
+
+            if (!_slot_names.Contains(slot_name))
+                _slot_names.append(slot_name);
+
+            if (!_slots.ContainsKey(var.UniqueId))
+                _slots[var.UniqueId] = new Dictionary<string, IVariableV1>();
+            var slot_dict = _slots[var.UniqueId];
+            if (!slot_dict.ContainsKey(slot_name))
+            {
+                var weight = tf.Variable(initializer,
+                    dtype: var.dtype,
+                    trainable: false,
+                    shape: var.shape,
+                    name: $"{var.Name}/{slot_name}");
+
+                slot_dict[slot_name] = weight;
+                _weights.append(weight);
+                return weight;
+            }
+            else
+            {
+                return slot_dict[slot_name];
+            }
+        }
+
         ResourceVariable add_weight(string name,
             TensorShape shape,
             TF_DataType dtype = TF_DataType.TF_FLOAT,

src/TensorFlowNET.Core/Keras/Optimizers/SGD.cs

@@ -40,7 +40,7 @@ protected override void _prepare_local(DeviceDType device_dtype,
                 _get_hyper("momentum", device_dtype.DType));
         }
 
-        protected override Operation _resource_apply_dense(IVariableV1 var, EagerTensor grad, Dictionary<DeviceDType, Dictionary<string, Tensor>> _apply_state)
+        protected override Operation _resource_apply_dense(IVariableV1 var, Tensor grad, Dictionary<DeviceDType, Dictionary<string, Tensor>> _apply_state)
         {
             if (_momentum)
             {