Allow Tensors to extend.

Oceania2018 · Oceania2018 · commit 505565550fd6 · 2020-12-26T20:42:42.000-06:00
diff --git a/src/TensorFlowNET.Core/Functions/ConcreteFunction.cs b/src/TensorFlowNET.Core/Functions/ConcreteFunction.cs
@@ -139,7 +139,7 @@ public Tensor[] CallFlat(Tensor[] args, Tensor[] captured_inputs)
                 "executor_type", "", 
                 "config_proto", tf.Context.FunctionCallOptions.config_proto_serialized()
             };
-            return tf.Runner.Execute(tf.Context, func_graph.FuncName, 1, args, attrs);
+            return tf.Runner.Execute(tf.Context, func_graph.FuncName, func_graph.Outputs.Length, args, attrs);
         }
 
         ForwardBackwardCall SelectForwardAndBackwardFunctions(Tensors args, int possible_gradient_type, bool executing_eagerly)
diff --git a/src/TensorFlowNET.Core/Functions/TapeGradientFunctions.cs b/src/TensorFlowNET.Core/Functions/TapeGradientFunctions.cs
@@ -1,5 +1,6 @@
 ﻿using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Text;
 using Tensorflow.Graphs;
 using static Tensorflow.Binding;
@@ -61,7 +62,7 @@ public void Record(Tensors flat_outputs, Tensors inference_args)
                     processed_args.add(arg);
                     input_index += 1;
                 }
-
+                tf.Logger.Debug($"Invoke backward function: {backward.Name}");
                 return backward.CallFlat(processed_args.ToArray(), outputs);
             };
 
@@ -91,6 +92,14 @@ public void Record(Tensors flat_outputs, Tensors inference_args)
                 grad_ys: gradients_wrt_outputs.ToArray(),
                 src_graph: _func_graph);
 
+            var captures_from_forward = backwards_graph.external_captures()
+                .Where(x => !x.IsEagerTensor && x.graph == _func_graph)
+                .ToArray();
+            foreach(var capture in captures_from_forward)
+            {
+                _func_graph.Outputs.Add(capture);
+            }
+
             var forward_function_name = $"{_FORWARD_PREFIX}_{ops.uid()}";
             var backward_function_attr = new Dictionary<string, string>();
             backward_function_attr[FORWARD_FUNCTION_ATTRIBUTE_NAME] = forward_function_name;
diff --git a/src/TensorFlowNET.Core/Graphs/AutoGraphAttribute.cs b/src/TensorFlowNET.Core/Graphs/AutoGraphAttribute.cs
@@ -18,7 +18,7 @@ public sealed class AutoGraphAttribute : OnMethodBoundaryAspect
 
         public override void OnEntry(MethodExecutionArgs args)
         {
-            func_name = $"autograph_{args.Instance.GetType().FullName}.{args.Method.Name}";
+            func_name = $"autograph_{args.Instance.GetHashCode()}.{args.Method.Name}";
 
             if (functions.ContainsKey(func_name))
             {
@@ -44,13 +44,13 @@ public override void OnEntry(MethodExecutionArgs args)
             }
             else
             {
-                originalInputs = new Tensors(args.Arguments.Length);
+                originalInputs = new Tensors();
                 // convert args to placeholder
                 for (var i = 0; i < args.Arguments.Length; i++)
                 {
                     if (args.Arguments[i] is EagerTensor tensor)
                     {
-                        originalInputs[i] = tensor;
+                        originalInputs.Add(tensor);
                         args.Arguments[i] = tf.placeholder(tensor.dtype, shape: tensor.TensorShape, name: "inputs");
                     }
                 }
diff --git a/src/TensorFlowNET.Core/Graphs/FuncGraph.cs b/src/TensorFlowNET.Core/Graphs/FuncGraph.cs
@@ -16,16 +16,23 @@ public class FuncGraph : Graph
         Graph outer_graph;
         public Graph OuterGraph => outer_graph;
 
-        string func_name;
-
         // _handle == IntPtr.Zero ? string.Empty : c_api.StringPiece(c_api.TF_FunctionName(_handle));
         IntPtr func_handle;
-        public string FuncName => func_name;
+        public string FuncName => _graph_key;
 
         public Tensors Inputs { get; set; }
         public Tensors Outputs { get; set; }
         public Dictionary<string, string> Attrs { get; set; }
 
+        public Dictionary<long, (Tensor, Tensor)> _captures 
+            = new Dictionary<long, (Tensor, Tensor)>();
+
+        public Tensor[] external_captures()
+            => _captures.Select(x => x.Value.Item1).ToArray();
+
+        public Tensor[] internal_captures()
+            => _captures.Select(x => x.Value.Item2).ToArray();
+
         // new Dictionary<long, (Tensor, Tensor)> _captures = new Dictionary<long, (Tensor, Tensor)>();
         // public new Tensor[] external_captures => _captures.Values.Select(x => x.Item1).ToArray();
 
@@ -35,7 +42,7 @@ public class FuncGraph : Graph
         public FuncGraph(string name) : base()
         {
             outer_graph = ops.get_default_graph();
-            func_name = name;
+            _graph_key = name;
 
             tf.Context.graph_mode();
             as_default();
@@ -44,7 +51,7 @@ public FuncGraph(string name) : base()
         public FuncGraph(IntPtr handle, string name, Dictionary<string, string> attrs) : base()
         {
             outer_graph = ops.get_default_graph();
-            func_name = name;
+            _graph_key = name;
             Attrs = attrs;
             // Will to test if FuncGraph has memory leak
             // c_api.TF_DeleteGraph(_handle);
@@ -60,7 +67,7 @@ public IntPtr ToGraph(Operation[] opers,
         {
             using var status = new Status();
             func_handle = c_api.TF_GraphToFunction(_handle,
-                func_name,
+                _graph_key,
                 false,
                 opers.Length,
                 opers.Select(x => (IntPtr)x).ToArray(),
@@ -82,7 +89,7 @@ public IntPtr ToGraph(Operation[] opers,
             c_api.TFE_ContextAddFunction(tf.Context.Handle, func_handle, status.Handle);
             status.Check(true);
 
-            func_name = c_api.StringPiece(c_api.TF_FunctionName(func_handle));
+            _graph_key = c_api.StringPiece(c_api.TF_FunctionName(func_handle));
 
             Inputs = inputs;
             // mark_as_return
@@ -131,7 +138,7 @@ Tensor capture(Tensor tensor, string name = null, TF_DataType shape = TF_DataTyp
         Tensor _capture_helper(Tensor tensor, string name, TensorShape shape = null)
         {
             Tensor placeholder = null;
-            if (!_captures.Contains(tensor.Id))
+            if (!_captures.ContainsKey(tensor.Id))
             {
                 placeholder = _create_substitute_placeholder(tensor,
                     name: name,
@@ -141,7 +148,7 @@ Tensor _capture_helper(Tensor tensor, string name, TensorShape shape = null)
             }
             else
             {
-                placeholder = (((Tensor, Tensor))_captures[tensor.Id]).Item2;
+                placeholder = _captures[tensor.Id].Item2;
             }
 
             BackwardFunction _backward_function_wrapper = (output_grads, unneeded_gradients) =>
@@ -160,7 +167,7 @@ Tensor _capture_helper(Tensor tensor, string name, TensorShape shape = null)
 
         void add_capture(Tensor tensor, Tensor placeholder)
         {
-            _captures[tensor.Id] = (tensor, placeholder);
+            _captures.Add(tensor.Id, (tensor, placeholder));
             if (Inputs == null)
                 Inputs = new Tensors(placeholder);
             else
diff --git a/src/TensorFlowNET.Core/Graphs/Graph.cs b/src/TensorFlowNET.Core/Graphs/Graph.cs
@@ -87,7 +87,7 @@ public partial class Graph : DisposableObject
         private List<Tensor> _unfeedable_tensors = new List<Tensor>();
 
         public string _name_stack = "";
-        private string _graph_key;
+        protected string _graph_key;
         public string graph_key => _graph_key;
         public string _last_loss_reduction;
         public bool _is_loss_scaled_by_optimizer { get; set; }
@@ -552,23 +552,5 @@ public static implicit operator IntPtr(Graph graph)
         {
             return graph._handle;
         }
-
-        public OrderedDictionary _captures => new OrderedDictionary();
-
-        public Tensor[] external_captures()
-        {
-            Tensor[] captures = new Tensor[_captures.Count];
-            ICollection inner = _captures.Keys; // c[0]
-            inner.CopyTo(captures, 0);
-            return captures;
-        }
-
-        public Tensor[] internal_captures()
-        {
-            Tensor[] captures = new Tensor[_captures.Count];
-            ICollection inner = _captures.Values; // c[1]
-            inner.CopyTo(captures, 0);
-            return captures;
-        }
     }
 }
diff --git a/src/TensorFlowNET.Core/Tensors/Tensors.cs b/src/TensorFlowNET.Core/Tensors/Tensors.cs
@@ -15,15 +15,15 @@ namespace Tensorflow
     /// </summary>
     public class Tensors : IEnumerable<Tensor>
     {
-        Tensor[] items;
+        List<Tensor> items = new List<Tensor>();
 
         public TF_DataType dtype => items.First().dtype;
         public TensorShape shape => items.First().TensorShape;
         public int rank => items.First().rank;
         public Graph graph => items.First().graph;
         public bool IsEagerTensor => items.First().IsEagerTensor;
         public bool IsList { get; set; }
-        public int Length => items.Length;
+        public int Length => items.Count();
 
         public Tensor this[int index]
         {
@@ -40,17 +40,12 @@ public Tensor this[int index]
 
         public Tensors(params Tensor[] tensors)
         {
-            items = tensors;
+            items.AddRange(tensors);
         }
 
         public Tensors(NDArray nd)
         {
-            items = new[] { ops.convert_to_tensor(nd) };
-        }
-
-        public Tensors(int count)
-        {
-            items = new Tensor[count];
+            items.Add(ops.convert_to_tensor(nd));
         }
 
         public IEnumerator<Tensor> GetEnumerator()
@@ -59,6 +54,9 @@ public IEnumerator<Tensor> GetEnumerator()
                 yield return tensor;
         }
 
+        public void Add(Tensor tensor)
+            => items.Add(tensor);
+
         IEnumerator IEnumerable.GetEnumerator()
         {
             throw new NotImplementedException();
@@ -80,11 +78,11 @@ public static implicit operator Tensor(Tensors tensors)
             => tensors.FirstOrDefault();
 
         public static implicit operator Tensor[](Tensors tensors)
-            => tensors.items;
+            => tensors.items.ToArray();
 
         public override string ToString()
-            => items.Length == 1
+            => items.Count() == 1
                ? items.First().ToString()
-               : items.Length + " Tensors" + ". " + string.Join(", ", items.Select(x => x.name));
+               : items.Count() + " Tensors" + ". " + string.Join(", ", items.Select(x => x.name));
     }
 }
diff --git a/src/TensorFlowNET.Core/Tensors/tensor_util.cs b/src/TensorFlowNET.Core/Tensors/tensor_util.cs
@@ -20,6 +20,7 @@ limitations under the License.
 using System.Linq;
 using System.Text;
 using Tensorflow.Eager;
+using Tensorflow.Graphs;
 using static Tensorflow.Binding;
 
 namespace Tensorflow
@@ -392,14 +393,14 @@ bool hasattr(Graph property, string attr)
             }
             else if (tensor.op.type == "Placeholder" &&
                   tensor.op.graph.building_function &&
-                  hasattr(tensor.op.graph, "internal_captures"))
+                  tensor.op.graph is FuncGraph func_graph)
             {
                 int i = 0;
-                foreach (Tensor capture in tensor.op.graph.internal_captures())
+                foreach (Tensor capture in func_graph.internal_captures())
                 {
                     if (capture.GetType() == typeof(Tensor))
                     {
-                        var external_capture = tensor.op.graph.external_captures()[i];
+                        var external_capture = func_graph.external_captures()[i];
                         return constant_value_as_shape(external_capture);
                     }
 
diff --git a/src/TensorFlowNET.Keras/Engine/Functional.cs b/src/TensorFlowNET.Keras/Engine/Functional.cs
@@ -337,10 +337,10 @@ Tensors run_internal_graph(Tensors inputs, bool training = false, Tensors mask =
 
                     var layer_inputs = node.MapArguments(tensor_dict);
 
-                    tf.Logger.Debug($"{depth}: {node.Layer}: {node.Layer.Name}");
+                    tf.Logger.Debug($"Depth {depth}: {node.Layer}: {node.Layer.Name}");
                     var outputs = node.Layer.Apply(layer_inputs, is_training: training);
                     foreach (var output in outputs.Where(x => x != null))
-                        tf.Logger.Debug($"{depth}: {node.Layer}: {node.Layer.Name} {output.TensorShape}");
+                        tf.Logger.Debug($"Depth {depth}: {node.Layer}: {node.Layer.Name} {output.TensorShape}");
                     // Update tensor_dict for next input
                     foreach (var (x_id, y) in zip(node.FlatOutputIds, outputs))
                         tensor_dict[x_id] = new Queue<Tensor>(Enumerable.Range(0, tensor_usage_count[x_id]).Select(x => y));
diff --git a/src/TensorFlowNET.Keras/Engine/TensorFlowOpLayer.cs b/src/TensorFlowNET.Keras/Engine/TensorFlowOpLayer.cs
@@ -60,7 +60,7 @@ Tensors MakOp(Tensors inputs)
 
             tf.Runner.RecordGradient(op_type, op.inputs._inputs, null, op.outputs);
 
-            return op.output;
+            return op.outputs;
         }
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -139,7 +139,7 @@ public Tensor[] CallFlat(Tensor[] args, Tensor[] captured_inputs)`
`139`	`139`	`"executor_type", "",`
`140`	`140`	`"config_proto", tf.Context.FunctionCallOptions.config_proto_serialized()`
`141`	`141`	`};`
`142`		`- return tf.Runner.Execute(tf.Context, func_graph.FuncName, 1, args, attrs);`
	`142`	`+ return tf.Runner.Execute(tf.Context, func_graph.FuncName, func_graph.Outputs.Length, args, attrs);`
`143`	`143`	`}`
`144`	`144`
`145`	`145`	`ForwardBackwardCall SelectForwardAndBackwardFunctions(Tensors args, int possible_gradient_type, bool executing_eagerly)`
Original file line number	Diff line number	Diff line change
`@@ -18,7 +18,7 @@ public sealed class AutoGraphAttribute : OnMethodBoundaryAspect`
`18`	`18`
`19`	`19`	`public override void OnEntry(MethodExecutionArgs args)`
`20`	`20`	`{`
`21`		`- func_name = $"autograph_{args.Instance.GetType().FullName}.{args.Method.Name}";`
	`21`	`+ func_name = $"autograph_{args.Instance.GetHashCode()}.{args.Method.Name}";`
`22`	`22`
`23`	`23`	`if (functions.ContainsKey(func_name))`
`24`	`24`	`{`
`@@ -44,13 +44,13 @@ public override void OnEntry(MethodExecutionArgs args)`
`44`	`44`	`}`
`45`	`45`	`else`
`46`	`46`	`{`
`47`		`- originalInputs = new Tensors(args.Arguments.Length);`
	`47`	`+ originalInputs = new Tensors();`
`48`	`48`	`// convert args to placeholder`
`49`	`49`	`for (var i = 0; i < args.Arguments.Length; i++)`
`50`	`50`	`{`
`51`	`51`	`if (args.Arguments[i] is EagerTensor tensor)`
`52`	`52`	`{`
`53`		`- originalInputs[i] = tensor;`
	`53`	`+ originalInputs.Add(tensor);`
`54`	`54`	`args.Arguments[i] = tf.placeholder(tensor.dtype, shape: tensor.TensorShape, name: "inputs");`
`55`	`55`	`}`
`56`	`56`	`}`
Original file line number	Diff line number	Diff line change
`@@ -15,15 +15,15 @@ namespace Tensorflow`
`15`	`15`	`/// </summary>`
`16`	`16`	`public class Tensors : IEnumerable<Tensor>`
`17`	`17`	`{`
`18`		`- Tensor[] items;`
	`18`	`+ List<Tensor> items = new List<Tensor>();`
`19`	`19`
`20`	`20`	`public TF_DataType dtype => items.First().dtype;`
`21`	`21`	`public TensorShape shape => items.First().TensorShape;`
`22`	`22`	`public int rank => items.First().rank;`
`23`	`23`	`public Graph graph => items.First().graph;`
`24`	`24`	`public bool IsEagerTensor => items.First().IsEagerTensor;`
`25`	`25`	`public bool IsList { get; set; }`
`26`		`- public int Length => items.Length;`
	`26`	`+ public int Length => items.Count();`
`27`	`27`
`28`	`28`	`public Tensor this[int index]`
`29`	`29`	`{`
`@@ -40,17 +40,12 @@ public Tensor this[int index]`
`40`	`40`
`41`	`41`	`public Tensors(params Tensor[] tensors)`
`42`	`42`	`{`
`43`		`- items = tensors;`
	`43`	`+ items.AddRange(tensors);`
`44`	`44`	`}`
`45`	`45`
`46`	`46`	`public Tensors(NDArray nd)`
`47`	`47`	`{`
`48`		`- items = new[] { ops.convert_to_tensor(nd) };`
`49`		`- }`
`50`		`-`
`51`		`- public Tensors(int count)`
`52`		`- {`
`53`		`- items = new Tensor[count];`
	`48`	`+ items.Add(ops.convert_to_tensor(nd));`
`54`	`49`	`}`
`55`	`50`
`56`	`51`	`public IEnumerator<Tensor> GetEnumerator()`
`@@ -59,6 +54,9 @@ public IEnumerator<Tensor> GetEnumerator()`
`59`	`54`	`yield return tensor;`
`60`	`55`	`}`
`61`	`56`
	`57`	`+ public void Add(Tensor tensor)`
	`58`	`+ => items.Add(tensor);`
	`59`	`+`
`62`	`60`	`IEnumerator IEnumerable.GetEnumerator()`
`63`	`61`	`{`
`64`	`62`	`throw new NotImplementedException();`
`@@ -80,11 +78,11 @@ public static implicit operator Tensor(Tensors tensors)`
`80`	`78`	`=> tensors.FirstOrDefault();`
`81`	`79`
`82`	`80`	`public static implicit operator Tensor[](Tensors tensors)`
`83`		`- => tensors.items;`
	`81`	`+ => tensors.items.ToArray();`
`84`	`82`
`85`	`83`	`public override string ToString()`
`86`		`- => items.Length == 1`
	`84`	`+ => items.Count() == 1`
`87`	`85`	`? items.First().ToString()`
`88`		`- : items.Length + " Tensors" + ". " + string.Join(", ", items.Select(x => x.name));`
	`86`	`+ : items.Count() + " Tensors" + ". " + string.Join(", ", items.Select(x => x.name));`
`89`	`87`	`}`
`90`	`88`	`}`
Original file line number	Diff line number	Diff line change
`@@ -20,6 +20,7 @@ limitations under the License.`
`20`	`20`	`using System.Linq;`
`21`	`21`	`using System.Text;`
`22`	`22`	`using Tensorflow.Eager;`
	`23`	`+using Tensorflow.Graphs;`
`23`	`24`	`using static Tensorflow.Binding;`
`24`	`25`
`25`	`26`	`namespace Tensorflow`
`@@ -392,14 +393,14 @@ bool hasattr(Graph property, string attr)`
`392`	`393`	`}`
`393`	`394`	`else if (tensor.op.type == "Placeholder" &&`
`394`	`395`	`tensor.op.graph.building_function &&`
`395`		`- hasattr(tensor.op.graph, "internal_captures"))`
	`396`	`+ tensor.op.graph is FuncGraph func_graph)`
`396`	`397`	`{`
`397`	`398`	`int i = 0;`
`398`		`- foreach (Tensor capture in tensor.op.graph.internal_captures())`
	`399`	`+ foreach (Tensor capture in func_graph.internal_captures())`
`399`	`400`	`{`
`400`	`401`	`if (capture.GetType() == typeof(Tensor))`
`401`	`402`	`{`
`402`		`- var external_capture = tensor.op.graph.external_captures()[i];`
	`403`	`+ var external_capture = func_graph.external_captures()[i];`
`403`	`404`	`return constant_value_as_shape(external_capture);`
`404`	`405`	`}`
`405`	`406`
Original file line number	Diff line number	Diff line change
`@@ -60,7 +60,7 @@ Tensors MakOp(Tensors inputs)`
`60`	`60`
`61`	`61`	`tf.Runner.RecordGradient(op_type, op.inputs._inputs, null, op.outputs);`
`62`	`62`
`63`		`- return op.output;`
	`63`	`+ return op.outputs;`
`64`	`64`	`}`
`65`	`65`	`}`
`66`	`66`	`}`