Fix resize_nearest_neighbor_grad.

Author: Oceania2018

src/TensorFlowNET.Core/Functions/ConcreteFunction.cs

@@ -14,6 +14,7 @@ public class ConcreteFunction : IDisposable
     {
         IntPtr _handle;
         FuncGraph func_graph;
+        public Tensor[] Inputs => func_graph.Inputs;
         public Tensor[] CapturedInputs => func_graph.external_captures;
 
         public string Name
@@ -127,30 +128,53 @@ public void Exit()
             func_graph.Exit();
         }
 
-        public Tensors Invoke(Tensors inputs)
+        public Tensors FilteredCall(Tensors inputs)
         {
-            var forward_backward = SelectForwardAndBackwardFunctions(inputs, 1, tf.Context.executing_eagerly());
-            var (forward_function, args_with_tangents) = forward_backward.Forward();
-            Tensors flat_outputs = null;
-            if (tf.Context.executing_eagerly())
-                flat_outputs = forward_function.Call(args_with_tangents);
-            forward_backward.Record(flat_outputs);
-            return flat_outputs;
+            return CallFlat(inputs, CapturedInputs);
         }
 
+        /// <summary>
+        /// Executes the wrapped function.
+        /// </summary>
+        /// <param name="args"></param>
+        /// <param name="captured_inputs"></param>
+        /// <returns></returns>
         public Tensor[] CallFlat(Tensor[] args, Tensor[] captured_inputs)
         {
-            var new_args = new List<Tensor>();
-            new_args.AddRange(args);
-            new_args.AddRange(captured_inputs);
-            args = new_args.ToArray();
+            var executing_eagerly = tf.Context.executing_eagerly();
+            var default_graph = ops.get_default_graph();
+            var tensor_inputs = new Tensors();
+            foreach (var (i, arg) in enumerate(args))
+            {
+                tensor_inputs.Add(arg);
+                // If we're graph building, shape inference is on.
+                if (!executing_eagerly)
+                {
+                }
+            }
+            tensor_inputs.AddRange(captured_inputs);
+
+            args = tensor_inputs.ToArray();
 
-            var attrs = new object[] 
+            var possible_gradient_type = tf.Runner.MustRecordGradient() ? 1 : 0;
+            // No tape is watching; skip to running the function.
+            if (possible_gradient_type == 0 && executing_eagerly)
             {
-                "executor_type", "", 
-                "config_proto", tf.Context.FunctionCallOptions.config_proto_serialized()
-            };
-            return tf.Runner.Execute(tf.Context, func_graph.FuncName, func_graph.Outputs.Length, args, attrs);
+                var attrs = new object[]
+                {
+                    "executor_type", "",
+                    "config_proto", tf.Context.FunctionCallOptions.config_proto_serialized()
+                };
+                return tf.Runner.Execute(tf.Context, func_graph.FuncName, func_graph.Outputs.Length, args, attrs);
+            }
+
+            var forward_backward = SelectForwardAndBackwardFunctions(args, possible_gradient_type, executing_eagerly);
+            var (forward_function, args_with_tangents) = forward_backward.Forward();
+            Tensors flat_outputs = null;
+            if (executing_eagerly)
+                flat_outputs = forward_function.Call(args_with_tangents);
+            forward_backward.Record(flat_outputs);
+            return flat_outputs;
         }
 
         ForwardBackwardCall SelectForwardAndBackwardFunctions(Tensors args, int possible_gradient_type, bool executing_eagerly)

src/TensorFlowNET.Core/Functions/EagerDefinedFunction.cs

@@ -31,11 +31,17 @@ public EagerDefinedFunction(string name, FuncGraph graph,
 
         public Tensors Call(Tensors args)
         {
+            var attrs = new object[]
+                {
+                    "executor_type", "",
+                    "config_proto", tf.Context.FunctionCallOptions.config_proto_serialized()
+                };
+
             var results = tf.Runner.TFE_Execute(tf.Context,
                 tf.Context.DeviceName,
                 _func_graph.FuncName,
                 args,
-                null,
+                attrs,
                 _num_outputs);
 
             return results;

src/TensorFlowNET.Core/Functions/TapeGradientFunctions.cs

@@ -49,24 +49,61 @@ public void Record(Tensors flat_outputs, Tensors inference_args)
                 getBackwardFunction: () => backward_function);
         }
 
+        /// <summary>
+        /// Create a backward function given `outputs` from the forward function.
+        /// </summary>
+        /// <param name="forward_graph"></param>
+        /// <param name="backward"></param>
+        /// <param name="outputs"></param>
+        /// <returns></returns>
         (BackwardFunction, Tensors) _wrap_backward_function(FuncGraph forward_graph, ConcreteFunction backward, Tensors outputs)
         {
-            BackwardFunction _backward_function_wrapper = (output_grads, unneeded_gradients) =>
+            var capture_mapping = new Dictionary<long, Tensor>();
+            foreach(var (i, output) in enumerate(outputs))
+                capture_mapping[forward_graph.Outputs[i].Id] = output;
+
+            var remapped_captures = new Tensors();
+            foreach(var capture in backward.CapturedInputs)
+            {
+                if (capture_mapping.ContainsKey(capture.Id))
+                    remapped_captures.Add(capture_mapping[capture.Id]);
+            }
+
+            var backward_function_inputs = backward.Inputs.Length - backward.CapturedInputs.Length;
+            var recorded_outputs = new Tensors();
+            var relevant_outputs = outputs;
+            var trainable_recorded_outputs = 0;
+            var skip_positions = new List<int>();
+            foreach (var (output_index, output) in enumerate(relevant_outputs))
+            {
+                if (trainable_recorded_outputs < backward_function_inputs)
+                    recorded_outputs.Add(output);
+                if (gradients_util.IsTrainable(output))
+                    trainable_recorded_outputs += 1;
+                else
+                    skip_positions.Add(output_index);
+            }
+
+            BackwardFunction _backward_function_wrapper = (args, unneeded_gradients) =>
             {
-                var processed_args = new List<Tensor>();
+                var processed_args = new Tensors();
                 var input_index = 0;
-                foreach (var (output_index, arg) in enumerate(output_grads))
+                foreach (var (output_index, arg) in enumerate(args))
                 {
-                    if (arg is null)
+                    if (skip_positions.Contains(output_index))
+                        continue;
+                    if (arg == null)
                         throw new NotImplementedException("");
-                    processed_args.add(arg);
+                    processed_args.Add(arg);
                     input_index += 1;
+                    if (input_index >= backward_function_inputs)
+                        break;
                 }
                 tf.Logger.Debug($"Invoke backward function: {backward.Name}");
-                return backward.CallFlat(processed_args.ToArray(), outputs);
+                return backward.CallFlat(processed_args, remapped_captures);
             };
 
-            return (_backward_function_wrapper, outputs);
+            return (_backward_function_wrapper, recorded_outputs);
         }
 
         protected (EagerDefinedFunction, FuncGraph, ConcreteFunction, List<int>, int) 
@@ -103,7 +140,7 @@ public void Record(Tensors flat_outputs, Tensors inference_args)
             }
             backwards_graph.Exit();
 
-            var forward_function_name = $"{_FORWARD_PREFIX}_{ops.uid()}";
+            var forward_function_name = $"{_FORWARD_PREFIX}_{_func_graph.FuncName}_{ops.uid()}";
             var backward_function_attr = new Dictionary<string, string>();
             backward_function_attr[FORWARD_FUNCTION_ATTRIBUTE_NAME] = forward_function_name;
             gradients_wrt_outputs.append(backwards_graph.internal_captures);

src/TensorFlowNET.Core/Gradients/array_grad.cs

@@ -228,13 +228,14 @@ public static Tensor[] _PadGrad(Operation op, Tensor[] grads)
             var grad = grads[0];
             var x = op.inputs[0];
             var a = op.inputs[1];
-            var size = array_ops.stack(new object[] { array_ops.rank(x), 1 });
-            var pad_before = array_ops.slice(a, new[] { 0, 0 }, size);
+            var size = array_ops.stack(new Tensor[] { array_ops.rank(x), constant_op.constant(1) });
+            var begin = constant_op.constant(new[] { 0, 0 });
+            var pad_before = array_ops.slice(a, begin, size);
 
             // Make it a 1-D tensor.
-            var begin = array_ops.reshape(pad_before, new[] { -1 });
-            var sizes = array_ops.shape(x);
-            var x_grad = array_ops.slice(grad, begin, sizes);
+            begin = array_ops.reshape(pad_before, new[] { -1 });
+            size = array_ops.shape(x);
+            var x_grad = array_ops.slice(grad, begin, size);
 
             if (len(op.inputs) == 3)
                 return new Tensor[] { x_grad, null, null };

src/TensorFlowNET.Core/Gradients/image_grad.cs

@@ -30,7 +30,7 @@ public static Tensor[] _ResizeNearestNeighborGrad(Operation op, Tensor[] grads)
             var shape = new TensorShape(image.shape.Skip(1).Take(2).ToArray());
             Tensor image_shape = null;
             if (shape.is_fully_defined())
-                throw new NotImplementedException("_ResizeNearestNeighborGrad shape.is_fully_defined");
+                image_shape = constant_op.constant(image.shape[1..3]);
             else
                 image_shape = array_ops.shape(image)["1:3"];
 

src/TensorFlowNET.Core/Graphs/AutoGraphAttribute.cs

@@ -8,6 +8,9 @@
 
 namespace Tensorflow.Graphs
 {
+    /// <summary>
+    /// func_graph.py func_graph_from_py_func
+    /// </summary>
     [AllowChangingInputArguments]
     public sealed class AutoGraphAttribute : OnMethodBoundaryAspect
     {
@@ -18,15 +21,16 @@ public sealed class AutoGraphAttribute : OnMethodBoundaryAspect
 
         public override void OnEntry(MethodExecutionArgs args)
         {
-            func_name = $"{args.Method.Name}_{Guid.NewGuid()}";
+            // TODO: func_name can be cache in FullName + Args
+            func_name = $"{args.Method.DeclaringType.FullName}.{args.Method.Name}_{Guid.NewGuid()}";
 
             if (functions.ContainsKey(func_name))
             {
                 function = functions[func_name];
                 if (args.Arguments[0] is Tensors tensor_inputs)
-                    args.ReturnValue = ConvertReturnValue(function.Invoke(tensor_inputs));
+                    args.ReturnValue = ConvertReturnValue(function.FilteredCall(tensor_inputs));
                 else
-                    args.ReturnValue = ConvertReturnValue(function.Invoke(args.Arguments.Select(x => x as Tensor).ToArray()));
+                    args.ReturnValue = ConvertReturnValue(function.FilteredCall(args.Arguments.Select(x => x as Tensor).ToArray()));
                 args.FlowBehavior = FlowBehavior.Return;
                 return;
             }
@@ -62,22 +66,35 @@ public override void OnExit(MethodExecutionArgs args)
         {
             if (args.ReturnValue is Tensors outputs)
             {
-                if (args.Arguments[0] is Tensors inputs)
-                    function.ToGraph(inputs, outputs);
+                Tensors inputs = null;
+                outputs = mark_as_return(outputs);
+                if (args.Arguments[0] is Tensors inputs1)
+                    inputs = inputs1;
                 else
-                    function.ToGraph(args.Arguments.Select(x => x as Tensor).ToArray(), outputs);
+                    inputs = args.Arguments.Select(x => x as Tensor).ToArray();
+
+                inputs = inputs.Where(x => x.op.OpType == "Placeholder" 
+                    && x.op.name.StartsWith("inputs")).ToArray();
+
+                function.ToGraph(inputs, outputs);
             }
-            else
-                function.ToGraph(args.Arguments.Select(x => x as Tensor).ToArray(), args.ReturnValue as Tensor);
-            
+            else if (args.ReturnValue is Tensor output)
+            {
+                var inputs = args.Arguments.Select(x => x as Tensor)
+                    .Where(x => x.op.type == "Placeholder" && x.op.name.StartsWith("inputs"))
+                    .ToArray();
+                var outputs2 = array_ops.identity(output);
+                function.ToGraph(inputs, outputs2);
+            }
+
             function.Exit();
 
             // cache function.
             function.ReturnType = args.ReturnValue.GetType();
             functions[func_name] = function;
 
             // run function
-            args.ReturnValue = ConvertReturnValue(function.Invoke(originalInputs));
+            args.ReturnValue = ConvertReturnValue(function.FilteredCall(originalInputs));
         }
 
         object ConvertReturnValue(Tensors tensors)
@@ -87,5 +104,20 @@ object ConvertReturnValue(Tensors tensors)
             else
                 return tensors;
         }
+
+        /// <summary>
+        /// Acts like identity but marks the `Tensor` as a return value.
+        /// </summary>
+        /// <param name="tensors"></param>
+        /// <returns></returns>
+        public Tensors mark_as_return(Tensors tensors)
+        {
+            if (tensors == null)
+                return null;
+            var result = new Tensors();
+            foreach (var tensor in tensors)
+                result.Add(array_ops.identity(tensor));
+            return result;
+        }
     }
 }

src/TensorFlowNET.Core/Operations/array_ops.cs

@@ -925,7 +925,28 @@ public static Tensor slice(Tensor input, Tensor[] begin, Tensor[] size, string n
         public static Tensor slice<Tb, Ts>(Tensor input, Tb begin, Ts size, string name = null)
             => gen_array_ops.slice(input, begin, size, name: name);
 
-        public static Tensor stack(object values, int axis = 0, string name = "stack")
+        public static Tensor slice(Tensor input, Tensor begin, Tensor size, string name = null)
+            => tf.Context.RunInAutoMode2(
+                () => tf.OpDefLib._apply_op_helper("Slice", name, new
+                {
+                    input, begin, size
+                }).output,
+                () => tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
+                    "Slice", name,
+                    null,
+                    input, begin, size).FirstOrDefault(),
+                (op) =>
+                {
+                    var attrs = new object[]
+                    {
+                        "T", op.get_attr<TF_DataType>("T"),
+                        "Index", op.get_attr<int>("Index")
+                    };
+                    tf.Runner.RecordGradient("Slice", op.inputs, attrs, op.outputs);
+                },
+                new Tensors(input, begin, size));
+        
+    public static Tensor stack(object values, int axis = 0, string name = "stack")
         {
             if (axis == 0)
                 // If the input is a constant list, it can be converted to a constant op

src/TensorFlowNET.Core/Operations/gen_image_ops.cs

@@ -238,18 +238,32 @@ public static Tensor resize_nearest_neighbor<Tsize>(Tensor images, Tsize size, b
                     "half_pixel_centers", half_pixel_centers).FirstOrDefault(),
                 images);
 
-        public static Tensor resize_nearest_neighbor_grad<Tsize>(Tensor grads, Tsize size, bool align_corners = false,
+        public static Tensor resize_nearest_neighbor_grad(Tensor grads, Tensor size, bool align_corners = false,
             bool half_pixel_centers = false, string name = null)
-        {
-            var op = tf.OpDefLib._apply_op_helper("ResizeNearestNeighborGrad", name: name, args: new
-            {
-                grads,
-                size,
-                align_corners,
-                half_pixel_centers
-            });
-
-            return op.output;
-        }
+            => tf.Context.RunInAutoMode2(
+                () => tf.OpDefLib._apply_op_helper("ResizeNearestNeighborGrad", name, new
+                {
+                    grads,
+                    size,
+                    align_corners,
+                    half_pixel_centers
+                }).output,
+                () => tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
+                    "ResizeNearestNeighborGrad", name,
+                    null,
+                    grads, size,
+                    "align_corners", align_corners,
+                    "half_pixel_centers", half_pixel_centers).FirstOrDefault(),
+                (op) =>
+                {
+                    var attrs = new object[]
+                    {
+                        "T", op.get_attr<TF_DataType>("T"),
+                        "align_corners", op.get_attr<bool>("align_corners"),
+                        "half_pixel_centers", op.get_attr<bool>("half_pixel_centers")
+                    };
+                    tf.Runner.RecordGradient("ResizeNearestNeighborGrad", op.inputs, attrs, op.outputs);
+                },
+                new Tensors(grads, size));
     }
 }

src/TensorFlowNET.Core/Operations/gen_random_ops.cs

@@ -126,6 +126,16 @@ public static Tensor random_uniform(Tensor shape, TF_DataType dtype, int? seed =
         public static Tensor random_shuffle(Tensor value, int seed = 0, int seed2 = 0,
             string name = null)
         {
+            if (tf.Context.executing_eagerly())
+            {
+                var results = tf.Runner.TFE_FastPathExecute(tf.Context, tf.Context.DeviceName,
+                    "RandomShuffle", name,
+                    null,
+                    value, seed, seed2);
+
+                return results[0];
+            }
+
             var _op = tf.OpDefLib._apply_op_helper("RandomShuffle",
                 name: name,
                 args: new { value, seed, seed2 });

src/TensorFlowNET.Core/Tensorflow.Binding.csproj

@@ -83,6 +83,7 @@ TensorFlow .NET v0.30 is focused on making more Keras API work including:
 
   <ItemGroup>
     <PackageReference Include="MethodBoundaryAspect.Fody" Version="2.0.138" />
+    <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="5.0.1" />
     <PackageReference Include="NumSharp.Lite" Version="0.1.10" />
     <PackageReference Include="Protobuf.Text" Version="0.4.0" />
     <PackageReference Include="Serilog.Sinks.Console" Version="3.1.1" />