_ReluGrad, _BiasAddGrad

Author: Oceania2018

src/TensorFlowNET.Core/Gradients/array_grad.py.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Tensorflow.Gradients
+{
+    public class array_grad
+    {
+        public static (Tensor, Tensor) _ReshapeGrad(Operation op, Tensor grad)
+        {
+            return (array_ops.reshape(grad, array_ops.shape(op.inputs[0])), null);
+        }
+    }
+}

src/TensorFlowNET.Core/Gradients/math_grad.py.cs

@@ -3,7 +3,7 @@
 using System.Linq;
 using System.Text;
 
-namespace Tensorflow
+namespace Tensorflow.Gradients
 {
     /// <summary>
     /// Gradients for operators defined in math_ops.py.
@@ -57,6 +57,38 @@ public static (Tensor, Tensor) _MulGrad(Operation op, Tensor grad)
             return (reshape1, reshape2);
         }
 
+        public static (Tensor, Tensor) _MatMulGrad(Operation op, Tensor grad)
+        {
+            Tensor grad_a = null, grad_b = null;
+
+            var t_a = (bool)op.get_attr("transpose_a");
+            var t_b = (bool)op.get_attr("transpose_b");
+            var a = math_ops.conj(op.inputs[0]);
+            var b = math_ops.conj(op.inputs[1]);
+            if(!t_a && !t_b)
+            {
+                grad_a = gen_math_ops.mat_mul(grad, b, transpose_b: true);
+                grad_b = gen_math_ops.mat_mul(a, grad, transpose_a: true);
+            }
+            else if (!t_a && t_b)
+            {
+                grad_a = gen_math_ops.mat_mul(grad, b);
+                grad_b = gen_math_ops.mat_mul(grad, a, transpose_a: true);
+            }
+            else if (t_a && !t_b)
+            {
+                grad_a = gen_math_ops.mat_mul(grad, b);
+                grad_b = gen_math_ops.mat_mul(grad, a, transpose_a: true);
+            }
+            else if (t_a && t_b)
+            {
+                grad_a = gen_math_ops.mat_mul(b, grad, transpose_a: true, transpose_b: true);
+                grad_b = gen_math_ops.mat_mul(grad, a, transpose_a: true, transpose_b: true);
+            }
+
+            return (grad_a, grad_b);
+        }
+
         public static (Tensor, Tensor) _MeanGrad(Operation op, Tensor grad)
         {
             var sum_grad = _SumGrad(op, grad).Item1;

src/TensorFlowNET.Core/Gradients/nn_grad.py.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Tensorflow.Operations;
+
+namespace Tensorflow.Gradients
+{
+    public class nn_grad
+    {
+        /// <summary>
+        /// Return the gradients for the 2 inputs of bias_op.
+        /// </summary>
+        /// <param name="op"></param>
+        /// <param name="grad"></param>
+        /// <returns></returns>
+        public static (Tensor, Tensor) _BiasAddGrad(Operation op, Tensor grad)
+        {
+            string data_format = op.get_attr("data_format")?.ToString();
+            var bias_add_grad = gen_nn_ops.bias_add_grad(out_backprop: grad, data_format: data_format);
+            return (grad, bias_add_grad);
+        }
+
+        public static (Tensor, Tensor) _ReluGrad(Operation op, Tensor grad)
+        {
+            return (gen_nn_ops.relu_grad(grad, op.outputs[0]), null);
+        }
+    }
+}

src/TensorFlowNET.Core/Operations/Activation/gen_nn_ops.py.cs renamed to src/TensorFlowNET.Core/Operations/Activation/gen_nn_ops.activations.cs

@@ -53,6 +53,22 @@ public static Tensor bias_add(Tensor value,
             return _op.outputs[0];
         }
 
+        public static Tensor bias_add_grad(Tensor out_backprop,
+            string data_format = "NHWC",
+            string name = null)
+        {
+            if (data_format == null)
+                data_format = "NHWC";
+
+            var _op = _op_def_lib._apply_op_helper("BiasAddGrad", name: name, args: new
+            {
+                out_backprop,
+                data_format
+            });
+
+            return _op.outputs[0];
+        }
+
         public static Tensor[] _fused_batch_norm(Tensor x,
                 Tensor scale,
                 Tensor offset,
@@ -109,6 +125,17 @@ public static Tensor[] top_kv2(Tensor input, int k, bool sorted = true, string n
             return _op.outputs;
         }
 
+        public static Tensor relu_grad(Tensor gradients, Tensor features, string name = null)
+        {
+            var _op = _op_def_lib._apply_op_helper("ReluGrad", name: name, args: new
+            {
+                gradients,
+                features
+            });
+
+            return _op.outputs[0];
+        }
+
         /// <summary>
         /// Computes softmax cross entropy cost and gradients to backpropagate.
         /// </summary>

src/TensorFlowNET.Core/Operations/NnOps/gen_nn_ops.cs

@@ -9,6 +9,7 @@
 using System.Linq;
 using NumSharp.Core;
 using System.ComponentModel;
+using Tensorflow.Gradients;
 
 namespace Tensorflow
 {
@@ -380,6 +381,9 @@ public static Func<Operation, Tensor, Tensor[]> get_gradient_function(Operation
                     case "RealDiv":
                         var realdiv = math_grad._RealDivGrad(oper, out_grads);
                         return new Tensor[] { realdiv.Item1, realdiv.Item2 };
+                    case "Reshape":
+                        var reshape = array_grad._ReshapeGrad(oper, out_grads);
+                        return new Tensor[] { reshape.Item1, reshape.Item2 };
                     default:
                         throw new NotImplementedException($"get_gradient_function {oper.type}");
                 }