Update Adam.step.

* Update src/TorchSharp/Optimizers/Adam.cs.
  + Update Adam.step.
    - Declare TorchSharp.Scalar explicitly.
      - Add FIXME for possible unused weight_decay_scalar.
    - Cache weight_decay != 0.
    - Add FIXME for possible no denom disposing.

Author: masaru-kimura-hacarus

src/TorchSharp/Optimizers/Adam.cs

@@ -154,10 +154,18 @@ public override Tensor step(Func<Tensor> closure = null)
                     var options = group.Options as Options;
                     var beta1 = options.beta1.Value;
                     var beta2 = options.beta2.Value;
+                    using var beta1_scalar = beta1.ToScalar();
+                    using var beta2_scalar = beta2.ToScalar();
+                    var beta1_bar = 1 - beta1;
+                    var beta2_bar = 1 - beta2;
+                    using var beta1_bar_scalar = beta1_bar.ToScalar();
+                    using var beta2_bar_scalar = beta2_bar.ToScalar();
                     var weight_decay = options.weight_decay.Value;
+                    var need_weight_decay = weight_decay != 0;
+                    using var weight_decay_scalar = weight_decay.ToScalar(); // FIXME: Omit if not need_weight_decay?
                     var amsgrad = options.amsgrad.Value;
                     var maximize = options.maximize.Value;
-                    var eps = options.eps.Value;
+                    using var eps_scalar = options.eps.Value.ToScalar();
                     var lr = options.LearningRate.Value;
 
                     foreach (var param in group.Parameters) {
@@ -175,25 +183,24 @@ public override Tensor step(Func<Tensor> closure = null)
                         var bias_correction1 = 1 - Math.Pow(beta1, state.step);
                         var bias_correction2 = 1 - Math.Pow(beta2, state.step);
 
-                        if (weight_decay != 0) {
-                            grad = grad.add(param, alpha: weight_decay);
-                        }
+                        if (need_weight_decay) grad = grad.add(param, alpha: weight_decay_scalar);
 
-                        state.exp_avg.mul_(beta1).add_(grad, alpha: 1 - beta1);
-                        state.exp_avg_sq.mul_(beta2).addcmul_(grad, grad.conj(), value: 1 - beta2);
+                        state.exp_avg.mul_(beta1_scalar).add_(grad, alpha: beta1_bar_scalar);
+                        state.exp_avg_sq.mul_(beta2_scalar).addcmul_(grad, grad.conj(), value: beta2_bar_scalar);
 
-                        Tensor denom = null;
+                        Tensor denom = null; // FIXME: Need dispose?
                         if (amsgrad) {
                             var t0 = state.max_exp_avg_sq;
                             state.max_exp_avg_sq = torch.maximum(t0, state.exp_avg_sq).DetachFromDisposeScope();
                             t0.Dispose();
-                            denom = (state.max_exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(eps);
+                            denom = (state.max_exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(eps_scalar);
                         } else {
-                            denom = (state.exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(eps);
+                            denom = (state.exp_avg_sq.sqrt() / Math.Sqrt(bias_correction2)).add_(eps_scalar);
                         }
 
                         var step_size = lr / bias_correction1;
-                        param.addcdiv_(state.exp_avg, denom, value: -step_size);
+                        using var negative_step_size_scalar = (-step_size).ToScalar();
+                        param.addcdiv_(state.exp_avg, denom, value: negative_step_size_scalar);
                     }
                 }, closure);
             }