[Transform] Do not fuse div operation into hadamard matrices (#395)

Signed-off-by: Kyle Sayers <kylesayrs@gmail.com>

Author: kylesayrs

src/compressed_tensors/transform/factory/hadamard.py

@@ -14,6 +14,7 @@
 
 from typing import Optional, Union
 
+import math
 import torch
 from compressed_tensors.transform import TransformArgs, TransformScheme
 from compressed_tensors.transform.factory.base import TransformBase, TransformFactory
@@ -87,6 +88,7 @@ def __init__(
         self.weight = weight
         self.perm = perm
         self.args = args
+        self._scale = math.sqrt(weight.size(0))
 
     def forward(self, value: Tensor) -> Tensor:
         weight = self.weight
@@ -97,4 +99,4 @@ def forward(self, value: Tensor) -> Tensor:
         if self.args.inverse:
             weight = weight.T
 
-        return apply_transform_weight(weight, value, self.args.location)
+        return apply_transform_weight(weight, value, self.args.location) / self._scale

src/compressed_tensors/transform/utils/hadamard.py

@@ -59,7 +59,7 @@ def deterministic_hadamard_matrix(
     for _ in range(log2):
         H = torch.vstack((torch.hstack((H, H)), torch.hstack((H, -H))))
 
-    return H / math.sqrt(size)
+    return H
 
 
 def random_hadamard_matrix(
@@ -86,7 +86,7 @@ def random_hadamard_matrix(
     Q = Q.to(device=device)
     Q = Q * 2 - 1
     Q = torch.diag(Q)
-    return _matmul_hadU(Q) / math.sqrt(size)
+    return _matmul_hadU(Q)
 
 
 def is_pow2(n: int) -> bool:

tests/test_transform/utils/test_hadamard.py

@@ -45,7 +45,7 @@
 def test_random_hadamard_matrix_compliant(size):
     # (H / sqrt(n))(H.T / sqrt(n)) == I
     matrix = random_hadamard_matrix(size, device="cuda")
-    product = matrix @ matrix.T
+    product = (matrix @ matrix.T) / matrix.size(0)
     eye = torch.eye(size, dtype=product.dtype, device="cuda")
     assert torch.allclose(product, eye, atol=_atol)
 
@@ -85,6 +85,6 @@ def test_deterministic_hadamard_compliant(size):
 
     # (H / sqrt(n))(H.T / sqrt(n)) == I
     matrix = deterministic_hadamard_matrix(size, device="cuda")
-    product = matrix @ matrix.T
+    product = (matrix @ matrix.T) / matrix.size(0)
     eye = torch.eye(size, dtype=product.dtype, device="cuda")
     assert torch.allclose(product, eye, atol=_atol)