remove safe permute

Signed-off-by: Kyle Sayers <kylesayrs@gmail.com>

Author: kylesayrs

src/llmcompressor/observers/base.py

@@ -10,7 +10,6 @@
 )
 from compressed_tensors.quantization.utils import is_fp4
 from compressed_tensors.registry.registry import RegistryMixin
-from compressed_tensors.utils import safe_permute
 from loguru import logger
 from torch import FloatTensor, IntTensor, Tensor
 
@@ -56,7 +55,7 @@ def forward(
             # NOTE: this function updates running min/max values, which leads to
             # running values updating twice
             return self.get_gparam(observed=observed)
-        
+
         return self.get_qparams(
             observed=observed,
             g_idx=g_idx,
@@ -172,8 +171,7 @@ def get_qparams(
                     group_indices, group_sizes = torch.unique(g_idx, return_counts=True)
                     group_sizes = group_sizes[torch.argsort(group_indices)]
 
-                    perm = torch.argsort(g_idx)
-                    observed = safe_permute(observed, perm, dim=1)
+                    observed = observed.index_select(g_idx, -1)
 
                 # TODO: experiment with vectorizing for loop for performance
                 end = 0