Update woq code for intel GPU (#1404)

Author: PenghuiCheng

examples/huggingface/pytorch/text-generation/quantization/run_generation_gpu_woq.py

@@ -50,37 +50,50 @@
 parser.add_argument("--tasks", nargs='+', default=["lambada_openai"], type=str, \
                     help="tasks list for accuracy validation")
 # ============WeightOnlyQuant configs===============
+parser.add_argument("--bits", type=int, default=4, choices=[4])
 parser.add_argument("--woq", action="store_true")
 parser.add_argument("--woq_algo", default="RTN", choices=['RTN', 'GPTQ'], 
                     help="Weight-only parameter.")
-parser.add_argument("--woq_dtype", type=str, default="int4_fullrange",
+parser.add_argument("--weight_dtype", type=str, default="int4_fullrange",
                     choices=["int4_fullrange"])
-parser.add_argument("--woq_group_size", type=int, default=32)
-parser.add_argument("--woq_scheme", default="sym")
+parser.add_argument("--group_size", type=int, default=32)
+parser.add_argument("--scheme", default="sym")
 parser.add_argument("--woq_enable_mse_search", action="store_true")
 parser.add_argument("--device", default="xpu")
 parser.add_argument("--compute_dtype", default="fp16")
+# ============GPTQ configs==============
 parser.add_argument(
-    "--gptq_percdamp",
+    "--desc_act",
+    action="store_true",
+    help="Whether to apply the activation order GPTQ heuristic.",
+)
+parser.add_argument(
+    "--damp_percent",
     type=float,
     default=0.01,
     help="Percent of the average Hessian diagonal to use for dampening.",
 )
 parser.add_argument(
-    "--gptq_block_size",
+    "--blocksize",
     type=int,
     default=128,
     help="Block size. sub weight matrix size to run GPTQ.",
 )
 parser.add_argument(
-    "--gptq_nsamples", type=int, default=128, help="Number of calibration data samples."
+    "--nsamples", type=int, default=128, help="Number of calibration data samples."
 )
 parser.add_argument(
     "--max_input_length",
     type=int,
     default=2048,
     help="Calibration dataset sequence max length, this should align with your model config",
 )
+parser.add_argument(
+    "--static_groups",
+    action="store_true",
+    help="Use determined group to do quantization",
+)
+parser.add_argument("--calib_iters", default=100, type=int, help="Calibration iters.")
 # ============BitsAndBytes configs==============
 parser.add_argument("--bitsandbytes", action="store_true")
 parser.add_argument("--load_in_4bit", type=bool, default=False)
@@ -118,22 +131,22 @@
             dataset=args.dataset,
             bits=args.bits,
             desc_act=args.desc_act,
-            damp_percent=args.gptq_percdamp,
-            sym=True if args.woq_scheme == "sym" else False,
-            blocksize=args.gptq_block_size,
-            nsamples=args.gptq_nsamples,
+            damp_percent=args.damp_percent,
+            sym=True if args.scheme == "sym" else False,
+            blocksize=args.blocksize,
+            nsamples=args.nsamples,
             static_groups=args.static_groups,
-            group_size=args.woq_group_size,
+            group_size=args.group_size,
             max_input_length=args.max_input_length,
             compute_dtype=args.compute_dtype,
             scale_dtype=args.compute_dtype,
-            weight_dtype=args.woq_dtype,
+            weight_dtype=args.weight_dtype,
             calib_iters=args.calib_iters,
         )
     else:
         quantization_config = RtnConfig(
-            compute_dtype=args.compute_dtype, weight_dtype=args.woq_dtype,
-            group_size=args.woq_group_size, scale_dtype=args.compute_dtype
+            compute_dtype=args.compute_dtype, weight_dtype=args.weight_dtype,
+            group_size=args.group_size, scale_dtype=args.compute_dtype
         ) #default is A16W4G16
 
 # get model
@@ -260,16 +273,17 @@
         args.model, trust_remote_code=args.trust_remote_code, device_map=args.device, torch_dtype=torch_dtype) \
             if user_model is None else user_model
     if quantization_config is None:
-        quantization_config = WeightOnlyQuantConfig.from_pretrained(args.model)
+        quantization_config = user_model.quantization_config if hasattr(user_model, "quantization_config") else {}
     if not args.disable_optimize_transformers:
         print("Optimize with IPEX...")
         user_model = ipex.optimize_transformers(
             user_model.eval(), device=args.device, inplace=True, quantization_config=quantization_config, dtype=torch_dtype)
     else:
         print("Disabled optimization with IPEX...")
+
     results = evaluate(
         model="hf-causal",
-        model_args='pretrained='+args.model+',tokenizer=' + args.model + \
+        model_args='pretrained=' + "facebook/opt-125m" +',tokenizer=' + args.model + \
             ',dtype=float32,trust_remote_code=' + str(args.trust_remote_code),
         user_model=user_model,
         batch_size=args.batch_size,

intel_extension_for_transformers/transformers/llm/quantization/utils.py

@@ -514,6 +514,13 @@ def default_calib_func(model):
         if config.quant_method.value not in ["awq"]:
             calib_func = None
 
+        orig_dtype = torch.float32
+        for param in model.parameters():
+            orig_dtype = param.dtype
+            if orig_dtype != torch.float32:
+                model.to(dtype=torch.float32)
+            break
+
         inc_model = quantization.fit(
             model, conf, calib_func=calib_func, calib_dataloader=calib_dataloader
         )
@@ -538,6 +545,8 @@ def default_calib_func(model):
                     inc_model.model, None, None, config, device=device
                 )
 
+        if orig_dtype != torch.float32:
+            q_model.to(dtype=orig_dtype)
         return q_model.to(device)
 
 

intel_extension_for_transformers/transformers/modeling/modeling_auto.py

@@ -67,12 +67,21 @@
     convert_to_quantized_model,
     replace_linear,
 )
+from ...tools.utils import get_gpu_family, is_ipex_available
 from neural_compressor.adaptor.torch_utils.model_wrapper import WeightOnlyLinear
 from transformers.configuration_utils import PretrainedConfig
 from transformers import AutoConfig
 from transformers.utils import is_accelerate_available, is_bitsandbytes_available
 from typing import Union
 
+if is_ipex_available() and get_gpu_family() != "no_gpu":
+    # pylint: disable=E0401
+    from intel_extension_for_pytorch.nn.utils._quantize_convert import (
+        WeightOnlyQuantizedLinear,
+    )
+else:
+    from ..llm.quantization.nn.modules import QuantizedLinearQBits
+
 torch = LazyImport("torch")
 
 
@@ -82,8 +91,6 @@ def recover_export_model(model, current_key_name=None):
 
     Return optimum format model.
     """
-    from ..llm.quantization.nn.modules import QuantizedLinearQBits
-
     for name, module in model.named_children():
         if current_key_name is None:
             current_key_name = []
@@ -165,19 +172,15 @@ def build_woq_model(model, quantization_config):
 
 def convert_model_to_public(model):
     # reorder weight and scales if they have been transposed
-    if model.quantization_config.device == "xpu":
-        # pylint: disable=E0401
-        from intel_extension_for_pytorch.nn.utils._quantize_convert import (
-            WeightOnlyQuantizedLinear,
-        )
-
+    if model.device == "xpu":
         for name, module in model.named_modules():
             if isinstance(module, WeightOnlyQuantizedLinear):
                 if module.weight_transposed:
                     module.qweight.data = module.qweight.t_().contiguous()
                     module.scales.data = module.scales.t_().contiguous()
                     module.weight_transposed = False
-    else:
+    elif model.quantization_config.weight_dtype not in \
+        ["fp8_e5m2", "fp8_e4m3", "nf4", "fp4", "int4_fullrange"]:
         model = recover_export_model(model)
 
 
@@ -195,14 +198,7 @@ def save_low_bit(
         )
         return
 
-    if self.quantization_config.weight_dtype not in [
-        "fp8_e5m2",
-        "fp8_e4m3",
-        "nf4",
-        "fp4",
-        "int4_fullrange",
-    ]:
-        convert_model_to_public(self)
+    convert_model_to_public(self)
     os.makedirs(save_directory, exist_ok=True)
     # use transformers original `save_pretrained` function
     del self.save_pretrained
@@ -391,11 +387,6 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
                     "quantization_config: {}".format(config.quantization_config)
                 )
                 try:
-                    kwargs["device_map"] = (
-                        config.quantization_config["device"]
-                        if "device" in config.quantization_config.keys()
-                        else "auto"
-                    )
                     model = cls.load_low_bit(
                         pretrained_model_name_or_path,
                         *model_args,
@@ -598,7 +589,6 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs):
                     model.config.update({"low_cpu_mem_usage": True})
                 model.eval()
 
-                quantization_config.update(**{"device": "cpu"})
                 if use_xpu:
                     import intel_extension_for_pytorch
 

tests/CI/test_weight_only_gpu.py

@@ -73,7 +73,7 @@ def forward(self, x):
         return self.linear(x)
 
 
-@unittest.skipIf(not _ipex_available or gpu_name == "no_gpu",
+@unittest.skipIf(not is_ipex_available() or gpu_name == "no_gpu",
     "There is no Intel GPU in this machine, skip this test!")
 class TestArcWeightOnly(unittest.TestCase):