diff --git a/experimental/README.md b/experimental/README.md new file mode 100644 index 0000000000..c524173ba4 --- /dev/null +++ b/experimental/README.md @@ -0,0 +1,3 @@ +# Experimental Features + +This folder aims to highlights features that are still a work-in-progress or potentially supported in LLM Compressor and / or Compressed-Tensors but missing support in vLLM. diff --git a/experimental/mxfp4/llama3_mxfp4.py b/experimental/mxfp4/llama3_mxfp4.py new file mode 100644 index 0000000000..9da30d00a4 --- /dev/null +++ b/experimental/mxfp4/llama3_mxfp4.py @@ -0,0 +1,35 @@ +from transformers import AutoModelForCausalLM, AutoTokenizer + +from llmcompressor import oneshot +from llmcompressor.modifiers.quantization import QuantizationModifier +from llmcompressor.utils import dispatch_for_generation + +MODEL_ID = "meta-llama/Meta-Llama-3-8B-Instruct" + +# Load model. +model = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype="auto") +tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) + +# Configure the quantization algorithm and scheme. +# In this case, we: +# * quantize the weights to fp4 with per group 32 via ptq +recipe = QuantizationModifier(targets="Linear", scheme="MXFP4", ignore=["lm_head"]) + +# Apply quantization. +oneshot(model=model, recipe=recipe) + +print("\n\n") +print("========== SAMPLE GENERATION ==============") +dispatch_for_generation(model) +input_ids = tokenizer("Hello my name is", return_tensors="pt").input_ids.to( + model.device +) +output = model.generate(input_ids, max_new_tokens=100) +print(tokenizer.decode(output[0])) +print("==========================================\n\n") + + +# Save to disk in compressed-tensors format. +SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-MXFP4" +model.save_pretrained(SAVE_DIR, save_compressed=True) +tokenizer.save_pretrained(SAVE_DIR)