lora/qlora training w/wo unsloth on NVIDIA L4 (~20GB VRAM usage)

configs/config.yaml

@@ -0,0 +1,25 @@
+dataset_id: "ariG23498/license-detection-paligemma"
+model_id: "google/gemma-3-4b-pt" # "unsloth/gemma-3-4b-it" 
+checkpoint_id: "ajaymin28/Gemma3_ObjeDet"
+
+device: "cuda"
+dtype: "bfloat16"
+
+batch_size: 16
+learning_rate: 2e-5
+epochs: 1
+max_step_to_train: 100
+validate_steps_freq: 10
+
+finetune_method: "qlora"   # FFT | lora | qlora
+use_unsloth: false
+
+
+mm_tunable_parts:
+  - no_exist_layer   # basically not finetuning any base components
+  # - mlp
+  # - multi_modal_projector
+  # - vision_tower
+  # - language_model
+wandb_project_name:  "Gemma3_LoRA"
+push_model_to_hub: true

configs/lora_config.yaml

@@ -0,0 +1,17 @@
+r: 32
+alpha: 32
+dropout: 0.05
+target_modules:
+  - q_proj
+  - k_proj
+  - v_proj
+  - o_proj
+  - up_proj
+  - down_proj
+  - gate_proj
+max_seq_length: 2048  # Unsloth will RoPE-scale
+
+# LoRA-specific: no quantization
+load_in_4bit: false
+load_in_8bit: false
+quantization_config: null

configs/qlora_config.yaml

@@ -0,0 +1,20 @@
+r: 32
+alpha: 32
+dropout: 0.05
+target_modules:
+  - q_proj
+  - k_proj
+  - v_proj
+  - o_proj
+  - up_proj
+  - down_proj
+  - gate_proj
+max_seq_length: 2048  # Unsloth will RoPE-scale
+
+# QLoRA-specific: quantization enabled
+load_in_4bit: true
+load_in_8bit: false
+quantization_config:
+  bnb_4bit_use_double_quant: true
+  bnb_4bit_quant_type: "nf4"
+  bnb_4bit_compute_dtype: "bfloat16"

predict.py

@@ -5,8 +5,8 @@
 from torch.utils.data import DataLoader
 from transformers import AutoProcessor, Gemma3ForConditionalGeneration
 
-from config import Configuration
-from utils import test_collate_function, visualize_bounding_boxes
+from utils.config import Configuration
+from utils.utilities import test_collate_function, visualize_bounding_boxes
 
 os.makedirs("outputs", exist_ok=True)
 
@@ -23,7 +23,7 @@ def get_dataloader(processor):
 
 
 if __name__ == "__main__":
-    cfg = Configuration()
+    cfg = Configuration.from_args()
     processor = AutoProcessor.from_pretrained(cfg.checkpoint_id)
     model = Gemma3ForConditionalGeneration.from_pretrained(
         cfg.checkpoint_id,

requirements.txt

@@ -9,3 +9,6 @@ wandb
 peft
 albumentations
 argparse
+omegaconf
+unsloth==2025.5.7 
+unsloth-zoo==2025.5.8