From b11d112f23361b4f17a01b5d0604adcbe86553eb Mon Sep 17 00:00:00 2001
From: CSY-ModelCloud <csy@modelcloud.ai>
Date: Wed, 4 Dec 2024 15:49:01 +0800
Subject: [PATCH] add test_asym_gptq_v1.py (#740)

* add test_asym_gptq_v1.py

* add base score
---
 tests/models/model_test.py |  5 ++++-
 tests/test_asym_gptq_v1.py | 19 +++++++++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)
 create mode 100644 tests/test_asym_gptq_v1.py

diff --git a/tests/models/model_test.py b/tests/models/model_test.py
index aa48429a2..66b2734dd 100644
--- a/tests/models/model_test.py
+++ b/tests/models/model_test.py
@@ -35,7 +35,9 @@ class ModelTest(unittest.TestCase):
     DELETE_QUANTIZED_MODEL = True
 
     # quant config
+    QUANT_FORMAT = FORMAT.GPTQ
     DESC_ACT = True
+    SYM = True
 
     def generate(self, model, tokenizer, prompt=None):
         if prompt is None:
@@ -83,8 +85,9 @@ def quantModel(self, model_id_or_path, trust_remote_code=False, torch_dtype="aut
         quantize_config = QuantizeConfig(
             bits=4,
             group_size=128,
-            format=FORMAT.GPTQ,
+            format=self.QUANT_FORMAT,
             desc_act=self.DESC_ACT,
+            sym=self.SYM,
         )
         model = GPTQModel.load(
             model_id_or_path,
diff --git a/tests/test_asym_gptq_v1.py b/tests/test_asym_gptq_v1.py
new file mode 100644
index 000000000..4ae6f5427
--- /dev/null
+++ b/tests/test_asym_gptq_v1.py
@@ -0,0 +1,19 @@
+# -- do not touch
+import os
+
+os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
+# -- end do not touch
+from gptqmodel.quantization import FORMAT  # noqa: E402
+
+from models.model_test import ModelTest  # noqa: E402
+
+
+class Test(ModelTest):
+    NATIVE_MODEL_ID = "/monster/data/model/Llama-3.2-1B-Instruct"  # "meta-llama/Llama-3.2-1B-Instruct"
+    QUANT_FORMAT = FORMAT.GPTQ
+    SYM = False
+    NATIVE_ARC_CHALLENGE_ACC = 0.2747
+    NATIVE_ARC_CHALLENGE_ACC_NORM = 0.2935
+
+    def test(self):
+        self.quant_lm_eval()