Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

update score max negative delta #748

Merged
merged 5 commits into from
Dec 4, 2024
Merged

update score max negative delta #748

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
357aaf0
update deci delta
CSY-ModelCloud Dec 4, 2024
eda8706
update cohere delta
CSY-ModelCloud Dec 4, 2024
f5b26f8
update longllama delta
CSY-ModelCloud Dec 4, 2024
02a36dd
update hymba delta
CSY-ModelCloud Dec 4, 2024
87f9215
rename
CSY-ModelCloud Dec 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions tests/models/model_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
class ModelTest(unittest.TestCase):
TASK_NAME = "arc_challenge"
# sub test can modify
QUANT_ARC_MAX_NEGATIVE_DELTA = 0.15 # -15%
QUANT_ARC_MAX_DELTA_FLOOR_PERCENT = 0.15 # -15%
QUANT_ARC_MAX_POSITIVE_DELTA = 0.2 # 20%
TRUST_REMOTE_CODE = False
APPLY_CHAT_TEMPLATE = False
Expand Down Expand Up @@ -221,7 +221,7 @@ def quant_lm_eval(self):
def check_results(self, task_results):
for filter, value in task_results.items():
diff_pct = self.calculatorPer(filter=filter, value=value)
negative_pct = 100 * (1 - self.QUANT_ARC_MAX_NEGATIVE_DELTA)
negative_pct = 100 * (1 - self.QUANT_ARC_MAX_DELTA_FLOOR_PERCENT)
positive_pct = 100 * (1 + self.QUANT_ARC_MAX_POSITIVE_DELTA)
self.assertTrue(negative_pct <= diff_pct <= positive_pct,
f"{filter}: {value} diff {diff_pct:.2f}% is out of the expected range [{negative_pct}-{positive_pct}%]")
2 changes: 1 addition & 1 deletion tests/models/test_cohere.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ class TestCohere(ModelTest):
NATIVE_MODEL_ID = "/monster/data/model/aya-expanse-8b" # "CohereForAI/aya-expanse-8b"
NATIVE_ARC_CHALLENGE_ACC = 0.5401
NATIVE_ARC_CHALLENGE_ACC_NORM = 0.5640
QUANT_ARC_MAX_NEGATIVE_DELTA = 0.12
QUANT_ARC_MAX_DELTA_FLOOR_PERCENT = 0.15
BATCH_SIZE = 4

def test_cohere(self):
Expand Down
2 changes: 1 addition & 1 deletion tests/models/test_deci.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ class TestDeci(ModelTest):
NATIVE_MODEL_ID = "/monster/data/model/DeciLM-7B-instruct" # "Deci/DeciLM-7B-instruct"
NATIVE_ARC_CHALLENGE_ACC = 0.5239
NATIVE_ARC_CHALLENGE_ACC_NORM = 0.5222
QUANT_ARC_MAX_NEGATIVE_DELTA = 0.55
QUANT_ARC_MAX_DELTA_FLOOR_PERCENT = 0.8
TRUST_REMOTE_CODE = True
USE_VLLM = False
BATCH_SIZE = 6
Expand Down
3 changes: 2 additions & 1 deletion tests/models/test_falcon.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import torch # noqa: E402from tests.model_test import ModelTest

from model_test import ModelTest


Expand All @@ -9,7 +10,7 @@ class TestFalcon(ModelTest):
APPLY_CHAT_TEMPLATE = True
TRUST_REMOTE_CODE = True
TORCH_DTYPE = torch.float16
QUANT_ARC_MAX_NEGATIVE_DELTA = 0.52
QUANT_ARC_MAX_DELTA_FLOOR_PERCENT = 0.52
BATCH_SIZE = 6
USE_VLLM = False

Expand Down
1 change: 1 addition & 0 deletions tests/models/test_hymba.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ class TestHymba(ModelTest):
NATIVE_MODEL_ID = "/monster/data/model/Hymba-1.5B-Instruct/" # "baichuan-inc/Baichuan2-7B-Chat"
NATIVE_ARC_CHALLENGE_ACC = 0.2073
NATIVE_ARC_CHALLENGE_ACC_NORM = 0.2713
QUANT_ARC_MAX_DELTA_FLOOR_PERCENT = 0.75
MODEL_MAX_LEN = 8192
TRUST_REMOTE_CODE = True
APPLY_CHAT_TEMPLATE = True
Expand Down
2 changes: 1 addition & 1 deletion tests/models/test_llama3_2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ class TestLlama3_2(ModelTest):
NATIVE_MODEL_ID = "/monster/data/model/Llama-3.2-1B-Instruct" # "meta-llama/Llama-3.2-1B-Instruct"
NATIVE_ARC_CHALLENGE_ACC = 0.3567
NATIVE_ARC_CHALLENGE_ACC_NORM = 0.3805
QUANT_ARC_MAX_NEGATIVE_DELTA = 0.36
QUANT_ARC_MAX_DELTA_FLOOR_PERCENT = 0.36
APPLY_CHAT_TEMPLATE = True
TRUST_REMOTE_CODE = True

Expand Down
2 changes: 1 addition & 1 deletion tests/models/test_longllama.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ class TestLongLlama(ModelTest):
NATIVE_ARC_CHALLENGE_ACC = 0.3515
NATIVE_ARC_CHALLENGE_ACC_NORM = 0.3652
TRUST_REMOTE_CODE = True
QUANT_ARC_MAX_NEGATIVE_DELTA = 0.4
QUANT_ARC_MAX_DELTA_FLOOR_PERCENT = 0.5
USE_VLLM = False

def test_longllama(self):
Expand Down
2 changes: 1 addition & 1 deletion tests/models/test_qwen2_5.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

class TestQwen2_5(ModelTest):
NATIVE_MODEL_ID = "/monster/data/model/Qwen2.5-0.5B-Instruct"
QUANT_ARC_MAX_NEGATIVE_DELTA = 0.2
QUANT_ARC_MAX_DELTA_FLOOR_PERCENT = 0.2
NATIVE_ARC_CHALLENGE_ACC = 0.2739
NATIVE_ARC_CHALLENGE_ACC_NORM = 0.3055
TRUST_REMOTE_CODE = False
Expand Down
2 changes: 1 addition & 1 deletion tests/test_asym_gptq_v1.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ class Test(ModelTest):
NATIVE_MODEL_ID = "/monster/data/model/Llama-3.2-1B-Instruct" # "meta-llama/Llama-3.2-1B-Instruct"
NATIVE_ARC_CHALLENGE_ACC = 0.3567
NATIVE_ARC_CHALLENGE_ACC_NORM = 0.3805
QUANT_ARC_MAX_NEGATIVE_DELTA = 0.36
QUANT_ARC_MAX_DELTA_FLOOR_PERCENT = 0.36
QUANT_FORMAT = FORMAT.GPTQ
SYM = False

Expand Down
Loading