Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Added more llms to Regression test (#432)
Browse files Browse the repository at this point in the history
- Added more llms to Regression test and complete test run is here:

https://github.com/CentML/hidet/actions/runs/10528073493/job/29172874637

- ~~Accuracy check for masked llms with token ids instead of raw logits
tensor~~

---------

Co-authored-by: Zhumakhan <nazirzhumakhan@gmail,.com>
2 people authored and vadiklyutiy committed Dec 19, 2024
1 parent e251184 commit 9308c9f
Showing 2 changed files with 171 additions and 1 deletion.
2 changes: 1 addition & 1 deletion python/hidet/testing/torch_utils.py
Original file line number Diff line number Diff line change
@@ -158,7 +158,7 @@ def one_iter(inputs):
# print(text_output)
return (i + 1) * bs, inputs

torch._dynamo.mark_dynamic(inputs, 0) # pylint: disable=protected-access
# torch._dynamo.mark_dynamic(inputs, 0) # pylint: disable=protected-access
for _ in range(warmup_iters):
num_tokens, output_text = one_iter(inputs)
torch.cuda.empty_cache()
170 changes: 170 additions & 0 deletions tests/benchmarks/run_configs_full.json
Original file line number Diff line number Diff line change
@@ -9,6 +9,176 @@
"dtype_id": 1,
"dtype_name": "float16"
},
{
"type": "model",
"id": 16,
"name": "llama-2-7b",
"runfile": "bench_transformer.py",
"param_id": 15,
"param_name": "bs=16,genlen=1",
"dtype_id": 1,
"dtype_name": "float16"
},
{
"type": "model",
"id": 18,
"name": "mistral",
"runfile": "bench_transformer.py",
"param_id": 15,
"param_name": "bs=16,genlen=1",
"dtype_id": 1,
"dtype_name": "float16"
},
{
"type": "model",
"id": 21,
"name": "codellama-7b",
"runfile": "bench_transformer.py",
"param_id": 15,
"param_name": "bs=16,genlen=1",
"dtype_id": 1,
"dtype_name": "float16"
},
{
"type": "model",
"id": 41,
"name": "layoutlm",
"runfile": "bench_transformer.py",
"param_id": 14,
"param_name": "bs=16,seqlen=256",
"dtype_id": 1,
"dtype_name": "float16"
},
{
"type": "model",
"id": 42,
"name": "mobilebert-uncased",
"runfile": "bench_transformer.py",
"param_id": 14,
"param_name": "bs=16,seqlen=256",
"dtype_id": 1,
"dtype_name": "float16"
},
{
"type": "model",
"id": 43,
"name": "plt-bast",
"runfile": "bench_transformer.py",
"param_id": 15,
"param_name": "bs=16,genlen=1",
"dtype_id": 1,
"dtype_name": "float16"
},
{
"type": "model",
"id": 45,
"name": "conv-bert",
"runfile": "bench_transformer.py",
"param_id": 14,
"param_name": "bs=16,seqlen=256",
"dtype_id": 1,
"dtype_name": "float16"
},
{
"type": "model",
"id": 20,
"name": "mpt-7b",
"runfile": "bench_transformer.py",
"param_id": 14,
"param_name": "bs=16,genlen=1",
"dtype_id": 1,
"dtype_name": "float16"
},
{
"type": "model",
"id": 17,
"name": "gemma-2b",
"runfile": "bench_transformer.py",
"param_id": 14,
"param_name": "bs=16,genlen=1",
"dtype_id": 1,
"dtype_name": "float16"
},
{
"type": "model",
"id": 22,
"name": "mixtral-7b-8",
"runfile": "bench_transformer.py",
"param_id": 14,
"param_name": "bs=16,genlen=1",
"dtype_id": 1,
"dtype_name": "float16"
},
{
"type": "model",
"id": 35,
"name": "albert-base-v2",
"runfile": "bench_transformer.py",
"param_id": 14,
"param_name": "bs=16,seqlen=256",
"dtype_id": 1,
"dtype_name": "float16"
},
{
"type": "model",
"id": 36,
"name": "camembert",
"runfile": "bench_transformer.py",
"param_id": 14,
"param_name": "bs=16,genlen=1",
"dtype_id": 1,
"dtype_name": "float16"
},
{
"type": "model",
"id": 47,
"name": "distillgpt2",
"runfile": "bench_transformer.py",
"param_id": 14,
"param_name": "bs=16,genlen=1",
"dtype_id": 1,
"dtype_name": "float16"
},
{
"type": "model",
"id": 40,
"name": "gpt-neo",
"runfile": "bench_transformer.py",
"param_id": 14,
"param_name": "bs=16,genlen=1",
"dtype_id": 1,
"dtype_name": "float16"
},
{
"type": "model",
"id": 48,
"name": "opt",
"runfile": "bench_transformer.py",
"param_id": 14,
"param_name": "bs=16,genlen=1",
"dtype_id": 1,
"dtype_name": "float16"
},
{
"type": "model",
"id": 44,
"name": "roberta",
"runfile": "bench_transformer.py",
"param_id": 14,
"param_name": "bs=16,genlen=1",
"dtype_id": 1,
"dtype_name": "float16"
},
{
"type": "model",
"id": 50,
"name": "xglm",
"runfile": "bench_transformer.py",
"param_id": 14,
"param_name": "bs=16,genlen=1",
"dtype_id": 1,
"dtype_name": "float16"
},
{
"type": "model",
"id": 4,

0 comments on commit 9308c9f

Please sign in to comment.