Skip to content

Commit 878f58b

Browse files
LRL-ModelCloudLRL-ModelCloudQubitium
authored
add ipex bench code (#660)
* add ipex bench code. * update batch * use cli arg * print all arg values before benchmark starts * add quantized model support * cleanup * Rename prompts.json to prompts.jsonl --------- Co-authored-by: LRL-ModelCloud <lrl@modelcloud.ai> Co-authored-by: Qubitium-ModelCloud <qubitium@modelcloud.ai>
1 parent 29a3c1e commit 878f58b

File tree

2 files changed

+171
-0
lines changed

2 files changed

+171
-0
lines changed

examples/benchmark/ipex.py

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
import time
2+
import torch
3+
import argparse
4+
from datasets import load_dataset
5+
from transformers import AutoTokenizer, AutoConfig, AutoModelForCausalLM
6+
from gptqmodel import GPTQModel, BACKEND
7+
8+
def prepare_dataset_for_bench(tokenizer, batch_size=8):
9+
dataset = load_dataset("json", data_files="prompts.jsonl", split="train")[:batch_size]
10+
prompts = [[{"role": "user", "content": data}] for data in dataset['input']]
11+
input_tensors = tokenizer.apply_chat_template(prompts, add_generation_prompt=True, return_tensors="pt", padding=True)
12+
return input_tensors
13+
14+
parser = argparse.ArgumentParser(description="Benchmark IPEX vs HF on a pre-trained model.")
15+
parser.add_argument("--model", type=str, required=True, help="Path or name of the pre-trained model.")
16+
parser.add_argument("--cores", type=int, default=8, help="Number of CPU cores to use.")
17+
parser.add_argument("--batch", type=int, default=8, help="Batch size for processing messages.")
18+
parser.add_argument("--backend", type=str, choices=["ipex", "hf"], default="ipex", help="Backend to optimize the model. Choose between 'ipex' and 'hf'.")
19+
ars = parser.parse_args()
20+
21+
print("use model: ", ars.model)
22+
print("use cores: ", ars.cores)
23+
print("use batch: ", ars.batch)
24+
print("use backend: ", ars.backend)
25+
26+
# Set the number of threads to use
27+
torch.set_num_threads(ars.cores)
28+
29+
# load model, check model backend
30+
config = AutoConfig.from_pretrained(ars.model)
31+
is_quantized_model = hasattr(config, "quantization_config")
32+
if is_quantized_model:
33+
print("load quantized model, will use BACKEND.IPEX")
34+
model = GPTQModel.load(ars.model, backend=BACKEND.IPEX)
35+
else:
36+
model = AutoModelForCausalLM.from_pretrained(ars.model, device_map="cpu", torch_dtype=torch.bfloat16)
37+
38+
# set model to eval mode
39+
model.eval()
40+
41+
if ars.backend == "ipex" and not is_quantized_model:
42+
import intel_extension_for_pytorch as ipex
43+
model = ipex.llm.optimize(model, dtype=torch.bfloat16)
44+
model.forward = torch.compile(model.forward, dynamic=True, backend="ipex")
45+
46+
# load tokenizer, and normalize the pad_token_id
47+
tokenizer = AutoTokenizer.from_pretrained(ars.model)
48+
tokenizer.pad_token_id = 128004 # <|finetune_right_pad_id|>
49+
model.pad_token_id = tokenizer.pad_token_id
50+
51+
# prepare dataset for benchmark
52+
input_ids = prepare_dataset_for_bench(tokenizer, ars.batch)
53+
54+
# benchmark
55+
with torch.no_grad(), torch.amp.autocast("cpu", dtype=torch.bfloat16):
56+
start = time.time()
57+
new_token_len = 0
58+
outputs = model.generate(input_ids=input_ids.to(model.device), max_new_tokens=512, pad_token_id=tokenizer.pad_token_id)
59+
for i, output in enumerate(outputs):
60+
new_token_len += len(output) - len(input_ids[i])
61+
# debug print
62+
# result = tokenizer.decode(output, skip_special_tokens=False)
63+
# print(result)
64+
# print("="*50)
65+
66+
total_time = time.time() - start
67+
68+
# display benchmark result
69+
print(f"generate use :{total_time}")
70+
print(f"total new token: {new_token_len}")
71+
print(f"token/sec: {(new_token_len/total_time):.4f}")

examples/benchmark/prompts.jsonl

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
{"type":"Childcare","input":"What are the effects of parental involvement in children's education on their academic success?"}
2+
{"type":"WWII Weapons","input":"What were the specifications and operational roles of the Japanese Mitsubishi A6M Zero fighter aircraft?"}
3+
{"type":"Philosophy","input":"Discuss the principles of libertarianism in political philosophy and how they contrast with other political ideologies."}
4+
{"type":"Dinosaur","input":"What role did competition and predation play in the diversification of dinosaur species?"}
5+
{"type":"Home Economics","input":"What are the key principles of interior design?"}
6+
{"type":"Real Estate","input":"How do urban planning policies and infrastructure development projects influence long-term real estate trends in a city?"}
7+
{"type":"Game","input":"Can you explain the process of porting a game from one platform to another, including the technical and business considerations involved?"}
8+
{"type":"Eintsten Physcis","input":"What experimental evidence supports the theory of general relativity?"}
9+
{"type":"Home Economics","input":"How to choose and care for different types of fabrics?"}
10+
{"type":"Dentistry","input":"What precautions should be taken with dental crowns?"}
11+
{"type":"Economics","input":"Discuss how price elasticity affects consumer demand and business pricing strategies."}
12+
{"type":"Entertainment Stars","input":"Explain the journey of Dwayne \"The Rock\" Johnson from wrestling to acting."}
13+
{"type":"Excel","input":"How do you use the OFFSET function in Excel to create dynamic ranges? "}
14+
{"type":"Economics","input":"How do tariffs and quotas influence international trade and domestic industries?"}
15+
{"type":"Famous People","input":"What role did Alan Turing play in the development of computer science?"}
16+
{"type":"HRM","input":"What are the key components of a successful employee recognition program? "}
17+
{"type":"Memory and Counting","input":"Without looking, what is the 10th letter of the English alphabet?"}
18+
{"type":"Eintsten Physcis","input":"Explain the difference between Einstein's special relativity and general relativity."}
19+
{"type":"Philosophy","input":"Compare deontological ethics with consequentialist ethics and discuss their applications in moral decision-making."}
20+
{"type":"Capybara","input":"What differences exist between male and female capybaras in terms of size and behavior?"}
21+
{"type":"Excel","input":"How does one use Excel's advanced filter to extract specific data from a large dataset?"}
22+
{"type":"Famous People","input":"What was the significance of René Descartes' philosophical statement, \"I think, therefore I am\"? "}
23+
{"type":"Dinosaur","input":"How have digital technologies, like 3D modeling and scanning, advanced dinosaur paleontology?"}
24+
{"type":"Eintsten Physcis","input":"How do Einstein's theories of relativity influence our understanding of cosmological phenomena like the Big Bang and the expansion of the universe?"}
25+
{"type":"World","input":"How did Mexico achieve its independence?"}
26+
{"type":"Traffic Law","input":"How are traffic laws enforced through the use of speed cameras and automatic ticketing in the Netherlands?"}
27+
{"type":"World","input":"How has the Aztec civilization influenced Mexican culture?"}
28+
{"type":"Infamous People","input":"Explain the criminal empire of Dawood Ibrahim and his influence on global organized crime."}
29+
{"type":"Real Estate","input":"What are the primary methods of real estate valuation, and when is each method most appropriately used?"}
30+
{"type":"Dinosaur","input":"How did the teeth and jaw structures of different dinosaurs reflect their diets and feeding habits?"}
31+
{"type":"Dentistry","input":"What are the causes of chronic dry mouth, and how can it be managed? "}
32+
{"type":"Musical Instruments","input":"Discuss the role of timpani in orchestras and how they contribute to rhythm and harmony. "}
33+
{"type":"Modern Theoretical Physics","input":"What are emergent phenomena in condensed matter physics, and how do they provide insights into complex quantum systems?"}
34+
{"type":"Entertainment Stars","input":"How do celebrities maintain their public image?"}
35+
{"type":"Game","input":"Describe the role of game designers in the development process."}
36+
{"type":"Common Knowledge","input":"What are the benefits of learning a second language, both personally and professionally?"}
37+
{"type":"Modern Theoretical Physics","input":"How do extra dimensions in theoretical physics models like Kaluza-Klein theory contribute to our understanding of fundamental forces?"}
38+
{"type":"Politics","input":"Explain how elections work in a democratic society."}
39+
{"type":"WWII Weapons","input":"Discuss the design and effectiveness of the German Panzerfaust anti-tank weapon in infantry engagements."}
40+
{"type":"Dentistry","input":"How does poor oral hygiene contribute to systemic diseases like heart disease and diabetes? "}
41+
{"type":"Virology","input":"How do RNA interference mechanisms in host cells act as antiviral defenses, and how might viruses counteract these mechanisms?"}
42+
{"type":"World","input":"What were the purposes behind the construction of the ancient Egyptian pyramids?"}
43+
{"type":"Spanish","input":"Please translate \"Ayer fui al mercado y compré muchas frutas frescas.\" from Spanish to English."}
44+
{"type":"Journalism","input":"How do news organizations handle breaking news situations?"}
45+
{"type":"Geography","input":"What are the primary causes of desertification?"}
46+
{"type":"World","input":"What impact did the Greek debt crisis have on the Eurozone economy?"}
47+
{"type":"Eintsten Physcis","input":"Describe the concept of time dilation and provide an example."}
48+
{"type":"Criminology","input":"What is the role of forensic psychology in criminal investigations?"}
49+
{"type":"Aerospace","input":"Analyze the potential impact of emerging technologies such as artificial intelligence and quantum computing on aerospace engineering and space exploration."}
50+
{"type":"Aerospace","input":"Explain the function of gyroscopes and accelerometers in aircraft navigation systems. "}
51+
{"type":"Geography","input":"Describe the process of coastal erosion."}
52+
{"type":"Game","input":"Discuss the impact of virtual reality on modern gaming."}
53+
{"type":"World","input":"How did the Spanish language become a global language?"}
54+
{"type":"Stock Market","input":"How has the proliferation of commission-free trading apps affected stock market participation and price volatility?"}
55+
{"type":"WWII Weapons","input":"What were the differences in design, performance, and roles between the German Bf 109 and Fw 190 fighter aircraft?"}
56+
{"type":"Real Estate","input":"What are the different real estate brokerage models, and how do they impact agents and clients?"}
57+
{"type":"Journalism","input":"What is the role of public relations in shaping news narratives? "}
58+
{"type":"Geography","input":"Discuss the impacts of climate change on polar ice caps and the resulting effects on global sea levels and weather patterns."}
59+
{"type":"Graphic Design","input":"How does proportion and scale impact the effectiveness of a design? "}
60+
{"type":"Newton Physics","input":"What is impulse, and how does it relate to the change in momentum of an object?"}
61+
{"type":"Aerospace","input":"What are the advantages and limitations of vertical takeoff and landing (VTOL) aircraft? "}
62+
{"type":"Nutrition","input":"Is intermittent fasting effective for fat loss?"}
63+
{"type":"Puzzels","input":"A man was born in 1995 and died in 1953 at the age of 25. How is this possible?"}
64+
{"type":"Consumption","input":"How do environmental concerns shape the growing demand for eco-friendly products among consumers? "}
65+
{"type":"Animal Husbandry","input":"Best practices for milking goats?"}
66+
{"type":"Animal Husbandry","input":"How do you train a working dog for herding?"}
67+
{"type":"Geology","input":"How does weathering contribute to soil formation and landscape development?"}
68+
{"type":"Puzzels","input":"What can run but never walks, has a mouth but never talks, has a head but never weeps, has a bed but never sleeps?"}
69+
{"type":"Journalism","input":"What is the role of a news editor in managing the flow of news within a newsroom? "}
70+
{"type":"Religion","input":"What is the significance of religious festivals in fostering community and faith in different religions? "}
71+
{"type":"Musical Instruments","input":"What is the significance of the viola in string quartets and orchestral settings? "}
72+
{"type":"Geology","input":"What is the structure of Earth's interior?"}
73+
{"type":"Biology","input":"Describe the structure of DNA and its role in heredity."}
74+
{"type":"Eintsten Physcis","input":"What is frame-dragging in general relativity, and how has it been observed experimentally?"}
75+
{"type":"Preschool Education","input":"What are the benefits of using positive reinforcement in managing preschoolers' behavior, and how can it be effectively implemented?"}
76+
{"type":"Geology","input":"How do geologists identify and classify different types of sedimentary structures?"}
77+
{"type":"Traffic Law","input":"What are the regulations for child safety seats in vehicles according to French traffic law?"}
78+
{"type":"Modern Theoretical Physics","input":"What are the leading theories regarding the nature and composition of dark matter, and what experimental approaches are used to detect it?"}
79+
{"type":"Drugs","input":"What are the benefits and potential side effects of clopidogrel?"}
80+
{"type":"Tea Culture","input":"Describe the cultural importance of tea in the Georgian Supra feast. "}
81+
{"type":"Biology","input":"Describe the process of PCR (Polymerase Chain Reaction) and its applications."}
82+
{"type":"Eintsten Physcis","input":"How does general relativity account for the perihelion advance of planetary orbits beyond Mercury?"}
83+
{"type":"Real Estate","input":"How do homeowners associations (HOAs) operate, and what are their typical responsibilities?"}
84+
{"type":"Psychology","input":"What are the main symptoms of depression?"}
85+
{"type":"Religion","input":"How do creation myths vary across religions?"}
86+
{"type":"Psychology","input":"What are the long-term psychological effects of bullying on victims? "}
87+
{"type":"Graphic Design","input":"How do texture and patterns contribute to the overall feel of a graphic design? "}
88+
{"type":"Criminology","input":"Define recidivism in criminology."}
89+
{"type":"World","input":"How does agriculture affect Argentina's economy?"}
90+
{"type":"Art and Culture","input":"Name a traditional dance from Greece."}
91+
{"type":"Biology","input":"Define biodiversity."}
92+
{"type":"Middle East Situation","input":"How do historical grievances and colonial legacies continue to shape current conflicts in the Middle East?"}
93+
{"type":"Common Knowledge","input":"What are some effective time management techniques that can improve productivity in daily life?"}
94+
{"type":"World","input":"How was German reunification achieved, and what were the main challenges faced?"}
95+
{"type":"Common Knowledge","input":"Describe the process of booking a flight online, including how to compare prices and select seats."}
96+
{"type":"Excel","input":"How can macros be used to automate repetitive tasks in Excel?"}
97+
{"type":"Traffic Law","input":"Describe the process for appealing a traffic ticket in the United Kingdom."}
98+
{"type":"World History","input":"Analyze the impact of the Enlightenment on the American Revolution."}
99+
{"type":"Eintsten Physcis","input":"What are the theoretical implications of merging black holes for our understanding of spacetime?"}
100+
{"type":"Stock Market","input":"What is market liquidity, and why is it important for the efficient functioning of stock markets?"}

0 commit comments

Comments
 (0)