add vllm awq quantization

dongxiaolong · Sep 22, 2023 · c147858 · c147858
1 parent a040cdc
commit c147858
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/fastchat/serve/vllm_worker.py b/fastchat/serve/vllm_worker.py
@@ -210,6 +210,8 @@ async def api_model_details(request: Request):
         args.model = args.model_path
     if args.num_gpus > 1:
         args.tensor_parallel_size = args.num_gpus
+    if args.quantizaiton:
+        args.quantization = args.quantization
 
     engine_args = AsyncEngineArgs.from_cli_args(args)
     engine = AsyncLLMEngine.from_engine_args(engine_args)