Skip to content

Commit

Permalink
Fix the server CLI issue with use_flashinfer flag (#42)
Browse files Browse the repository at this point in the history
* fix refactor

* empty

* fix lint
  • Loading branch information
alfredgui2 authored and tjluyao committed Jul 8, 2024
1 parent 44ad40e commit 2c57eb0
Show file tree
Hide file tree
Showing 3 changed files with 6 additions and 2 deletions.
2 changes: 1 addition & 1 deletion server/text_generation_server/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def serve(
logger_level: str = "INFO",
json_output: bool = False,
otlp_endpoint: Optional[str] = None,
use_flashinfer: bool = True,
use_flashinfer: Optional[bool] = True,
):
if sharded:
assert (
Expand Down
5 changes: 5 additions & 0 deletions server/text_generation_server/models_flashinfer/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,11 @@ class ModelType(enum.Enum):
}


__GLOBALS = locals()
for data in ModelType:
__GLOBALS[data.name] = data.value["type"]


def get_model(
model_id: str,
revision: Optional[str],
Expand Down
1 change: 0 additions & 1 deletion server/text_generation_server/server_flashinfer.py
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,6 @@ async def serve_inner(
revision,
sharded,
quantize,
speculate,
dtype,
trust_remote_code,
lora_ids,
Expand Down

0 comments on commit 2c57eb0

Please sign in to comment.