Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions tpu_commons/platforms/tpu_jax.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,23 @@
from torchax.ops.mappings import j2t_dtype
from tpu_info import device
from vllm.inputs import ProcessorInputs, PromptType
from vllm.platforms.interface import Platform, PlatformEnum, _Backend
from vllm.platforms.interface import Platform, PlatformEnum
from vllm.sampling_params import SamplingParams, SamplingType

from tpu_commons.logger import init_logger
from tpu_commons.models.jax.utils.quantization.quantization_utils import \
update_vllm_config_for_qwix_quantization

if TYPE_CHECKING:
from vllm.attention.backends.registry import _Backend
from vllm.config import BlockSize, ModelConfig, VllmConfig
from vllm.pooling_params import PoolingParams
else:
BlockSize = None
ModelConfig = None
VllmConfig = None
PoolingParams = None
_Backend = None

logger = init_logger(__name__)

Expand Down Expand Up @@ -51,10 +53,11 @@ class TpuPlatform(Platform):
]

@classmethod
def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
def get_attn_backend_cls(cls, selected_backend: "_Backend", head_size: int,
dtype: jnp.dtype, kv_cache_dtype: Optional[str],
block_size: int, use_v1: bool, use_mla: bool,
has_sink: bool, use_sparse: bool) -> str:
from vllm.attention.backends.registry import _Backend
if selected_backend != _Backend.PALLAS:
logger.info("Cannot use %s backend on TPU.", selected_backend)

Expand Down
Loading