|
| 1 | +"""TVM-FFI based builder for CUDA kernels with automatic caching.""" |
| 2 | + |
| 3 | +from __future__ import annotations |
| 4 | + |
| 5 | +import logging |
| 6 | +from pathlib import Path |
| 7 | +from typing import Any, Dict, List, Tuple |
| 8 | + |
| 9 | +import tvm_ffi |
| 10 | + |
| 11 | +from flashinfer_bench.compile.builder import Builder, BuildError, create_pkg_name |
| 12 | +from flashinfer_bench.compile.runnable import Runnable, TVMFFIRunnable |
| 13 | +from flashinfer_bench.data import Definition, Solution, SupportedLanguages |
| 14 | +from flashinfer_bench.env import get_fib_cache_path |
| 15 | + |
| 16 | +logger = logging.getLogger(__name__) |
| 17 | + |
| 18 | +CUDA_EXTENSIONS = [".cu"] |
| 19 | +CPP_EXTENSIONS = [".cpp", ".cc", ".cxx", ".c"] |
| 20 | + |
| 21 | + |
| 22 | +class TVMFFIBuilder(Builder): |
| 23 | + """Builder using TVM-FFI with automatic caching and multi-process sharing. |
| 24 | +
|
| 25 | + Build strategy: |
| 26 | + 1. Check if .so exists in cache (multi-process safe) |
| 27 | + 2. If not, compile with tvm_ffi.cpp.build_inline() to cache |
| 28 | + 3. Load with tvm_ffi.load_module() |
| 29 | +
|
| 30 | + Benefits: |
| 31 | + - Multi-process benchmark: Only first process compiles, others load from cache |
| 32 | + - Cross-framework: Same .so works with PyTorch, JAX, CuPy (DLPack) |
| 33 | + - No JIT/AOT distinction: Smart caching handles both cases |
| 34 | + """ |
| 35 | + |
| 36 | + def __init__(self) -> None: |
| 37 | + super().__init__() |
| 38 | + self._extra_include_paths: Dict[str, str] = {} |
| 39 | + self._extra_ldflags: Dict[str, List[str]] = {} |
| 40 | + |
| 41 | + def can_build(self, sol: Solution) -> bool: |
| 42 | + return sol.spec.language == SupportedLanguages.CUDA |
| 43 | + |
| 44 | + def _make_key(self, solution: Solution) -> str: |
| 45 | + return f"tvm_ffi_{create_pkg_name(solution)}" |
| 46 | + |
| 47 | + def _make_closer(self): |
| 48 | + return lambda: None |
| 49 | + |
| 50 | + def _get_build_path(self, key: str) -> Path: |
| 51 | + return get_fib_cache_path() / "tvm_ffi" / key |
| 52 | + |
| 53 | + def _write_sources(self, path: Path, sol: Solution) -> Tuple[List[str], List[str]]: |
| 54 | + """Extract and write all source files to the given path.""" |
| 55 | + path.mkdir(parents=True, exist_ok=True) |
| 56 | + cpp_files: List[str] = [] |
| 57 | + cuda_files: List[str] = [] |
| 58 | + for src in sol.sources: |
| 59 | + src_path = path / src.path |
| 60 | + if src_path.is_dir(): |
| 61 | + raise BuildError(f"Source path is a directory: {src_path}") |
| 62 | + |
| 63 | + src_path.write_text(src.content) |
| 64 | + |
| 65 | + if str(src_path).endswith(tuple(CPP_EXTENSIONS)): |
| 66 | + cpp_files.append(str(src_path)) |
| 67 | + elif str(src_path).endswith(tuple(CUDA_EXTENSIONS)): |
| 68 | + cuda_files.append(str(src_path)) |
| 69 | + |
| 70 | + if len(cpp_files) == 0 and len(cuda_files) == 0: |
| 71 | + raise BuildError("No sources found") |
| 72 | + return cpp_files, cuda_files |
| 73 | + |
| 74 | + def _get_language(self, cpp_files: List[str], cuda_files: List[str]) -> str: |
| 75 | + return "cuda" if len(cuda_files) > 0 else "cpp" |
| 76 | + |
| 77 | + def _get_entry_symbol(self, sol: Solution) -> str: |
| 78 | + """Extract function symbol from entry_point.""" |
| 79 | + entry_point = sol.spec.entry_point |
| 80 | + if "::" not in entry_point: |
| 81 | + raise BuildError( |
| 82 | + f"Invalid entry_point format: {entry_point}. Expected 'file.cu::symbol'" |
| 83 | + ) |
| 84 | + return entry_point.split("::")[-1] |
| 85 | + |
| 86 | + def _make_runnable( |
| 87 | + self, mod: tvm_ffi.Module, entry_symbol: str, defn: Definition, metadata: Dict[str, Any] |
| 88 | + ) -> Runnable: |
| 89 | + """Create Runnable from TVM-FFI module.""" |
| 90 | + try: |
| 91 | + fn = getattr(mod, entry_symbol) |
| 92 | + except AttributeError as e: |
| 93 | + raise BuildError(f"Entry point '{entry_symbol}' not found in module") from e |
| 94 | + |
| 95 | + # Create keyword adapter to match definition interface |
| 96 | + arg_order = list(defn.inputs.keys()) + list(defn.outputs.keys()) |
| 97 | + |
| 98 | + def _kw_adapter(**kwargs): |
| 99 | + args = [kwargs[name] for name in arg_order] |
| 100 | + return fn(*args) |
| 101 | + |
| 102 | + return TVMFFIRunnable( |
| 103 | + fn=_kw_adapter, closer=self._make_closer(), meta=metadata, definition=defn |
| 104 | + ) |
| 105 | + |
| 106 | + def _build(self, defn: Definition, sol: Solution) -> Runnable: |
| 107 | + """Build with automatic caching - compile once, load from cache afterwards.""" |
| 108 | + key = self._make_key(sol) |
| 109 | + build_path = self._get_build_path(key) |
| 110 | + entry_symbol = self._get_entry_symbol(sol) |
| 111 | + cpp_files, cuda_files = self._write_sources(build_path, sol) |
| 112 | + language = self._get_language(cpp_files, cuda_files) |
| 113 | + extra_include_paths = [str(build_path)] |
| 114 | + |
| 115 | + try: |
| 116 | + # Use build_inline instead of build to |
| 117 | + output_lib_path = tvm_ffi.cpp.build( |
| 118 | + name=key, |
| 119 | + cpp_files=cpp_files, |
| 120 | + cuda_files=cuda_files, |
| 121 | + extra_include_paths=extra_include_paths, |
| 122 | + build_directory=build_path, |
| 123 | + ) |
| 124 | + except Exception as e: |
| 125 | + raise BuildError(f"TVM-FFI compilation failed for '{sol.name}': {e}") from e |
| 126 | + |
| 127 | + # Load the compiled module |
| 128 | + try: |
| 129 | + mod = tvm_ffi.load_module(output_lib_path) |
| 130 | + except Exception as e: |
| 131 | + raise BuildError(f"Failed to load compiled module: {e}") from e |
| 132 | + |
| 133 | + metadata = { |
| 134 | + "definition": defn.name, |
| 135 | + "solution": sol.name, |
| 136 | + "language": language, |
| 137 | + "binding": "tvm_ffi", |
| 138 | + "key": key, |
| 139 | + "symbol": entry_symbol, |
| 140 | + "binary": output_lib_path, |
| 141 | + } |
| 142 | + |
| 143 | + return self._make_runnable(mod, entry_symbol, defn, metadata) |
0 commit comments