[Build][CI] Build and test SDist in release CI (tile-ai#1098)

Author: XuehaiPan

.github/workflows/ci.yml

@@ -22,6 +22,7 @@ env:
   PYTHONDEVMODE: "1"
   PYTHONUNBUFFERED: "1"
   PYTHONPATH: "" # explicit cleanup
+  COLUMNS: "100"
   FORCE_COLOR: "1"
   CLICOLOR_FORCE: "1"
   UV_INDEX_STRATEGY: "unsafe-best-match"

.github/workflows/dist.yml

@@ -28,7 +28,74 @@ concurrency:
   group: "${{ github.workflow }}-${{ github.ref }}"
   cancel-in-progress: true
 
+env:
+  PYTHONDEVMODE: "1"
+  PYTHONUNBUFFERED: "1"
+  COLUMNS: "100"
+  FORCE_COLOR: "1"
+  CLICOLOR_FORCE: "1"
+
 jobs:
+  build-sdist:
+    name: Build SDist
+    if: |
+      github.repository_owner == 'tile-ai' &&
+      (github.event_name != 'pull_request' || !github.event.pull_request.draft)
+    runs-on: macos-latest
+    timeout-minutes: 30
+    env:
+      NO_VERSION_LABEL: ${{ github.event_name == 'release' && 'OFF' || 'ON' }}
+      # NO_GIT_VERSION disables embedding the git commit hash in version metadata.
+      # Otherwise, the version of the SDist has a git hash suffix (e.g., 0.1.0+gitabcdef12),
+      # but the package built from the SDist has no way to get the git hash (it is not a git repo),
+      # leading to inconsistent versions between SDist and built packages (+gitabcdef12 vs. +gitunknown).
+      NO_GIT_VERSION: "ON"
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+        with:
+          fetch-depth: 1
+          submodules: recursive
+
+      - name: Setup Python and uv with caching
+        id: setup-uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          python-version: "3.12"
+          activate-environment: true
+
+      - name: Build SDist
+        run: |
+          uv run --no-project --with=build -m -- build --sdist --outdir=dist
+
+      - name: Setup ccache
+        uses: hendrikmuhs/ccache-action@v1
+        with:
+          create-symlink: true
+          key: ccache-${{ runner.os }}-${{ runner.arch }}
+          evict-old-files: "7d"
+
+      - name: Test SDist buildable
+        run: |
+          TEMP_DIR="$(mktemp -d -t tilelang-sdist-test)"
+          cp -r dist "${TEMP_DIR}/dist"
+          uv venv --seed "${TEMP_DIR}/venv"
+          source "${TEMP_DIR}/venv/bin/activate"
+          cd "${TEMP_DIR}"
+          python3 -m pip install --upgrade pip setuptools wheel
+          python3 -m pip install -v dist/*.tar.gz
+          python3 -c "import tilelang; print(tilelang.__version__)"
+
+      - name: Upload SDist
+        # Not PR to save artifact storage, as SDist is only needed for releases.
+        if: github.event_name != 'pull_request'
+        uses: actions/upload-artifact@v4
+        with:
+          name: sdist
+          path: dist/*.tar.gz
+          if-no-files-found: error
+
   build-wheels:
     name: Build wheels for Python ${{ matrix.python-version }} on ${{ matrix.target.runner }} with ${{ matrix.target.toolkit }}
     if: |
@@ -102,12 +169,20 @@ jobs:
 
   list-artifacts:
     name: List artifacts
-    # Not PR to save artifact storage, as wheels are only needed for releases.
+    # Not PR to save artifact storage, as artifacts are only needed for releases.
     if: github.event_name != 'pull_request'
     runs-on: ubuntu-latest
-    needs: [build-wheels]
+    needs: [build-sdist, build-wheels]
     timeout-minutes: 15
     steps:
+      - name: Download built SDist
+        uses: actions/download-artifact@v5
+        with:
+          # unpacks default artifact into dist/
+          # if `name: artifact` is omitted, the action will create extra parent dir
+          name: sdist
+          path: dist
+
       - name: Download built wheels
         uses: actions/download-artifact@v6
         with:

MANIFEST.in

@@ -1,10 +1,35 @@
+# Reference: https://setuptools.pypa.io/en/latest/userguide/miscellaneous.html
+
+# Include licenses
 include VERSION
-include CMakeLists.txt
-include requirements.txt
-include requirements-test.txt
-include requirements-dev.txt
+include LICENSE
+include THIRDPARTYNOTICES.txt
+
+# Version and dependency files
+include version_provider.py
+include requirements*.txt
 include tilelang/jit/adapter/cython/cython_wrapper.pyx
-recursive-include src *
-recursive-include 3rdparty *
-recursive-exclude 3rdparty/clang* * 
-recursive-exclude 3rdparty/llvm* *
+
+# Include source files in SDist
+include CMakeLists.txt
+graft src
+graft cmake
+graft 3rdparty
+
+# Include test suites in SDist
+graft testing
+graft examples
+global-exclude .coverage .coverage.* coverage.xml coverage-*.xml coverage.*.xml
+global-exclude .junit .junit.* junit.xml junit-*.xml junit.*.xml
+
+# Exclude unneeded files and directories
+prune .git
+prune .github
+prune */.git
+prune */.github
+prune 3rdparty/clang*
+prune 3rdparty/llvm*
+
+# Prune compiled files
+prune */__pycache__
+global-exclude *~ *.py[cod] *.so *.a *.dylib *.pxd *.dll *.lib *.o *.obj

pyproject.toml

@@ -3,8 +3,8 @@ name = "tilelang"
 description = "A tile level programming language to generate high performance code."
 readme = "README.md"
 requires-python = ">=3.8"
-authors = [{name = "TileLang Contributors"}, {name = "Tile-AI"}]
-maintainers = [{name = "Lei Wang", email = "leiwang1999@outlook.com"}]
+authors = [{ name = "TileLang Contributors" }, { name = "Tile-AI" }]
+maintainers = [{ name = "Lei Wang", email = "leiwang1999@outlook.com" }]
 license = "MIT"
 keywords = ["BLAS", "CUDA", "HIP", "Code Generation", "TVM"]
 classifiers = [
@@ -58,16 +58,39 @@ metadata.version.provider = "version_provider"
 metadata.version.provider-path = "."
 experimental = true
 
+[tool.scikit-build.sdist]
+# See MANIFEST.in for details
+include = [
+    "VERSION",
+    "LICENSE",
+    "THIRDPARTYNOTICES.txt",
+    "version_provider.py",
+    "requirements*.txt",
+    "tilelang/jit/adapter/cython/cython_wrapper.pyx",
+    "CMakeLists.txt",
+    "src/**",
+    "cmake/**",
+    "3rdparty/**",
+    "testing/**",
+    "examples/**",
+]
+exclude = [
+    ".git",
+    ".github",
+    "**/.git",
+    "**/.github",
+    "3rdparty/clang**",
+    "3rdparty/llvm**",
+    "build",
+]
+
 [tool.scikit-build.wheel.packages]
 tilelang = "tilelang"
 "tilelang/src" = "src"
+# NOTE: The mapping below places the contents of '3rdparty' inside 'tilelang/3rdparty' in the wheel.
+# This is necessary to find TVM shared libraries at runtime.
 "tilelang/3rdparty" = "3rdparty"
 
-# TODO: we might want to not include these in wheel?
-"tilelang/benchmark" = "benchmark"
-"tilelang/examples" = "examples"
-"tilelang/testing" = "testing"
-
 [tool.yapf]
 based_on_style = "yapf"
 column_limit = 100
@@ -142,18 +165,27 @@ filterwarnings = ["always"]
 
 [tool.cibuildwheel]
 archs = ["auto64"]
+skip = "*musllinux*"
+build-frontend = "build"
+environment = { PYTHONDEVMODE = "1", PYTHONUNBUFFERED = "1" }
+environment-pass = [
+    "CUDA_VERSION",
+    "COLUMNS",
+    "FORCE_COLOR",
+    "CLICOLOR_FORCE",
+]
+before-build = "env -0 | sort -z | tr '\\0' '\\n'"
+windows.before-build = "set"
 # Pin to glibc 2.17 for x86 and 2.28 for aarch64 for now
 manylinux-x86_64-image = "manylinux2014"
 manylinux-aarch64-image = "manylinux_2_28"
-skip = "*musllinux*"
-environment-pass = ["CUDA_VERSION"]
 
 [tool.cibuildwheel.linux]
+environment = { PYTHONDEVMODE = "1", PYTHONUNBUFFERED = "1", PATH = "/usr/local/cuda/bin:$PATH" }
 repair-wheel-command = [
     "auditwheel repair --exclude libcuda.so.1 --exclude '/usr/local/cuda*' -w {dest_dir} {wheel}",
     "pipx run abi3audit --strict --report {wheel}",
 ]
-environment.PATH = "/usr/local/cuda/bin:$PATH"
 # Install CUDA runtime and stub driver library
 # manylinux_2_28 uses gcc 14, which needs CUDA 12.8
 before-all = """