[Windows] Re-enable tokenizers

.ci/scripts/setup-windows.ps1

@@ -13,6 +13,14 @@ conda activate et
 # Install test dependencies
 pip install -r .ci/docker/requirements-ci.txt
 
+# Create a symlink to work around path length issues when building submodules (tokenizers, mainly).
+Push-Location
+$repoBase = $pwd.Path
+mkdir C:\_et
+cd C:\_et
+New-Item -ItemType SymbolicLink -Path "executorch" -Target "$repoBase"
+cd executorch
+
 if ($editable -eq 'true') {
     install_executorch.bat --editable
 } else {
@@ -22,3 +30,5 @@ if ($LASTEXITCODE -ne 0) {
     Write-Host "Installation was unsuccessful. Exit code: $LASTEXITCODE."
     exit $LASTEXITCODE
 }
+
+Pop-Location

.ci/scripts/wheel/pre_build_script.sh

@@ -9,18 +9,6 @@ set -euxo pipefail
 
 # This script is run before building ExecuTorch binaries
 
-if [[ "$(uname -m)" == "aarch64" ]]; then
-  # On some Linux aarch64 systems, the "atomic" library is not found during linking.
-  # To work around this, replace "atomic" with the literal ${ATOMIC_LIB} so the
-  # build system uses the full path to the atomic library.
-  file="extension/llm/tokenizers/third-party/sentencepiece/src/CMakeLists.txt"
-  sed 's/list(APPEND SPM_LIBS "atomic")/list(APPEND SPM_LIBS ${ATOMIC_LIB})/' \
-    "$file" > "${file}.tmp" && mv "${file}.tmp" "$file"
-
-  grep -n 'list(APPEND SPM_LIBS ${ATOMIC_LIB})' "$file" && \
-    echo "the file $file has been modified for atomic to use full path"
-fi
-
 # Clone nested submodules for tokenizers - this is a workaround for recursive
 # submodule clone failing due to path length limitations on Windows. Eventually,
 # we should update the core job in test-infra to enable long paths before
@@ -32,7 +20,9 @@ popd
 # On Windows, enable symlinks and re-checkout the current revision to create
 # the symlinked src/ directory. This is needed to build the wheel.
 UNAME_S=$(uname -s)
+IS_WINDOWS=0
 if [[ $UNAME_S == *"MINGW"* || $UNAME_S == *"MSYS"* ]]; then
+    IS_WINDOWS=1
     echo "Enabling symlinks on Windows"
     git config core.symlinks true
     git checkout -f HEAD
@@ -43,4 +33,11 @@ fi
 # which does install them. Though we'd need to disable build isolation to be
 # able to see the installed torch package.
 
-"${GITHUB_WORKSPACE}/${REPOSITORY}/install_requirements.sh" --example
+if [[ $IS_WINDOWS -eq 1 ]]; then
+    # Use a symlink'd path to work around path length limitations when building
+    # dependencies from source.
+    cmd.exe -c "${GITHUB_WORKSPACE}/${REPOSITORY}/.ci/scripts/wheel/setup_symlink.bat"
+    cd /c/_et/executorch/
+else
+    "${GITHUB_WORKSPACE}/${REPOSITORY}/install_requirements.sh" --example
+fi

.ci/scripts/wheel/setup_symlink.bat

@@ -0,0 +1,19 @@
+set work_dir=%CD%
+echo "Evaluting symlink for %work_dir%"
+if exist setup.py (
+    REM Relocate the repo to a shorter path. Setup a symlink to preserve the original usage.
+    REM Note that the ET directory has to be named "executorch".
+    if not exist C:\_et\executorch\ (
+        echo "Relocating executorch repo..."
+        cd C:\
+        if not exist _et\ (
+            mkdir _et
+        )
+        cd _et
+        move $work_dir% .
+        cd executorch
+        mklink /d %work_dir% C:\_et\executorch\
+    ) else (
+        cd C:\_et\executorch\
+    )
+)

.ci/scripts/wheel/vc_env_helper.bat

@@ -38,24 +38,10 @@ goto start
 :done
 if "%args%" == "" (
     echo Usage: vc_env_helper.bat [command] [args]
-    echo e.g. vc_env_helper.bat cl /c test.cpp
+    echo e.g. vc_env_helper.bat cl /c test.
 )
 
-set work_dir=%CD%
-if exist setup.py (
-    echo "Creating symlink..."
-    REM Setup a symlink to shorten the path length.
-    REM Note that the ET directory has to be named "executorch".
-    cd %GITHUB_WORKSPACE%
-    if not exist et\ (
-        mkdir et
-    )
-    cd et
-    echo Work dir: %work_dir%
-    if not exist executorch\ (
-        mklink /d executorch %work_dir%
-    )
-    cd executorch
-)
+call "%~dp0setup_symlink.bat"
+
 
 %args% || exit /b 1

install_requirements.py

@@ -112,13 +112,8 @@ def install_requirements(use_pytorch_nightly):
 
     LOCAL_REQUIREMENTS = [
         "third-party/ao",  # We need the latest kernels for fast iteration, so not relying on pypi.
-    ] + (
-        [
-            "extension/llm/tokenizers",  # TODO(larryliu0820): Setup a pypi package for this.
-        ]
-        if sys.platform != "win32"
-        else []
-    )  # TODO(gjcomer): Re-enable when buildable on Windows.
+        "extension/llm/tokenizers",  # TODO(larryliu0820): Setup a pypi package for this.
+    ]
 
     # Install packages directly from local copy instead of pypi.
     # This is usually not recommended.