Run Python unit test CI on Windows

ghstack-source-id: dcb321c
ghstack-comment-id: 3226055310
Pull-Request: #13716

Author: GregoryComer

.ci/scripts/unittest-windows.ps1

@@ -0,0 +1,29 @@
+param (
+    [bool]$editable
+)
+
+Set-PSDebug -Trace 1
+$ErrorActionPreference = 'Stop'
+$PSNativeCommandUseErrorActionPreference = $true
+
+conda create --yes --quiet -n et python=3.12
+conda activate et
+
+# Activate the VS environment - this is required for Dynamo to work, as it uses MSVC.
+# There are a bunch of environment variables that it requires.
+# See https://learn.microsoft.com/en-us/cpp/build/building-on-the-command-line.
+& "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\Tools\Launch-VsDevShell.ps1" -Arch amd64
+
+install_executorch.bat --editable:$editable
+if ($LASTEXITCODE -ne 0) {
+    Write-Host "Installation was unsuccessful. Exit code: $LASTEXITCODE."
+    exit $LASTEXITCODE
+}
+
+# Run pytest with coverage
+# pytest -n auto --cov=./ --cov-report=xml
+pytest -n auto --continue-on-collection-errors
+if ($LASTEXITCODE -ne 0) {
+    Write-Host "Pytest invocation was unsuccessful. Exit code: $LASTEXITCODE."
+    exit $LASTEXITCODE
+}

.github/workflows/_unittest.yml

@@ -52,3 +52,13 @@ jobs:
         # This is needed to get the prebuilt PyTorch wheel from S3
         ${CONDA_RUN} --no-capture-output pip install awscli==1.37.21
         .ci/scripts/unittest-macos.sh --build-tool "${{ inputs.build-tool }}" --build-mode "${{ inputs.build-mode }}" --editable "${{ inputs.editable }}"
+
+  windows:
+    uses: pytorch/test-infra/.github/workflows/windows_job.yml@main
+    with:
+      if: ${{ inputs.build-tool == 'cmake' }}
+      submodules: 'recursive'
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      script: |
+        conda init powershell
+        powershell .ci/scripts/unittest-windows.ps1 -editable "${{ inputs.editable }}"

.gitignore

@@ -65,3 +65,7 @@ xcuserdata/
 
 # Android
 *.aar
+
+# Windows
+*.dll
+*.pyd

backends/xnnpack/CMakeLists.txt

@@ -62,7 +62,7 @@ endforeach()
 if(WIN32 AND NOT CMAKE_CROSSCOMPILING)
   set(MV_COMMAND
       powershell -Command
-      "Move-Item -Path ${_xnnpack_flatbuffer__outputs} -Destination ${_xnnpack_schema__outputs}"
+      "Move-Item -Path ${_xnnpack_flatbuffer__outputs} -Destination ${_xnnpack_schema__outputs} -Force"
   )
 else()
   set(MV_COMMAND mv ${_xnnpack_flatbuffer__outputs} ${_xnnpack_schema__outputs})

examples/models/llama/source_transformation/custom_kv_cache.py

@@ -269,7 +269,7 @@ def replace_kv_cache_with_quantized_kv_cache(module):
         executorch_package_path = executorch.__path__[-1]
         libs = list(
             glob.glob(
-                f"{executorch_package_path}/**/libquantized_ops_aot_lib.*",
+                f"{executorch_package_path}/**/*quantized_ops_aot_lib.*",
                 recursive=True,
             )
         )

extension/llm/custom_ops/custom_ops.py

@@ -33,7 +33,7 @@
     package_path = Path(__file__).parent.resolve()
     logging.info(f"Looking for libcustom_ops_aot_lib.so in {package_path}")
 
-    libs = list(package_path.glob("**/libcustom_ops_aot_lib.*"))
+    libs = list(package_path.glob("**/*custom_ops_aot_lib.*"))
 
     assert len(libs) == 1, f"Expected 1 library but got {len(libs)}"
     logging.info(f"Loading custom ops library: {libs[0]}")

extension/llm/custom_ops/op_tile_crop_aot.py

@@ -13,7 +13,7 @@
     tile_crop = torch.ops.preprocess.tile_crop.default
     assert tile_crop is not None
 except:
-    libs = list(Path(__file__).parent.resolve().glob("libcustom_ops_aot_lib.*"))
+    libs = list(Path(__file__).parent.resolve().glob("*custom_ops_aot_lib.*"))
     assert len(libs) == 1, f"Expected 1 library but got {len(libs)}"
     logging.info(f"Loading custom ops library: {libs[0]}")
     torch.ops.load_library(libs[0])

install_executorch.py

@@ -194,14 +194,6 @@ def main(args):
         clean()
         return
 
-    cmake_args = [os.getenv("CMAKE_ARGS", "")]
-    # Use ClangCL on Windows.
-    # ClangCL is an alias to Clang that configures it to work in an MSVC-compatible
-    # mode. Using it on Windows to avoid compiler compatibility issues for MSVC.
-    if os.name == "nt":
-        cmake_args.append("-T ClangCL")
-    os.environ["CMAKE_ARGS"] = " ".join(cmake_args)
-
     check_and_update_submodules()
     # This option is used in CI to make sure that PyTorch build from the pinned commit
     # is used instead of nightly. CI jobs wouldn't be able to catch regression from the

kernels/quantized/__init__.py

@@ -7,7 +7,7 @@
 try:
     from pathlib import Path
 
-    libs = list(Path(__file__).parent.resolve().glob("**/libquantized_ops_aot_lib.*"))
+    libs = list(Path(__file__).parent.resolve().glob("**/*quantized_ops_aot_lib.*"))
     del Path
     assert len(libs) == 1, f"Expected 1 library but got {len(libs)}"
     import torch as _torch

setup.py

@@ -672,6 +672,10 @@ def run(self):  # noqa C901
             f"-DCMAKE_BUILD_TYPE={cmake_build_type}",
         ]
 
+        # Use ClangCL on Windows.
+        if _is_windows():
+            cmake_configuration_args += ["-T ClangCL"]
+
         # Allow adding extra cmake args through the environment. Used by some
         # tests and demos to expand the set of targets included in the pip
         # package.
@@ -769,7 +773,7 @@ def run(self):  # noqa C901
     # platform-specific files using InstallerBuildExt.
     ext_modules=[
         BuiltFile(
-            src_dir="%CMAKE_CACHE_DIR%/third-party/flatbuffers_external_project/bin/",
+            src_dir="%CMAKE_CACHE_DIR%/third-party/flatc_proj/bin/",
             src_name="flatc",
             dst="executorch/data/bin/",
             is_executable=True,
@@ -795,7 +799,8 @@ def run(self):  # noqa C901
             dependent_cmake_flags=["EXECUTORCH_BUILD_EXTENSION_TRAINING"],
         ),
         BuiltExtension(
-            src="codegen/tools/selective_build.*",
+            src_dir="%CMAKE_CACHE_DIR%/codegen/tools/%BUILD_TYPE%/",
+            src="selective_build.cp*" if _is_windows() else "selective_build.*",
             modpath="executorch.codegen.tools.selective_build",
             dependent_cmake_flags=["EXECUTORCH_BUILD_PYBIND"],
         ),