[build.ps1] Enable LLDB tests in Windows builds

utils/build-windows-toolchain.bat

@@ -60,7 +60,7 @@ set TMPDIR=%BuildRoot%\tmp
 set NINJA_STATUS=[%%f/%%t][%%p][%%es] 
 
 :: Build the -Test argument, if any, by subtracting skipped tests
-set TestArg=-Test lld,swift,dispatch,foundation,xctest,swift-format,sourcekit-lsp,
+set TestArg=-Test lld,lldb,swift,dispatch,foundation,xctest,swift-format,sourcekit-lsp,
 for %%I in (%SKIP_TESTS%) do (call set TestArg=%%TestArg:%%I,=%%)
 if "%TestArg:~-1%"=="," (set TestArg=%TestArg:~0,-1%) else (set TestArg= )
 

utils/build.ps1

@@ -725,6 +725,14 @@ function Fetch-Dependencies {
 
   DownloadAndVerify $PinnedBuild "$BinaryCache\$PinnedToolchain.exe" $PinnedSHA256
 
+  if ($Test -contains "lldb") {
+    # The make tool isn't part of MSYS
+    $GnuWin32MakeURL = "https://downloads.sourceforge.net/project/ezwinports/make-4.4.1-without-guile-w32-bin.zip"
+    $GnuWin32MakeHash = "fb66a02b530f7466f6222ce53c0b602c5288e601547a034e4156a512dd895ee7"
+    DownloadAndVerify $GnuWin32MakeURL "$BinaryCache\GnuWin32Make-4.4.1.zip" $GnuWin32MakeHash
+    Extract-ZipFile GnuWin32Make-4.4.1.zip $BinaryCache GnuWin32Make-4.4.1
+  }
+
   # TODO(compnerd) stamp/validate that we need to re-extract
   New-Item -ItemType Directory -ErrorAction Ignore $BinaryCache\toolchains | Out-Null
   Extract-Toolchain "$PinnedToolchain.exe" $BinaryCache $PinnedToolchain
@@ -772,6 +780,28 @@ function Fetch-Dependencies {
       Write-Output "Installing 'setuptools-75.1.0-py3-none-any.whl' ..."
       Invoke-Program -OutNull $Python '-I' -m pip install "$BinaryCache\python\setuptools-75.1.0-py3-none-any.whl" --disable-pip-version-check
     }
+    if ($Test -contains "lldb") {
+      # 'psutil' is required for testing LLDB
+      try {
+        Invoke-Program -OutNull $Python -c 'import psutil' *> $null
+      } catch {
+        $WheelURL = "https://files.pythonhosted.org/packages/11/91/87fa6f060e649b1e1a7b19a4f5869709fbf750b7c8c262ee776ec32f3028/psutil-6.1.0-cp37-abi3-win_amd64.whl"
+        $WheelHash = "a8fb3752b491d246034fa4d279ff076501588ce8cbcdbb62c32fd7a377d996be"
+        DownloadAndVerify $WheelURL "$BinaryCache\python\psutil-6.1.0-cp37-abi3-win_amd64.whl" $WheelHash
+        Write-Output "Installing 'psutil-6.1.0-cp37-abi3-win_amd64.whl' ..."
+        Invoke-Program -OutNull $Python '-I' -m pip install "$BinaryCache\python\psutil-6.1.0-cp37-abi3-win_amd64.whl" --disable-pip-version-check
+      }
+      # 'unittest2' is required for testing LLDB
+      try {
+        Invoke-Program -OutNull $Python -c 'import unittest2' *> $null
+      } catch {
+        $WheelURL = "https://files.pythonhosted.org/packages/72/20/7f0f433060a962200b7272b8c12ba90ef5b903e218174301d0abfd523813/unittest2-1.1.0-py2.py3-none-any.whl"
+        $WheelHash = "13f77d0875db6d9b435e1d4f41e74ad4cc2eb6e1d5c824996092b3430f088bb8"
+        DownloadAndVerify $WheelURL "$BinaryCache\python\unittest2-1.1.0-py2.py3-none-any.whl" $WheelHash
+        Write-Output "Installing 'unittest2-1.1.0-py2.py3-none-any.whl' ..."
+        Invoke-Program -OutNull $Python '-I' -m pip install "$BinaryCache\python\unittest2-1.1.0-py2.py3-none-any.whl" --disable-pip-version-check
+      }
+    }
   }
 
   Download-Python $HostArchName
@@ -1205,6 +1235,13 @@ function Build-CMakeProject {
     } elseif ($UsePinnedCompilers.Contains("Swift")) {
       $env:Path = "$(Get-PinnedToolchainRuntime);${env:Path}"
     }
+
+    if ($ToBatch) {
+      Write-Output ""
+      Write-Output "echo cmake.exe $cmakeGenerateArgs"
+    } else {
+      Write-Host "cmake.exe $cmakeGenerateArgs"
+    }
     Invoke-Program cmake.exe @cmakeGenerateArgs
 
     # Build all requested targets
@@ -1463,7 +1500,48 @@ function Build-Compilers() {
 
       if ($TestClang) { $Targets += @("check-clang") }
       if ($TestLLD) { $Targets += @("check-lld") }
-      if ($TestLLDB) { $Targets += @("check-lldb") }
+      if ($TestLLDB) {
+        $Targets += @("check-lldb", "check-lldb", "check-lldb", "check-lldb", "check-lldb", "check-lldb", "check-lldb", "check-lldb", "check-lldb", "check-lldb")
+
+        function Select-LitTestOverrides {
+          param([string] $TestStatus)
+
+          $MatchingLines=(Get-Content $PSScriptRoot/windows-llvm-lit-test-overrides.txt | Select-String -Pattern "`^${TestStatus}.*$")
+          $TestNames=$MatchingLines | ForEach-Object { ($_ -replace $TestStatus,"").Trim() }
+          return $TestNames
+        }
+
+        # Override some test results with llvm-lit.
+        $TestsToXFail=Select-LitTestOverrides "xfail"
+        $TestsToSkip=Select-LitTestOverrides "skip"
+        $env:LIT_XFAIL=$TestsToXFail -join ";"
+        $env:LIT_FILTER_OUT="($($TestsToSkip -join '|'))"
+
+        # Transitive dependency of _lldb.pyd
+        $RuntimeBinaryCache = Get-TargetProjectBinaryCache $Arch Runtime
+        cp $RuntimeBinaryCache\bin\swiftCore.dll "$CompilersBinaryCache\lib\site-packages\lldb"
+
+        # Runtime dependencies of repl_swift.exe
+        $SwiftRTSubdir = "lib\swift\windows"
+        Write-Host "Copying '$RuntimeBinaryCache\$SwiftRTSubdir\$($Arch.LLVMName)\swiftrt.obj' to '$CompilersBinaryCache\$SwiftRTSubdir'"
+        cp "$RuntimeBinaryCache\$SwiftRTSubdir\$($Arch.LLVMName)\swiftrt.obj" "$CompilersBinaryCache\$SwiftRTSubdir"
+        Write-Host "Copying '$RuntimeBinaryCache\bin\swiftCore.dll' to '$CompilersBinaryCache\bin'"
+        cp "$RuntimeBinaryCache\bin\swiftCore.dll" "$CompilersBinaryCache\bin"
+        Write-Host "Copying '$RuntimeBinaryCache\bin\swiftCore.dll' to '$CompilersBinaryCache\$SwiftRTSubdir'"
+        cp "$RuntimeBinaryCache\bin\swiftCore.dll" "$CompilersBinaryCache\$SwiftRTSubdir"
+
+        $TestingDefines += @{
+          LLDB_INCLUDE_TESTS = "YES";
+          # Check for required Python modules in CMake
+          LLDB_ENFORCE_STRICT_TEST_REQUIREMENTS = "YES";
+          # No watchpoint support on windows: https://github.com/llvm/llvm-project/issues/24820
+          LLDB_TEST_USER_ARGS = "--skip-category=watchpoint";
+          # gtest sharding breaks llvm-lit's --xfail and LIT_XFAIL inputs: https://github.com/llvm/llvm-project/issues/102264
+          LLVM_LIT_ARGS = "-v --no-gtest-sharding --show-xfail --show-unsupported";
+          # LLDB Unit tests link against this library
+          LLVM_UNITTEST_LINK_FLAGS = "$($Arch.SDKInstallRoot)\usr\lib\swift\windows\$($Arch.LLVMName)\swiftCore.lib";
+        }
+      }
       if ($TestLLVM) { $Targets += @("check-llvm") }
       if ($TestSwift) { $Targets += @("check-swift", "SwiftCompilerPlugin") }
     } else {
@@ -1487,6 +1565,15 @@ function Build-Compilers() {
       $SwiftFlags += @("-Xcc", "-D_ALLOW_COMPILER_AND_STL_VERSION_MISMATCH");
     }
 
+    if (-not $IsCrossCompiling) {
+      # We hardcode LLVM_DEFAULT_TARGET_TRIPLE to x86_64-unknown-windows-msvc,
+      # but the host triple might be inferred as x86_64-pc-windows-msvc, which
+      # causes llvm-lit to skip `REQUIRES: native` tests.
+      $TestingDefines += @{
+        LLVM_HOST_TRIPLE = $Arch.LLVMTarget
+      }
+    }
+
     Build-CMakeProject `
       -Src $SourceCache\llvm-project\llvm `
       -Bin $CompilersBinaryCache `
@@ -1507,6 +1594,7 @@ function Build-Compilers() {
         LLDB_PYTHON_EXT_SUFFIX = ".pyd";
         LLDB_PYTHON_RELATIVE_PATH = "lib/site-packages";
         LLDB_TABLEGEN = (Join-Path -Path $BuildTools -ChildPath "lldb-tblgen.exe");
+        LLDB_TEST_MAKE = "$BinaryCache\GnuWin32Make-4.4.1\bin\make.exe";
         LLVM_CONFIG_PATH = (Join-Path -Path $BuildTools -ChildPath "llvm-config.exe");
         LLVM_EXTERNAL_SWIFT_SOURCE_DIR = "$SourceCache\swift";
         LLVM_NATIVE_TOOL_DIR = $BuildTools;

utils/windows-llvm-lit-test-overrides.txt

@@ -0,0 +1,191 @@
+# build.ps1 overrides the status of each test in this file when LLVM tests are run with lit.
+#
+# Only full test names should be used. For example: "lldb-api :: TestName/Foo.py".
+# Please attach an issue URL to each test or group of tests, especially those that
+# are skipped. For flakey tests, please file an issue at swiftlang/llvm-project with
+# the "flakey-test" label.
+#
+# Allowed labels:
+# - skip      Skip the test (Use sparingly, for UNRESOLVED tests which can't be marked XFAIL).
+# - xfail     Mark the test as XFAIL
+
+# https://github.com/thebrowsercompany/swift-build/issues/196
+
+### Tests that time out occasionally ###
+
+skip lldb-api :: commands/dwim-print/TestDWIMPrint.py
+skip lldb-api :: commands/thread/backtrace/TestThreadBacktraceRepeat.py
+skip lldb-api :: functionalities/inferior-crashing/TestInferiorCrashing.py
+skip lldb-api :: functionalities/step_scripted/TestStepScripted.py
+
+### Tests that fail occasionally ###
+
+# https://github.com/swiftlang/llvm-project/issues/9101
+skip lldb-api :: tools/lldb-server/TestGdbRemote_qThreadStopInfo.py
+
+# https://github.com/swiftlang/llvm-project/issues/9705
+skip lldb-api :: python_api/section/TestSectionAPI.py
+
+# Untriaged
+skip lldb-api :: lang/BoundsSafety/array_of_ptrs/TestArrayOfBoundsSafetyPointers.py
+skip lldb-api :: lang/BoundsSafety/out_of_bounds_pointer/TestOutOfBoundsPointer.py
+skip lldb-shell :: Commands/command-expr-diagnostics.test
+skip lldb-api :: commands/statistics/basic/TestStats.py
+skip lldb-unit :: Host/./HostTests.exe
+
+### Tests that pass occasionally ###
+
+# Fixed upstream: https://github.com/llvm/llvm-project/commit/ec009994a06338995dfb6431c943b299f9327fd2
+# But patches don't apply on stable branch due to downstream changes.
+skip lldb-api :: functionalities/archives/TestBSDArchives.py
+skip lldb-api :: macosx/duplicate-archive-members/TestDuplicateMembers.py
+
+skip lldb-api :: tools/lldb-server/TestGdbRemoteThreadsInStopReply.py
+
+### Tests that pass accidentally ###
+
+https://github.com/llvm/llvm-project/issues/116972
+skip lldb-api :: api/command-return-object/TestSBCommandReturnObject.py
+skip lldb-api :: api/multiple-targets/TestMultipleTargets.py
+
+### Tests that fail reliably ###
+
+xfail lldb-api :: lang/cpp/unique-types4/TestUniqueTypes4.py
+xfail lldb-shell :: Recognizer/verbose_trap.test
+xfail lldb-shell :: Settings/TestEchoCommands.test
+xfail lldb-shell :: Swift/MissingVFSOverlay.test
+xfail lldb-shell :: Swift/No.swiftmodule.test
+xfail lldb-shell :: Swift/ToolchainMismatch.test
+xfail lldb-shell :: Swift/global.test
+xfail lldb-shell :: Swift/runtime-initialization.test
+xfail lldb-shell :: SwiftREPL/BreakpointSimple.test
+xfail lldb-shell :: SwiftREPL/ClosureScope.test
+xfail lldb-shell :: SwiftREPL/ExistentialTypes.test
+xfail lldb-shell :: SwiftREPL/LookupAfterImport.test
+xfail lldb-shell :: SwiftREPL/LookupWithAttributedImport.test
+xfail lldb-shell :: SwiftREPL/OpenClass.test
+xfail lldb-shell :: SwiftREPL/OptionalUnowned.test
+xfail lldb-shell :: SwiftREPL/RedirectInputUnreadable.test
+xfail lldb-shell :: SwiftREPL/SwiftInterface.test
+xfail lldb-shell :: SymbolFile/DWARF/x86/dead-code-filtering.yaml
+
+### Unittests that fail reliably ###
+
+# Skip SymbolTests because we cannot xfail unittests by name. We would need to
+# specify their indexes, but these are subject to change. Right now, failures
+# are:
+#  FAIL: lldb-unit :: Symbol/./SymbolTests.exe/24/27 (1 of 27)
+#  FAIL: lldb-unit :: Symbol/./SymbolTests.exe/21/27 (3 of 27)
+#  FAIL: lldb-unit :: Symbol/./SymbolTests.exe/22/27 (9 of 27)
+#  FAIL: lldb-unit :: Symbol/./SymbolTests.exe/25/27 (11 of 27)
+#
+# Respective tests are:
+#  lldb-unit :: Symbol/./SymbolTests.exe/ClangArgs/UniquingCollisionWithAddedFlags
+#  lldb-unit :: Symbol/./SymbolTests.exe/ClangArgs/UniquingCollisionWithExistingFlags
+#  lldb-unit :: Symbol/./SymbolTests.exe/TestSwiftASTContext/ApplyWorkingDir
+#  lldb-unit :: Symbol/./SymbolTests.exe/TestSwiftASTContext/PluginPath
+#
+skip lldb-unit :: Symbol/./SymbolTests.exe
+
+### Tests that pass locally, but not in CI ###
+
+# https://github.com/swiftlang/llvm-project/issues/9539
+xfail lldb-shell :: SwiftREPL/Basic.test
+xfail lldb-shell :: SwiftREPL/Class.test
+xfail lldb-shell :: SwiftREPL/ComputedProperties.test
+xfail lldb-shell :: SwiftREPL/Deadlock.test
+xfail lldb-shell :: SwiftREPL/DiagnosticOptions.test
+xfail lldb-shell :: SwiftREPL/Dict.test
+xfail lldb-shell :: SwiftREPL/ErrorReturn.test
+xfail lldb-shell :: SwiftREPL/ExclusivityREPL.test
+xfail lldb-shell :: SwiftREPL/GenericTypealias.test
+xfail lldb-shell :: SwiftREPL/Generics.test
+xfail lldb-shell :: SwiftREPL/ImportError.test
+xfail lldb-shell :: SwiftREPL/MetatypeRepl.test
+xfail lldb-shell :: SwiftREPL/Optional.test
+xfail lldb-shell :: SwiftREPL/PropertyWrapperTopLevel.test
+xfail lldb-shell :: SwiftREPL/RecursiveClass.test
+xfail lldb-shell :: SwiftREPL/Redefinition.test
+xfail lldb-shell :: SwiftREPL/RedirectInput.test
+xfail lldb-shell :: SwiftREPL/RedirectInputNoSuchFile.test
+xfail lldb-shell :: SwiftREPL/Regex.test
+xfail lldb-shell :: SwiftREPL/SimpleExpressions.test
+xfail lldb-shell :: SwiftREPL/Struct.test
+xfail lldb-shell :: SwiftREPL/Subclassing.test
+xfail lldb-shell :: SwiftREPL/SwiftInterfaceForceModuleLoadMode.test
+xfail lldb-shell :: SwiftREPL/SwiftTypeLookup.test
+xfail lldb-shell :: SwiftREPL/SyntaxError.test
+xfail lldb-shell :: SwiftREPL/UninitVariables.test
+xfail lldb-shell :: SwiftREPL/ZeroSizeStruct.test
+xfail lldb-shell :: SwiftREPL/enum-singlecase.test
+xfail lldb-shell :: SwiftREPL/one-char-string.test
+
+xfail lldb-api :: commands/apropos/with-process/TestAproposWithProcess.py
+xfail lldb-api :: commands/command/nested_alias/TestNestedAlias.py
+xfail lldb-api :: commands/expression/entry-bp/TestExprEntryBP.py
+xfail lldb-api :: commands/memory/write/TestMemoryWrite.py
+xfail lldb-api :: functionalities/breakpoint/address_breakpoints/TestAddressBreakpoints.py
+xfail lldb-api :: functionalities/breakpoint/auto_continue/TestBreakpointAutoContinue.py
+xfail lldb-api :: functionalities/breakpoint/breakpoint_conditions/TestBreakpointConditions.py
+xfail lldb-api :: functionalities/breakpoint/breakpoint_options/TestBreakpointOptions.py
+xfail lldb-api :: functionalities/breakpoint/step_over_breakpoint/TestStepOverBreakpoint.py
+xfail lldb-api :: functionalities/conditional_break/TestConditionalBreak.py
+xfail lldb-api :: functionalities/memory/find/TestMemoryFind.py
+xfail lldb-api :: lang/c/anonymous/TestAnonymous.py
+xfail lldb-api :: lang/c/array_types/TestArrayTypes.py
+xfail lldb-api :: lang/c/enum_types/TestEnumTypes.py
+xfail lldb-api :: lang/c/forward/TestForwardDeclaration.py
+xfail lldb-api :: lang/c/function_types/TestFunctionTypes.py
+xfail lldb-api :: lang/c/register_variables/TestRegisterVariables.py
+xfail lldb-api :: lang/c/set_values/TestSetValues.py
+xfail lldb-api :: lang/c/shared_lib/TestSharedLib.py
+xfail lldb-api :: lang/cpp/class_types/TestClassTypes.py
+xfail lldb-api :: lang/cpp/inlines/TestInlines.py
+xfail lldb-api :: python_api/compile_unit/TestCompileUnitAPI.py
+xfail lldb-api :: python_api/thread/TestThreadAPI.py
+xfail lldb-api :: source-manager/TestSourceManager.py
+xfail lldb-shell :: Driver/TestConvenienceVariables.test
+
+# https://github.com/swiftlang/llvm-project/issues/9620
+xfail lldb-shell :: Swift/expression-diagnostics.test
+
+# https://github.com/swiftlang/llvm-project/issues/9540
+xfail lldb-shell :: SymbolFile/NativePDB/local-variables.cpp
+xfail lldb-shell :: SymbolFile/NativePDB/stack_unwinding01.cpp
+xfail lldb-api :: lang/c/local_variables/TestLocalVariables.py
+
+# https://github.com/swiftlang/llvm-project/issues/9637
+xfail lldb-api :: python_api/address_range/TestAddressRange.py
+
+# https://github.com/swiftlang/llvm-project/issues/9643
+xfail lldb-shell :: Commands/command-process-launch-user-entry.test
+
+### Tests that fail the build, but show up as UNRESOLVED ***
+
+skip lldb-api :: functionalities/breakpoint/breakpoint_command/TestBreakpointCommandsFromPython.py
+skip lldb-api :: tools/lldb-dap/instruction-breakpoint/TestDAP_instruction_breakpoint.py
+skip lldb-api :: tools/lldb-dap/output/TestDAP_output.py
+
+### Unconfirmed timeouts/xfails/xpasses: skip for conservative runs ###
+
+# https://github.com/llvm/llvm-project/issues/62983
+skip lldb-api :: functionalities/var_path/TestVarPath.py
+
+# https://github.com/swiftlang/llvm-project/issues/9073
+skip lldb-api :: lang/c/trampoline_stepping/TestTrampolineStepping.py
+
+# https://github.com/swiftlang/llvm-project/issues/9072
+skip lldb-api :: lang/cpp/bitfields/TestCppBitfields.py
+
+# https://github.com/swiftlang/llvm-project/issues/9099
+skip lldb-api :: tools/lldb-server/TestGdbRemoteLaunch.py
+
+# https://github.com/swiftlang/llvm-project/issues/9100
+skip lldb-api :: tools/lldb-server/TestLldbGdbServer.py
+
+skip lldb-shell :: Swift/astcontext_error.test
+skip lldb-shell :: Swift/cond-breakpoint.test
+
+skip lldb-api :: functionalities/data-formatter/data-formatter-cpp/TestDataFormatterCpp.py
+skip lldb-api :: lang/cpp/frame-var-depth-and-elem-count/TestFrameVarDepthAndElemCount.py
+skip lldb-api :: functionalities/inferior-crashing/recursive-inferior/TestRecursiveInferiorStep.py