[tests] Use dotnet test to run AndroidSdk-Tests (#102)

Update `Xamarin.Android.Tools.AndroidSdk-Tests.csproj` to target only
`netcoreapp3.1`, and updates all test running logic to use
`dotnet test`.  This allows us to remove the `globalPackagesFolder`
override from `NuGet.config`, as `NUnit.ConsoleRunner` is no longer
referenced or used.

The YAML pipeline has also been consolidated a bit to reduce step
duplication.  A matrix strategy is now used to run the same steps in
parallel on both macOS and Windows.  OS-specific steps are now hidden
behind a condition so that we don't duplicate artifact uploading and
publishing.

Author: pjcollins

.gitignore

@@ -12,4 +12,5 @@ bin
 **/obj
 packages/
 TestResult-*.xml
-.vs/
+**/TestResults/*.trx
+.vs/

.vscode/extensions.json

@@ -2,7 +2,8 @@
     "recommendations": [
       "ms-vscode.csharp",
       "ms-vscode.mono-debug",
-      "wghats.vscode-nxunit-test-adapter",
+      "hbenl.vscode-test-explorer",
+      "derivitec-ltd.vscode-dotnet-adapter",
       "visualstudioexptteam.vscodeintellicode",
     ]
 }

.vscode/settings.json

@@ -1,6 +1,3 @@
 {
-    "nxunitExplorer.nunit": "packages/NUnit.ConsoleRunner.3.9.0/tools/nunit3-console.exe",
-    "nxunitExplorer.modules": [
-        "bin/TestDebug/Xamarin.Android.Tools.AndroidSdk-Tests.dll"
-    ]
-}
+    "dotnetCoreExplorer.searchpatterns": "bin/Test*net*/**/*-Tests.dll",
+}

.vscode/tasks.json

@@ -30,7 +30,11 @@
         {
             "label": "Run Unit Tests",
             "type": "shell",
-            "command": "make run-all-tests",
+            "command": "dotnet",
+            "args": [
+                "test",
+                "${workspaceFolder}/tests/Xamarin.Android.Tools.AndroidSdk-Tests/Xamarin.Android.Tools.AndroidSdk-Tests.csproj"
+            ],
             "group": {
                 "kind": "test",
                 "isDefault": true

Makefile

@@ -1,7 +1,5 @@
 CONFIGURATION   := Debug
-NUNIT_CONSOLE   := packages/nunit.consolerunner/3.9.0/tools/nunit3-console.exe
 OS              := $(shell uname)
-RUNTIME         := mono --debug=casts
 V               ?= 0
 
 include build-tools/scripts/msbuild.mk
@@ -15,26 +13,6 @@ clean:
 prepare:
 	nuget restore Xamarin.Android.Tools.sln
 
-run-all-tests: run-nunit-tests
-
-# $(call RUN_NUNIT_TEST,filename,log-lref?)
-define RUN_NUNIT_TEST
-	MONO_TRACE_LISTENER=Console.Out \
-	$(RUNTIME) \
-		$(NUNIT_CONSOLE) $(NUNIT_EXTRA) $(1) \
-		$(if $(RUN),-run:$(RUN)) \
-		--result="TestResult-$(basename $(notdir $(1))).xml" \
-		-output=bin/Test$(CONFIGURATION)/TestOutput-$(basename $(notdir $(1))).txt \
-	|| true ; \
-	if [ -f "bin/Test$(CONFIGURATION)/TestOutput-$(basename $(notdir $(1))).txt" ] ; then \
-		cat bin/Test$(CONFIGURATION)/TestOutput-$(basename $(notdir $(1))).txt ; \
-	fi
-endef
-
-$(NUNIT_CONSOLE): prepare
-
-NUNIT_TESTS = \
-	bin/Test$(CONFIGURATION)/Xamarin.Android.Tools.AndroidSdk-Tests.dll
-
-run-nunit-tests: $(NUNIT_TESTS)
-	$(foreach t,$(NUNIT_TESTS), $(call RUN_NUNIT_TEST,$(t),1))
+run-all-tests:
+	dotnet test -l "console;verbosity=detailed" -l trx \
+		tests/Xamarin.Android.Tools.AndroidSdk-Tests/Xamarin.Android.Tools.AndroidSdk-Tests.csproj

NuGet.config

@@ -1,8 +1,4 @@
 <?xml version="1.0" encoding="utf-8"?>
-<!--
-     Our testing infrastructure depends on a known location for:
-     - nunit3-console.exe
--->
 <configuration>
   <packageSources>
     <clear />
@@ -15,7 +11,4 @@
      <!-- Android binary, to support delta APK install -->
      <add key="xamarin.android util" value="https://pkgs.dev.azure.com/xamarin/public/_packaging/Xamarin.Android/nuget/v3/index.json" />
   </packageSources>
-  <config>
-    <add key="globalPackagesFolder" value="packages" />
-  </config>
 </configuration>

README.md

@@ -38,7 +38,11 @@ The default `make all` target accepts the following optional
 
 # Build
 
-To build **xamarin-android-tools**, first prepare the project:
+To build **xamarin-android-tools**:
+
+	msbuild /restore Xamarin.Android.Tools.sln
+
+Alternatively, first prepare the project:
 
 	make prepare
 
@@ -53,7 +57,7 @@ Next, run `make`:
 
 To run the unit tests:
 
-	make run-all-tests
+	dotnet test tests/Xamarin.Android.Tools.AndroidSdk-Tests/Xamarin.Android.Tools.AndroidSdk-Tests.csproj -l "console;verbosity=detailed"
 
 # Build Output Directory Structure
 

azure-pipelines.yaml

@@ -11,77 +11,82 @@ pr:
 # Global variables
 variables:
   DotNetCoreVersion: 3.1.100
-  HostedMac: Hosted Mac Internal
-  HostedWinVS2019: Hosted Windows 2019 with VS2019
 
 jobs:
-- job: windows
-  displayName: windows
-  pool: $(HostedWinVS2019)
+- job: build
+  displayName: Build and Test
+  timeoutInMinutes: 60
+  cancelTimeoutInMinutes: 2
+
+  strategy:
+    matrix:
+      macOS:
+        vmImage: macOS-10.15
+      win2019:
+        vmImage: windows-2019
+
+  pool:
+    vmImage: $(vmImage)
+
+  workspace:
+    clean: all
+
   steps:
+  - checkout: self
+    clean: true
+
   - task: UseDotNet@2
     displayName: Use .NET Core $(DotNetCoreVersion)
     inputs:
       version: $(DotNetCoreVersion)
+
   - task: NuGetToolInstaller@0
     displayName: 'Install NuGet'
     inputs:
       versionSpec: 5.x
+
+  - script: |
+      dotnet tool install --global boots
+      boots https://download.mono-project.com/archive/6.12.0/macos-10-universal/MonoFramework-MDK-6.12.0.107.macos10.xamarin.universal.pkg
+    displayName: Install Mono 6.12
+    condition: and(succeeded(), eq(variables['agent.os'], 'Darwin'))
+
   - task: NuGetCommand@2
     displayName: 'NuGet Restore'
     inputs:
       restoreSolution: Xamarin.Android.Tools.sln
       feedsToUse: config
       nugetConfigPath: NuGet.config
+
   - task: MSBuild@1
     displayName: 'Build solution Xamarin.Android.Tools.sln'
     inputs:
       solution: Xamarin.Android.Tools.sln
-  - task: VSTest@2
+
+  - task: DotNetCoreCLI@2
     displayName: 'Run Tests'
     inputs:
-      testAssemblyVer2: 'bin\TestDebug\*-Tests.dll'
-      testRunTitle: windows-tests
+      command: test
+      projects: bin/TestDebug-net*/**/*-Tests.dll
+      testRunTitle: Xamarin.Android.Tools Tests - $(vmImage)
+
   - powershell: |
       $hashOfLastVersionChange = & "git" "log" "--follow" "-1" "--pretty=%H" "nuget.version"
       $commitsSinceVersionChange = & "git" "rev-list" "--count" "$hashOfLastVersionChange..HEAD"
       $majorMinor = Get-Content "nuget.version"
       $version = "$majorMinor.$commitsSinceVersionChange"
       Write-Host "##vso[task.setvariable variable=xat.nuget.version]$version"
+    condition: and(succeeded(), eq(variables['agent.os'], 'Windows_NT'))
+
   - task: MSBuild@1
     displayName: 'Build NuGet'
     inputs:
       solution: 'src\Xamarin.Android.Tools.AndroidSdk\Xamarin.Android.Tools.AndroidSdk.csproj'
       msbuildArguments: '/t:pack /p:Version=$(xat.nuget.version) /p:OutputPath=$(Build.ArtifactStagingDirectory)'
+    condition: and(succeeded(), eq(variables['agent.os'], 'Windows_NT'))
+
   - task: PublishBuildArtifacts@1
     displayName: Upload Artifacts
     inputs:
       pathtoPublish: $(Build.ArtifactStagingDirectory)
-- job: mac
-  displayName: mac
-  pool: $(HostedMac)
-  steps:
-  - task: UseDotNet@2
-    displayName: Use .NET Core $(DotNetCoreVersion)
-    inputs:
-      version: $(DotNetCoreVersion)
-  - script: |
-      dotnet tool install --global boots
-      boots https://download.mono-project.com/archive/6.4.0/macos-10-universal/MonoFramework-MDK-6.4.0.198.macos10.xamarin.universal.pkg
-    displayName: Install Mono 6.4
-  - script: make prepare CONFIGURATION=$(Build.Configuration)
-    displayName: make prepare
-
-  - script: make all CONFIGURATION=$(Build.Configuration)
-    displayName: make all
-
-  - script: make run-all-tests CONFIGURATION=$(Build.Configuration)
-    displayName: make run-all-tests
-
-  - task: PublishTestResults@2
-    condition: always()
-    inputs:
-      testResultsFormat: NUnit
-      testResultsFiles: TestResult*.xml
-      testRunTitle: mac-tests
-      failTaskOnFailedTests: true
+    condition: and(succeeded(), eq(variables['agent.os'], 'Windows_NT'))

tests/Xamarin.Android.Tools.AndroidSdk-Tests/Xamarin.Android.Tools.AndroidSdk-Tests.csproj

@@ -1,20 +1,19 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFrameworks>net461;netcoreapp3.1</TargetFrameworks>
+    <!-- $(TargetFrameworks) is used to allow the $(TargetFramework) check in Directory.Build.props to work.
+        If $(TargetFramework) is declared here instead, it will not be evaluated before Directory.Build.props
+        is loaded and the wrong $(TestOutputFullPath) will be used. -->
+    <TargetFrameworks>netcoreapp3.1</TargetFrameworks>
     <IsPackable>false</IsPackable>
-    <AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
-  </PropertyGroup>
-
-  <PropertyGroup>
     <OutputPath>$(TestOutputFullPath)</OutputPath>
+    <AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="nunit" Version="3.11.0" />
-    <PackageReference Include="NUnit.ConsoleRunner" Version="3.9.0" />
-    <PackageReference Include="NUnit3TestAdapter" Version="3.13.0" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.2.0" />
+    <PackageReference Include="NUnit" Version="3.12.0" />
+    <PackageReference Include="NUnit3TestAdapter" Version="3.16.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0"/>
   </ItemGroup>
 
   <ItemGroup>