Skip to content

Commit 85a50c8

Browse files
pjcollinspjcollins
authored
[ci] Improve build and test result packaging (#6411)
Attempt to improve our ability to analyze build and test failures with a handful of changes. The Xamarin.Android.Build.Tests and MSBuildDeviceIntegration test suites will now generate and build test projects in a [temporary directory][0] when running on CI. A timestamped directory has been appended to this new test output path, to serve as an extra precaution for identifying any potentially old results. However, the docs mention that the output folder should always be clean at the start of each job: Note: Build.ArtifactStagingDirectory and Build.StagingDirectory are interchangeable. This directory is purged before each new build, so you don't have to clean it up yourself. The result packaging MSBuild targets have been replaced with a step in the xaprepare tool. We will now upload an unzipped directory structure that can be expanded and viewed in the pipelines artifact tab. This should make it easier to download an individual log file for a failing test without having to download zip (or in some cases nested zip) files. Finally, we will now only upload results when a job has a failing step. This should make it easier to quickly find the artifact that corresponds to a failing job, as we will no longer have to search through dozens of similarly named artifacts for a single test failure. [0]: https://docs.microsoft.com/en-us/azure/devops/pipelines/build/variables?view=azure-devops&tabs=yaml#build-variables-devops-services
1 parent 8c300ed commit 85a50c8

File tree

24 files changed

+263
-224
lines changed

24 files changed

+263
-224
lines changed

‎build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,4 @@
1919
<ProjectReference Include="..\..\external\xamarin-android-tools\src\Xamarin.Android.Tools.AndroidSdk\Xamarin.Android.Tools.AndroidSdk.csproj" />
2020
</ItemGroup>
2121

22-
<Import Project="result-packaging.targets" />
23-
2422
</Project>

‎build-tools/Xamarin.Android.Tools.BootstrapTasks/result-packaging.targets‎

Lines changed: 0 additions & 102 deletions
This file was deleted.

‎build-tools/automation/azure-pipelines-nightly.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,9 @@ stages:
6464

6565
- template: yaml-templates/upload-results.yaml
6666
parameters:
67-
solution: $(System.DefaultWorkingDirectory)/xamarin-android/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj
67+
xaSourcePath: $(System.DefaultWorkingDirectory)/xamarin-android
6868
artifactName: Build Results - Nightly macOS
69+
includeBuildResults: true
6970

7071
- stage: test
7172
displayName: Test

‎build-tools/automation/azure-pipelines-oss.yaml‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -130,12 +130,6 @@ stages:
130130
workingDirectory: $(Build.SourcesDirectory)
131131
displayName: make all-tests
132132
133-
- script: >
134-
echo "make package-build-status CONFIGURATION=$(XA.Build.Configuration)" &&
135-
make package-build-status CONFIGURATION=$(XA.Build.Configuration)
136-
workingDirectory: $(Build.SourcesDirectory)
137-
displayName: package build status
138-
139133
- script: >
140134
echo "make run-performance-tests CONFIGURATION=$(XA.Build.Configuration)" &&
141135
make run-performance-tests CONFIGURATION=$(XA.Build.Configuration)
@@ -145,7 +139,8 @@ stages:
145139
146140
- template: yaml-templates/upload-results.yaml
147141
parameters:
148-
artifactName: Build Results - macOS
142+
includeBuildResults: true
143+
artifactName: OSS Build Results - macOS
149144

150145
- stage: linux_stage
151146
displayName: Linux
@@ -199,5 +194,5 @@ stages:
199194

200195
- template: yaml-templates/upload-results.yaml
201196
parameters:
202-
configuration: $(XA.Build.Configuration)
203197
artifactName: OSS Build Results - Linux
198+
includeBuildResults: true

‎build-tools/automation/azure-pipelines.yaml‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -138,8 +138,9 @@ stages:
138138

139139
- template: yaml-templates/upload-results.yaml
140140
parameters:
141-
solution: $(System.DefaultWorkingDirectory)/xamarin-android/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj
141+
xaSourcePath: $(System.DefaultWorkingDirectory)/xamarin-android
142142
artifactName: Build Results - macOS
143+
includeBuildResults: true
143144

144145
- template: yaml-templates/run-xaprepare.yaml
145146
parameters:
@@ -270,6 +271,7 @@ stages:
270271
- template: yaml-templates\upload-results.yaml
271272
parameters:
272273
artifactName: Build Results - Windows
274+
includeBuildResults: true
273275

274276
- template: yaml-templates\fail-on-issue.yaml
275277

@@ -382,6 +384,7 @@ stages:
382384
- template: yaml-templates\upload-results.yaml
383385
parameters:
384386
artifactName: Build Results - Windows DotNet
387+
includeBuildResults: true
385388

386389
- template: yaml-templates\fail-on-issue.yaml
387390

@@ -463,8 +466,9 @@ stages:
463466

464467
- template: yaml-templates/upload-results.yaml
465468
parameters:
466-
solution: $(System.DefaultWorkingDirectory)/xamarin-android/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj
469+
xaSourcePath: $(System.DefaultWorkingDirectory)/xamarin-android
467470
artifactName: Build Results - Linux
471+
includeBuildResults: true
468472

469473
- template: yaml-templates\fail-on-issue.yaml
470474

@@ -819,7 +823,6 @@ stages:
819823

820824
- template: yaml-templates/upload-results.yaml
821825
parameters:
822-
configuration: $(XA.Build.Configuration)
823826
artifactName: Test Results - APKs .NET - macOS
824827

825828
- task: MSBuild@1
@@ -1580,8 +1583,9 @@ stages:
15801583

15811584
- template: yaml-templates/upload-results.yaml
15821585
parameters:
1583-
solution: $(System.DefaultWorkingDirectory)/xamarin-android/build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj
1586+
xaSourcePath: $(System.DefaultWorkingDirectory)/xamarin-android
15841587
artifactName: Legacy Notarization Results - macOS
1588+
includeBuildResults: true
15851589

15861590
# Check - "Xamarin.Android (Finalize Installers Queue Vsix Signing)"
15871591
- job: queue_vsix_signing

‎build-tools/automation/yaml-templates/upload-results.yaml‎

Lines changed: 31 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,37 @@
11
parameters:
2-
solution: build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj
2+
xaSourcePath: $(System.DefaultWorkingDirectory)
33
configuration: $(XA.Build.Configuration)
44
artifactName: results
5+
includeBuildResults: false
56

67
steps:
7-
- task: MSBuild@1
8-
displayName: package build and test results
9-
inputs:
10-
solution: ${{ parameters.solution }}
11-
configuration: ${{ parameters.configuration }}
12-
msbuildArguments: /restore /t:Build,ZipBuildStatus,ZipTestResults /p:BuildStatusZipOutputPath=$(Build.ArtifactStagingDirectory) /p:TestResultZipOutputPath=$(Build.ArtifactStagingDirectory)
13-
condition: always()
14-
15-
- template: publish-artifact.yaml
8+
- template: run-xaprepare.yaml
169
parameters:
17-
displayName: upload build and test results
18-
artifactName: ${{ parameters.artifactName }}
10+
configuration: ${{ parameters.configuration }}
11+
arguments: --s=CopyExtraResultFilesForCI --verbosity v
12+
xaSourcePath: ${{ parameters.xaSourcePath }}
13+
displayName: Copy extra result files
14+
condition: ne(variables['Agent.JobStatus'], 'Succeeded')
15+
16+
- ${{ if eq(parameters.includeBuildResults, false) }}:
17+
- template: publish-artifact.yaml
18+
parameters:
19+
displayName: upload test results
20+
artifactName: ${{ parameters.artifactName }}
21+
targetPath: $(Build.StagingDirectory)/Test${{ parameters.configuration }}
22+
condition: ne(variables['Agent.JobStatus'], 'Succeeded')
23+
24+
# Copy Build$(Configuration) folder into test result root and upload single artifact
25+
- ${{ if eq(parameters.includeBuildResults, true) }}:
26+
- task: CopyFiles@2
27+
inputs:
28+
sourceFolder: $(Build.StagingDirectory)/Build${{ parameters.configuration }}
29+
targetFolder: $(Build.StagingDirectory)/Test${{ parameters.configuration }}/Build${{ parameters.configuration }}
30+
condition: ne(variables['Agent.JobStatus'], 'Succeeded')
31+
32+
- template: publish-artifact.yaml
33+
parameters:
34+
displayName: upload build and test results
35+
artifactName: ${{ parameters.artifactName }}
36+
targetPath: $(Build.StagingDirectory)/Test${{ parameters.configuration }}
37+
condition: ne(variables['Agent.JobStatus'], 'Succeeded')

‎build-tools/scripts/Packaging.mk‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -78,14 +78,3 @@ package-deb: $(ZIP_OUTPUT)
7878
cp LICENSE $(ZIP_OUTPUT_BASENAME)/debian/copyright
7979
cd $(ZIP_OUTPUT_BASENAME) && DEBEMAIL="Xamarin Public Jenkins (auto-signing) <releng@xamarin.com>" dch --create -v $(PRODUCT_VERSION).$(-num-commits-since-version-change) --package xamarin.android-oss --force-distribution --distribution alpha "New release - please see git log for $(GIT_COMMIT)"
8080
cd $(ZIP_OUTPUT_BASENAME) && dpkg-buildpackage -us -uc -rfakeroot
81-
82-
83-
_RESULT_PACKAGE_SUFFIX = -v$(PRODUCT_VERSION).$(-num-commits-since-version-change)_$(OS_NAME)-$(OS_ARCH)_$(GIT_BRANCH)_$(GIT_COMMIT)-$(CONFIGURATION)
84-
85-
"xa-test-results$(_RESULT_PACKAGE_SUFFIX).zip" package-test-results:
86-
msbuild build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj /t:ZipTestResults \
87-
/p:Configuration=$(CONFIGURATION) /p:TestResultZipName="xa-test-results$(_RESULT_PACKAGE_SUFFIX)"
88-
89-
"xa-build-status$(_RESULT_PACKAGE_SUFFIX).zip" package-build-status:
90-
msbuild build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj /t:ZipBuildStatus \
91-
/p:Configuration=$(CONFIGURATION) /p:BuildStatusZipName="xa-build-status$(_RESULT_PACKAGE_SUFFIX)"

‎build-tools/xaprepare/xaprepare/Application/GeneratedMakeRulesFile.cs‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -51,12 +51,7 @@ void Generate (Context context, StreamWriter sw)
5151

5252
WriteVariable ("ZIP_EXTENSION", context.OS.ZipExtension);
5353
WriteVariable ("ZIP_OUTPUT_BASENAME", GetOutputFileName (context, "xamarin.android-oss"));
54-
WriteVariable ("_TEST_RESULTS_BASENAME", GetOutputFileName (context, "xa-test-results"));
55-
WriteVariable ("_BUILD_STATUS_BASENAME", GetOutputFileName (context, "xa-build-status"));
56-
5754
WriteVariable ("ZIP_OUTPUT", "$(ZIP_OUTPUT_BASENAME).$(ZIP_EXTENSION)");
58-
WriteVariable ("_BUILD_STATUS_ZIP_OUTPUT", "$(_BUILD_STATUS_BASENAME).$(ZIP_EXTENSION)");
59-
WriteVariable ("_TEST_RESULTS_ZIP_OUTPUT", "$(_TEST_RESULTS_BASENAME).$(ZIP_EXTENSION)");
6055

6156
var allApiLevels = new List <string> ();
6257
var allPlatformIDs = new List <string> ();
@@ -118,11 +113,6 @@ void Generate (Context context, StreamWriter sw)
118113

119114
WriteListVariable ("_BUNDLE_ZIPS_INCLUDE", Configurables.Defaults.BundleZipsInclude);
120115
WriteListVariable ("_BUNDLE_ZIPS_EXCLUDE", Configurables.Defaults.BundleZipsExclude);
121-
WriteListVariable ("_TEST_RESULTS_BUNDLE_INCLUDE", Configurables.Defaults.TestResultsBundleInclude);
122-
WriteListVariable ("_TEST_RESULTS_BUNDLE_EXCLUDE", Configurables.Defaults.TestResultsBundleExclude);
123-
WriteListVariable ("_BUILD_STATUS_BUNDLE_INCLUDE", Configurables.Defaults.BuildStatusBundleInclude);
124-
WriteListVariable ("_BUILD_STATUS_BUNDLE_INCLUDE", Configurables.Defaults.BuildStatusBundleIncludeConditional, true);
125-
WriteListVariable ("_BUILD_STATUS_BUNDLE_EXCLUDE", Configurables.Defaults.BuildStatusBundleExclude);
126116

127117
sw.WriteLine ();
128118
sw.WriteLine (".PHONY: framework-assemblies");

‎build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs‎

Lines changed: 0 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -218,62 +218,6 @@ public static partial class Defaults
218218
"$(ZIP_OUTPUT_BASENAME)/bin/*/bundle-*.zip"
219219
};
220220

221-
/// <summary>
222-
/// Used in rules.mk generator. Files to include in test results bundle. Must be syntactically
223-
/// correct for GNU Make.
224-
/// </summary>
225-
public static readonly List <string> TestResultsBundleInclude = new List <string> {
226-
"$(wildcard TestResult-*.xml)",
227-
"$(wildcard bin/Test$(CONFIGURATION)/compatibility)",
228-
"$(wildcard bin/Test$(CONFIGURATION)/logcat*)",
229-
"$(wildcard bin/Test$(CONFIGURATION)/msbuild*.binlog*)",
230-
"$(wildcard bin/Test$(CONFIGURATION)/temp)",
231-
"$(wildcard bin/Test$(CONFIGURATION)/EmbeddedDSO)",
232-
"$(wildcard bin/Test$(CONFIGURATION)/CodeBehind)",
233-
"$(wildcard bin/Test$(CONFIGURATION)/TestOutput-*.txt)",
234-
"$(wildcard bin/Test$(CONFIGURATION)/Timing_*)",
235-
"$(wildcard *.csv)",
236-
};
237-
238-
/// <summary>
239-
/// Used in rules.mk generator. Files to exclude from the test results bundle. Must be syntactically
240-
/// correct for GNU Make.
241-
/// </summary>
242-
public static readonly List <string> TestResultsBundleExclude = new List <string> {
243-
};
244-
245-
/// <summary>
246-
/// Used in rules.mk generator. Files to include in build status bundle archive. Must be syntactically
247-
/// correct for GNU Make.
248-
/// </summary>
249-
public static readonly List <string> BuildStatusBundleInclude = new List <string> {
250-
"Configuration.OperatingSystem.props",
251-
"$(wildcard bin/Build$(CONFIGURATION)/msbuild*.binlog)",
252-
"$(shell find . -name 'config.log')",
253-
"$(shell find . -name 'config.status')",
254-
"$(shell find . -name 'config.h')",
255-
"$(shell find . -name 'CMakeCache.txt')",
256-
"$(shell find . -name 'config.h')",
257-
"$(shell find . -name '.ninja_log')",
258-
"$(shell find . -name 'android-*.config.cache')",
259-
"bin/Build$(CONFIGURATION)/XABuildConfig.cs",
260-
};
261-
262-
/// <summary>
263-
/// Used in rules.mk generator. Optional files to include in the build status bundle (included only if
264-
/// they exist). Must be syntactically correct for GNU Make.
265-
/// </summary>
266-
public static readonly List <string> BuildStatusBundleIncludeConditional = new List <string> {
267-
"Configuration.Override.props",
268-
};
269-
270-
/// <summary>
271-
/// Used in rules.mk generator. Files to exclude from the build status bundle. Must be syntactically
272-
/// correct for GNU Make.
273-
/// </summary>
274-
public static readonly List <string> BuildStatusBundleExclude = new List <string> {
275-
};
276-
277221
public static readonly List <NDKTool> NDKTools = new List<NDKTool> {
278222
new NDKTool (name: "as"),
279223
new NDKTool (name: "ld"),

‎build-tools/xaprepare/xaprepare/Scenarios/Scenario_CopyExtraResultFilesForCI.cs‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
using System.IO;
2+
3+
namespace Xamarin.Android.Prepare
4+
{
5+
[Scenario (isDefault: false)]
6+
partial class Scenario_CopyExtraResultFilesForCI : ScenarioNoStandardEndSteps
7+
{
8+
public Scenario_CopyExtraResultFilesForCI ()
9+
: base ("CopyExtraResultFilesForCI", "Copy extra result files to artifact directory")
10+
{
11+
// Do not create a log that we would attempt to copy during this scenario.
12+
Log.Instance.SetLogFile (Path.Combine (Context.Instance.LogDirectory, $"package-results-{Context.Instance.BuildTimeStamp}.txt"));
13+
}
14+
15+
protected override void AddSteps (Context context)
16+
{
17+
Steps.Add (new Step_CopyExtraResultFilesForCI ());
18+
19+
// disable installation of missing programs...
20+
context.SetCondition (KnownConditions.AllowProgramInstallation, false);
21+
22+
// ...but do not signal an error when any are missing
23+
context.SetCondition (KnownConditions.IgnoreMissingPrograms, true);
24+
}
25+
}
26+
}

0 commit comments

Comments
 (0)