Skip to content

Commit bb5a258

Browse files
jmagmanjmagman
authored
[ci] Upload screenshots, logs, and Xcode test results for drive and integration_test runs (flutter#7430)
1. Native Xcode tests will output a helpful "xcresult" package on failure containing logs, screenshots, and screen recordings. Zip and upload these results when tests fail. 2. Pass `flutter test --debug-logs-dir` flag to upload logs like flutter#142643. 3. Pass `flutter drive --screenshot` flag to upload screenshots on timeout like flutter#96973. Example of [failing Xcode analyzer build](https://ci.chromium.org/ui/p/flutter/builders/try/Mac_arm64%20ios_platform_tests_shard_5%20master/17374/overview) has the [zipped xcresult](https://storage.googleapis.com/flutter_logs/flutter/ff98c32e-18ca-4ad4-a910-9db1d7f7e4b0/xcode%20analyze/ff98c32e-18ca-4ad4-a910-9db1d7f7e4b0/xcodebuild-2024-10-25T09:56:46.440913.zip) attached as a log. ![Screenshot 2024-10-25 at 10 10 36�AM](https://github.com/user-attachments/assets/dd7ae9bc-6161-4381-8a4f-f10b8c981801) The unzipped xcresult looks like this in Xcode: ![Screenshot 2024-10-25 at 10 11 55�AM](https://github.com/user-attachments/assets/d4dd8420-f272-459c-9785-ab0c03887a74) A [failing "native test" step build](https://ci.chromium.org/ui/p/flutter/builders/try/Mac_arm64%20macos_platform_tests%20master%20-%20packages/17315/overview): ![Screenshot 2024-10-25 at 10 19 55�AM](https://github.com/user-attachments/assets/76a86a15-2150-482a-8b15-e3e7ac90485e) Fixes flutter#144795
1 parent d3d563d commit bb5a258

File tree

8 files changed

+242
-40
lines changed

8 files changed

+242
-40
lines changed

‎script/tool/lib/src/common/core.dart‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
// found in the LICENSE file.
44

55
import 'package:file/file.dart';
6+
import 'package:platform/platform.dart';
67
import 'package:pub_semver/pub_semver.dart';
78

89
/// The signature for a print handler for commands that allow overriding the
@@ -127,3 +128,13 @@ const int exitCommandFoundErrors = 1;
127128

128129
/// A exit code for [ToolExit] for a failure to run due to invalid arguments.
129130
const int exitInvalidArguments = 2;
131+
132+
/// The directory to which to write logs and other artifacts, if set in CI.
133+
Directory? ciLogsDirectory(Platform platform, FileSystem fileSystem) {
134+
final String? logsDirectoryPath = platform.environment['FLUTTER_LOGS_DIR'];
135+
Directory? logsDirectory;
136+
if (logsDirectoryPath != null) {
137+
logsDirectory = fileSystem.directory(logsDirectoryPath);
138+
}
139+
return logsDirectory;
140+
}

‎script/tool/lib/src/common/xcode.dart‎

Lines changed: 71 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@ import 'dart:convert';
66
import 'dart:io' as io;
77

88
import 'package:file/file.dart';
9+
import 'package:platform/platform.dart';
910

11+
import 'core.dart';
1012
import 'output_utils.dart';
1113
import 'process_runner.dart';
1214

@@ -33,36 +35,81 @@ class Xcode {
3335
/// Runs an `xcodebuild` in [directory] with the given parameters.
3436
Future<int> runXcodeBuild(
3537
Directory exampleDirectory,
36-
String platform, {
38+
String targetPlatform, {
3739
List<String> actions = const <String>['build'],
3840
required String workspace,
3941
required String scheme,
4042
String? configuration,
4143
List<String> extraFlags = const <String>[],
42-
}) {
43-
File? disabledSandboxEntitlementFile;
44-
if (actions.contains('test') && platform.toLowerCase() == 'macos') {
45-
disabledSandboxEntitlementFile = _createDisabledSandboxEntitlementFile(
46-
exampleDirectory.childDirectory(platform.toLowerCase()),
47-
configuration ?? 'Debug',
48-
);
49-
}
50-
final List<String> args = <String>[
51-
_xcodeBuildCommand,
52-
...actions,
53-
...<String>['-workspace', workspace],
54-
...<String>['-scheme', scheme],
55-
if (configuration != null) ...<String>['-configuration', configuration],
56-
...extraFlags,
57-
if (disabledSandboxEntitlementFile != null)
58-
'CODE_SIGN_ENTITLEMENTS=${disabledSandboxEntitlementFile.path}',
59-
];
60-
final String completeTestCommand = '$_xcRunCommand ${args.join(' ')}';
61-
if (log) {
62-
print(completeTestCommand);
44+
required Platform hostPlatform,
45+
}) async {
46+
final FileSystem fileSystem = exampleDirectory.fileSystem;
47+
String? resultBundlePath;
48+
final Directory? logsDirectory = ciLogsDirectory(hostPlatform, fileSystem);
49+
Directory? resultBundleTemp;
50+
try {
51+
if (logsDirectory != null) {
52+
resultBundleTemp =
53+
fileSystem.systemTempDirectory.createTempSync('flutter_xcresult.');
54+
resultBundlePath = resultBundleTemp.childDirectory('result').path;
55+
}
56+
File? disabledSandboxEntitlementFile;
57+
if (actions.contains('test') && targetPlatform.toLowerCase() == 'macos') {
58+
disabledSandboxEntitlementFile = _createDisabledSandboxEntitlementFile(
59+
exampleDirectory.childDirectory(targetPlatform.toLowerCase()),
60+
configuration ?? 'Debug',
61+
);
62+
}
63+
final List<String> args = <String>[
64+
_xcodeBuildCommand,
65+
...actions,
66+
...<String>['-workspace', workspace],
67+
...<String>['-scheme', scheme],
68+
if (resultBundlePath != null) ...<String>[
69+
'-resultBundlePath',
70+
resultBundlePath
71+
],
72+
if (configuration != null) ...<String>['-configuration', configuration],
73+
...extraFlags,
74+
if (disabledSandboxEntitlementFile != null)
75+
'CODE_SIGN_ENTITLEMENTS=${disabledSandboxEntitlementFile.path}',
76+
];
77+
final String completeTestCommand = '$_xcRunCommand ${args.join(' ')}';
78+
if (log) {
79+
print(completeTestCommand);
80+
}
81+
final int resultExit = await processRunner
82+
.runAndStream(_xcRunCommand, args, workingDir: exampleDirectory);
83+
84+
if (resultExit != 0 && resultBundleTemp != null) {
85+
final Directory xcresultBundle =
86+
resultBundleTemp.childDirectory('result.xcresult');
87+
if (logsDirectory != null) {
88+
if (xcresultBundle.existsSync()) {
89+
// Zip the test results to the artifacts directory for upload.
90+
final File zipPath = logsDirectory.childFile(
91+
'xcodebuild-${DateTime.now().toLocal().toIso8601String()}.zip');
92+
await processRunner.run(
93+
'zip',
94+
<String>[
95+
'-r',
96+
'-9',
97+
'-q',
98+
zipPath.path,
99+
xcresultBundle.basename,
100+
],
101+
workingDir: resultBundleTemp,
102+
);
103+
} else {
104+
print(
105+
'xcresult bundle ${xcresultBundle.path} does not exist, skipping upload');
106+
}
107+
}
108+
}
109+
return resultExit;
110+
} finally {
111+
resultBundleTemp?.deleteSync(recursive: true);
63112
}
64-
return processRunner.runAndStream(_xcRunCommand, args,
65-
workingDir: exampleDirectory);
66113
}
67114

68115
/// Returns true if [project], which should be an .xcodeproj directory,

‎script/tool/lib/src/drive_examples_command.dart‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -228,8 +228,12 @@ class DriveExamplesCommand extends PackageLoopingCommand {
228228
}
229229
for (final File driver in drivers) {
230230
final List<File> failingTargets = await _driveTests(
231-
example, driver, testTargets,
232-
deviceFlags: deviceFlags);
231+
example,
232+
driver,
233+
testTargets,
234+
deviceFlags: deviceFlags,
235+
exampleName: exampleName,
236+
);
233237
for (final File failingTarget in failingTargets) {
234238
errors.add(
235239
getRelativePosixPath(failingTarget, from: package.directory));
@@ -376,10 +380,16 @@ class DriveExamplesCommand extends PackageLoopingCommand {
376380
File driver,
377381
List<File> targets, {
378382
required List<String> deviceFlags,
383+
required String exampleName,
379384
}) async {
380385
final List<File> failures = <File>[];
381386

382387
final String enableExperiment = getStringArg(kEnableExperiment);
388+
final String screenshotBasename =
389+
'${exampleName.replaceAll(platform.pathSeparator, '_')}-drive';
390+
final Directory? screenshotDirectory =
391+
ciLogsDirectory(platform, driver.fileSystem)
392+
?.childDirectory(screenshotBasename);
383393

384394
for (final File target in targets) {
385395
final int exitCode = await processRunner.runAndStream(
@@ -389,6 +399,8 @@ class DriveExamplesCommand extends PackageLoopingCommand {
389399
...deviceFlags,
390400
if (enableExperiment.isNotEmpty)
391401
'--enable-experiment=$enableExperiment',
402+
if (screenshotDirectory != null)
403+
'--screenshot=${screenshotDirectory.path}',
392404
'--driver',
393405
getRelativePosixPath(driver, from: example.directory),
394406
'--target',
@@ -416,6 +428,8 @@ class DriveExamplesCommand extends PackageLoopingCommand {
416428
required List<File> testFiles,
417429
}) async {
418430
final String enableExperiment = getStringArg(kEnableExperiment);
431+
final Directory? logsDirectory =
432+
ciLogsDirectory(platform, testFiles.first.fileSystem);
419433

420434
// Workaround for https://github.com/flutter/flutter/issues/135673
421435
// Once that is fixed on stable, this logic can be removed and the command
@@ -438,6 +452,7 @@ class DriveExamplesCommand extends PackageLoopingCommand {
438452
...deviceFlags,
439453
if (enableExperiment.isNotEmpty)
440454
'--enable-experiment=$enableExperiment',
455+
if (logsDirectory != null) '--debug-logs-dir=${logsDirectory.path}',
441456
target,
442457
],
443458
workingDir: example.directory);

‎script/tool/lib/src/native_test_command.dart‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -431,7 +431,7 @@ this command.
431431
/// usually at "example/{ios,macos}/Runner.xcworkspace".
432432
Future<_PlatformResult> _runXcodeTests(
433433
RepositoryPackage plugin,
434-
String platform,
434+
String targetPlatform,
435435
_TestMode mode, {
436436
List<String> extraFlags = const <String>[],
437437
}) async {
@@ -456,7 +456,7 @@ this command.
456456
final String? targetToCheck =
457457
testTarget ?? (mode.unit ? unitTestTarget : null);
458458
final Directory xcodeProject = example.directory
459-
.childDirectory(platform.toLowerCase())
459+
.childDirectory(targetPlatform.toLowerCase())
460460
.childDirectory('Runner.xcodeproj');
461461
if (targetToCheck != null) {
462462
final bool? hasTarget =
@@ -473,16 +473,17 @@ this command.
473473
}
474474
}
475475

476-
_printRunningExampleTestsMessage(example, platform);
476+
_printRunningExampleTestsMessage(example, targetPlatform);
477477
final int exitCode = await _xcode.runXcodeBuild(
478478
example.directory,
479-
platform,
479+
targetPlatform,
480480
// Clean before testing to remove cached swiftmodules from previous
481481
// runs, which can cause conflicts.
482482
actions: <String>['clean', 'test'],
483-
workspace: '${platform.toLowerCase()}/Runner.xcworkspace',
483+
workspace: '${targetPlatform.toLowerCase()}/Runner.xcworkspace',
484484
scheme: 'Runner',
485485
configuration: 'Debug',
486+
hostPlatform: platform,
486487
extraFlags: <String>[
487488
if (testTarget != null) '-only-testing:$testTarget',
488489
...extraFlags,
@@ -494,9 +495,10 @@ this command.
494495
const int xcodebuildNoTestExitCode = 66;
495496
switch (exitCode) {
496497
case xcodebuildNoTestExitCode:
497-
_printNoExampleTestsMessage(example, platform);
498+
_printNoExampleTestsMessage(example, targetPlatform);
498499
case 0:
499-
printSuccess('Successfully ran $platform xctest for $exampleName');
500+
printSuccess(
501+
'Successfully ran $targetPlatform xctest for $exampleName');
500502
// If this is the first test, assume success until something fails.
501503
if (overallResult == RunState.skipped) {
502504
overallResult = RunState.succeeded;

‎script/tool/lib/src/xcode_analyze_command.dart‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -97,36 +97,37 @@ class XcodeAnalyzeCommand extends PackageLoopingCommand {
9797
multiplePlatformsRequested ? failures : <String>[]);
9898
}
9999

100-
/// Analyzes [plugin] for [platform], returning true if it passed analysis.
100+
/// Analyzes [plugin] for [targetPlatform], returning true if it passed analysis.
101101
Future<bool> _analyzePlugin(
102102
RepositoryPackage plugin,
103-
String platform, {
103+
String targetPlatform, {
104104
List<String> extraFlags = const <String>[],
105105
}) async {
106106
bool passing = true;
107107
for (final RepositoryPackage example in plugin.getExamples()) {
108108
// Running tests and static analyzer.
109109
final String examplePath = getRelativePosixPath(example.directory,
110110
from: plugin.directory.parent);
111-
print('Running $platform tests and analyzer for $examplePath...');
111+
print('Running $targetPlatform tests and analyzer for $examplePath...');
112112
final int exitCode = await _xcode.runXcodeBuild(
113113
example.directory,
114-
platform,
114+
targetPlatform,
115115
// Clean before analyzing to remove cached swiftmodules from previous
116116
// runs, which can cause conflicts.
117117
actions: <String>['clean', 'analyze'],
118-
workspace: '${platform.toLowerCase()}/Runner.xcworkspace',
118+
workspace: '${targetPlatform.toLowerCase()}/Runner.xcworkspace',
119119
scheme: 'Runner',
120120
configuration: 'Debug',
121+
hostPlatform: platform,
121122
extraFlags: <String>[
122123
...extraFlags,
123124
'GCC_TREAT_WARNINGS_AS_ERRORS=YES',
124125
],
125126
);
126127
if (exitCode == 0) {
127-
printSuccess('$examplePath ($platform) passed analysis.');
128+
printSuccess('$examplePath ($targetPlatform) passed analysis.');
128129
} else {
129-
printError('$examplePath ($platform) failed analysis.');
130+
printError('$examplePath ($targetPlatform) failed analysis.');
130131
passing = false;
131132
}
132133
}

‎script/tool/test/common/xcode_test.dart‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,7 @@ void main() {
165165
'ios',
166166
workspace: 'A.xcworkspace',
167167
scheme: 'AScheme',
168+
hostPlatform: MockPlatform(),
168169
);
169170

170171
expect(exitCode, 0);
@@ -193,6 +194,7 @@ void main() {
193194
workspace: 'A.xcworkspace',
194195
scheme: 'AScheme',
195196
configuration: 'Debug',
197+
hostPlatform: MockPlatform(),
196198
extraFlags: <String>['-a', '-b', 'c=d']);
197199

198200
expect(exitCode, 0);
@@ -230,6 +232,7 @@ void main() {
230232
'ios',
231233
workspace: 'A.xcworkspace',
232234
scheme: 'AScheme',
235+
hostPlatform: MockPlatform(),
233236
);
234237

235238
expect(exitCode, 1);
@@ -264,6 +267,7 @@ void main() {
264267
'macos',
265268
workspace: 'A.xcworkspace',
266269
scheme: 'AScheme',
270+
hostPlatform: MockPlatform(),
267271
actions: <String>['test'],
268272
);
269273

0 commit comments

Comments
 (0)