Add a build_bucket_golden_scraper tool.

testing/run_tests.py

@@ -953,6 +953,25 @@ def gather_clang_tidy_tests(build_dir):
     )
 
 
+def gather_build_bucket_golden_scraper_tests(build_dir):
+  test_dir = os.path.join(
+      BUILDROOT_DIR, 'flutter', 'tools', 'build_bucket_golden_scraper'
+  )
+  dart_tests = glob.glob('%s/test/*_test.dart' % test_dir)
+  for dart_test_file in dart_tests:
+    opts = [
+        '--disable-dart-dev',
+        dart_test_file,
+    ]
+    yield EngineExecutableTask(
+        build_dir,
+        os.path.join('dart-sdk', 'bin', 'dart'),
+        None,
+        flags=opts,
+        cwd=test_dir
+    )
+
+
 def gather_engine_repo_tools_tests(build_dir):
   test_dir = os.path.join(
       BUILDROOT_DIR, 'flutter', 'tools', 'pkg', 'engine_repo_tools'
@@ -1249,6 +1268,7 @@ def main():
     tasks += list(gather_litetest_tests(build_dir))
     tasks += list(gather_githooks_tests(build_dir))
     tasks += list(gather_clang_tidy_tests(build_dir))
+    tasks += list(gather_build_bucket_golden_scraper_tests(build_dir))
     tasks += list(gather_engine_repo_tools_tests(build_dir))
     tasks += list(gather_api_consistency_tests(build_dir))
     tasks += list(gather_path_ops_tests(build_dir))

tools/build_bucket_golden_scraper/README.md

@@ -0,0 +1,86 @@
+# `build_bucket_golden_scraper`
+
+Given logging on Flutter's CI, scrapes the log for golden file changes.
+
+```shell
+$ dart bin/main.dart <path to log file, which can be http or a file>
+
+Wrote 3 golden file changes:
+  testing/resources/performance_overlay_gold_60fps.png
+  testing/resources/performance_overlay_gold_90fps.png
+  testing/resources/performance_overlay_gold_120fps.png
+```
+
+It can also be run with `--dry-run` to just print what it _would_ do:
+
+```shell
+$ dart bin/main.dart --dry-run <path to log file, which can be http or a file>
+
+Found 3 golden file changes:
+  testing/resources/performance_overlay_gold_60fps.png
+  testing/resources/performance_overlay_gold_90fps.png
+  testing/resources/performance_overlay_gold_120fps.png
+
+Run again without --dry-run to apply these changes.
+```
+
+You're recommended to still use `git diff` to verify the changes look good.
+
+## Upgrading `git diff`
+
+By default, `git diff` is not very helpful for binary files. You can install
+[`imagemagick`](https://imagemagick.org/) and configure your local git client
+to make `git diff` show a PNG diff:
+
+```shell
+# On MacOS.
+$ brew install imagemagick
+
+# Create a comparison script.
+$ cat > ~/bin/git-imgdiff <<EOF
+#!/bin/sh
+echo "Comparing $2 and $5"
+
+# Find a temporary directory to store the diff.
+if [ -z "$TMPDIR" ]; then
+  TMPDIR=/tmp
+fi
+
+compare \
+  "$2" "$5" \
+  /tmp/git-imgdiff-diff.png
+
+# Display the diff.
+open /tmp/git-imgdiff-diff.png
+EOF
+
+# Setup git.
+git config --global core.attributesfile '~/.gitattributes'
+
+# Add the following to ~/.gitattributes.
+cat >> ~/.gitattributes <<EOF
+*.png diff=imgdiff
+*.jpg diff=imgdiff
+*.gif diff=imgdiff
+EOF
+
+git config --global diff.imgdiff.command '~/bin/git-imgdiff'
+```
+
+## Motivation
+
+Due to <https://github.com/flutter/flutter/issues/53784>, on non-Linux OSes
+there is no way to get golden-file changes locally for a variety of engine
+tests.
+
+This tool, given log output from a Flutter CI run, will scrape the log for:
+
+```txt
+Golden file mismatch. Please check the difference between /b/s/w/ir/cache/builder/src/flutter/testing/resources/performance_overlay_gold_90fps.png and /b/s/w/ir/cache/builder/src/flutter/testing/resources/performance_overlay_gold_90fps_new.png, and  replace the former with the latter if the difference looks good.
+S
+See also the base64 encoded /b/s/w/ir/cache/builder/src/flutter/testing/resources/performance_overlay_gold_90fps_new.png:
+iVBORw0KGgoAAAANSUhEUgAAA+gAAAPoCAYAAABNo9TkAAAABHNCSVQICAgIfAhkiAAAIABJREFUeJzs3elzFWeeJ/rnHB3tSEILktgEBrPvYBbbUF4K24X3t (...omitted)
+```
+
+And convert the base64 encoded image into a PNG file, and overwrite the old
+golden file with the new one.

tools/build_bucket_golden_scraper/bin/main.dart

@@ -0,0 +1,20 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io' as io;
+
+import 'package:build_bucket_golden_scraper/build_bucket_golden_scraper.dart';
+
+void main(List<String> arguments) async {
+  final int result;
+  try {
+    result = await BuildBucketGoldenScraper.fromCommandLine(arguments).run();
+  } on FormatException catch (e) {
+    io.stderr.writeln(e.message);
+    io.exit(1);
+  }
+  if (result != 0) {
+    io.exit(result);
+  }
+}

tools/build_bucket_golden_scraper/lib/build_bucket_golden_scraper.dart

@@ -0,0 +1,205 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:convert';
+import 'dart:io' as io;
+
+import 'package:args/args.dart';
+import 'package:engine_repo_tools/engine_repo_tools.dart';
+import 'package:meta/meta.dart';
+import 'package:path/path.dart' as p;
+
+/// "Downloads" (i.e. decodes base64 encoded strings) goldens from buildbucket.
+///
+/// See ../README.md for motivation and usage.
+final class BuildBucketGoldenScraper {
+  /// Creates a scraper with the given configuration.
+  BuildBucketGoldenScraper({
+    required this.pathOrUrl,
+    this.dryRun = false,
+    String? engineSrcPath,
+    StringSink? outSink,
+  }) :
+      engine = engineSrcPath != null ?
+          Engine.fromSrcPath(engineSrcPath) :
+          Engine.findWithin(p.dirname(p.fromUri(io.Platform.script))),
+      _outSink = outSink ?? io.stdout;
+
+  /// Creates a scraper from the command line arguments.
+  ///
+  /// Throws [FormatException] if the arguments are invalid.
+  factory BuildBucketGoldenScraper.fromCommandLine(
+    List<String> args, {
+    StringSink? outSink,
+    StringSink? errSink,
+  }) {
+    outSink ??= io.stdout;
+    errSink ??= io.stderr;
+
+    final ArgResults argResults = _argParser.parse(args);
+    if (argResults['help'] as bool) {
+      _usage(args);
+    }
+    final String? pathOrUrl = argResults.rest.isEmpty ? null : argResults.rest.first;
+    if (pathOrUrl == null) {
+      _usage(args);
+    }
+    return BuildBucketGoldenScraper(
+      pathOrUrl: pathOrUrl,
+      dryRun: argResults['dry-run'] as bool,
+      outSink: outSink,
+      engineSrcPath: argResults['engine-src-path'] as String?,
+    );
+  }
+
+  static Never _usage(List<String> args) {
+    final StringBuffer output = StringBuffer();
+    output.writeln('Usage: build_bucket_golden_scraper [options] <path or URL>');
+    output.writeln();
+    output.writeln(_argParser.usage);
+    throw FormatException(output.toString(), args.join(' '));
+  }
+
+  static final ArgParser _argParser = ArgParser()
+    ..addFlag(
+      'help',
+      abbr: 'h',
+      help: 'Print this help message.',
+      negatable: false,
+    )
+    ..addFlag(
+      'dry-run',
+      help: "If true, don't write any files to disk (other than temporary files).",
+      negatable: false,
+    )
+    ..addOption(
+      'engine-src-path',
+      help: 'The path to the engine source code.',
+      valueHelp: 'path/that/contains/src (defaults to the directory containing this script)',
+    );
+
+  /// A local path or a URL to a buildbucket log file.
+  final String pathOrUrl;
+
+  /// If true, don't write any files to disk (other than temporary files).
+  final bool dryRun;
+
+  /// The path to the engine source code.
+  final Engine engine;
+
+  /// How to print output, typically [io.stdout].
+  final StringSink _outSink;
+
+  /// Runs the scraper.
+  Future<int> run() async {
+    // If the path is a URL, download it and store it in a temporary file.
+    final Uri? maybeUri = Uri.tryParse(pathOrUrl);
+    if (maybeUri == null) {
+      throw FormatException('Invalid path or URL: $pathOrUrl');
+    }
+
+    final String contents;
+    if (maybeUri.hasScheme) {
+      contents = await _downloadFile(maybeUri);
+    } else {
+      final io.File readFile = io.File(pathOrUrl);
+      if (!readFile.existsSync()) {
+        throw FormatException('File does not exist: $pathOrUrl');
+      }
+      contents = readFile.readAsStringSync();
+    }
+
+    // Check that it is a buildbucket log file.
+    if (!contents.contains(_buildBucketMagicString)) {
+      throw FormatException('Not a buildbucket log file: $pathOrUrl');
+    }
+
+    // Check for occurences of a base64 encoded string.
+    //
+    // The format looks something like this:
+    // [LINE N+0]: See also the base64 encoded /b/s/w/ir/cache/builder/src/flutter/testing/resources/performance_overlay_gold_120fps_new.png:
+    // [LINE N+1]: {{BASE_64_ENCODED_IMAGE}}
+    //
+    // We want to extract the file name (relative to the engine root) and then
+    // decode the base64 encoded string (and write it to disk if we are not in
+    // dry-run mode).
+    final List<_Golden> goldens = <_Golden>[];
+    final List<String> lines = contents.split('\n');
+    for (int i = 0; i < lines.length; i++) {
+      final String line = lines[i];
+      if (line.startsWith(_base64MagicString)) {
+        final String relativePath = line.split(_buildBucketMagicString).last.split(':').first;
+
+        // Remove the _new suffix from the file name.
+        final String pathWithouNew = relativePath.replaceAll('_new', '');
+
+        final String base64EncodedString = lines[i + 1];
+        final List<int> bytes = base64Decode(base64EncodedString);
+        final io.File outFile = io.File(p.join(engine.srcDir.path, pathWithouNew));
+        goldens.add(_Golden(outFile, bytes));
+      }
+    }
+
+    if (goldens.isEmpty) {
+      _outSink.writeln('No goldens found.');
+      return 0;
+    }
+
+    // Sort and de-duplicate the goldens.
+    goldens.sort();
+    final Set<_Golden> uniqueGoldens = goldens.toSet();
+
+    // Write the goldens to disk (or pretend to in dry-run mode).
+    _outSink.writeln('${dryRun ? 'Found' : 'Wrote'} ${uniqueGoldens.length} golden file changes:');
+    for (final _Golden golden in uniqueGoldens) {
+      final String truncatedPathAfterFlutterDir = golden.outFile.path.split('flutter${p.separator}').last;
+      _outSink.writeln('  $truncatedPathAfterFlutterDir');
+      if (!dryRun) {
+        await golden.outFile.writeAsBytes(golden.bytes);
+      }
+    }
+    if (dryRun) {
+      _outSink.writeln('Run again without --dry-run to apply these changes.');
+    }
+
+    return 0;
+  }
+
+  static const String _buildBucketMagicString = '/b/s/w/ir/cache/builder/src/';
+  static const String _base64MagicString = 'See also the base64 encoded $_buildBucketMagicString';
+
+  static Future<String> _downloadFile(Uri uri) async {
+    final io.HttpClient client = io.HttpClient();
+    final io.HttpClientRequest request = await client.getUrl(uri);
+    final io.HttpClientResponse response = await request.close();
+    final StringBuffer contents = StringBuffer();
+    await response.transform(utf8.decoder).forEach(contents.write);
+    client.close();
+    return contents.toString();
+  }
+}
+
+@immutable
+final class _Golden implements Comparable<_Golden> {
+  const _Golden(this.outFile, this.bytes);
+
+  /// Where to write the golden file.
+  final io.File outFile;
+
+  /// The bytes of the golden file to write.
+  final List<int> bytes;
+
+  @override
+  int get hashCode => outFile.path.hashCode;
+
+  @override
+  bool operator ==(Object other) {
+    return other is _Golden && other.outFile.path == outFile.path;
+  }
+
+  @override
+  int compareTo(_Golden other) {
+    return outFile.path.compareTo(other.outFile.path);
+  }
+}

tools/build_bucket_golden_scraper/pubspec.yaml

@@ -0,0 +1,47 @@
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+name: build_bucket_golden_scraper
+publish_to: none
+environment:
+  sdk: ^3.0.0
+
+# Do not add any dependencies that require more than what is provided in
+# //third_party/pkg, //third_party/dart/pkg, or
+# //third_party/dart/third_party/pkg. In particular, package:test is not usable
+# here.
+
+# If you do add packages here, make sure you can run `pub get --offline`, and
+# check the .packages and .package_config to make sure all the paths are
+# relative to this directory into //third_party/dart
+
+dependencies:
+  args: any
+  engine_repo_tools: any
+  meta: any
+  path: any
+
+dev_dependencies:
+  async_helper: any
+  expect: any
+  litetest: any
+  smith: any
+
+dependency_overrides:
+  async_helper:
+    path: ../../../third_party/dart/pkg/async_helper
+  args:
+    path: ../../../third_party/dart/third_party/pkg/args
+  engine_repo_tools:
+    path: ../pkg/engine_repo_tools
+  expect:
+    path: ../../../third_party/dart/pkg/expect
+  litetest:
+    path: ../../testing/litetest
+  meta:
+    path: ../../../third_party/dart/pkg/meta
+  path:
+    path: ../../../third_party/dart/third_party/pkg/path
+  smith:
+    path: ../../../third_party/dart/pkg/smith