Integration test 2

packages/devtools_app/integration_test/README.md

@@ -0,0 +1,36 @@
+# Instructions for running a DevTools integration test
+
+## Set up ChromeDriver (one time only)
+
+1. Follow the instructions [here](https://docs.flutter.dev/cookbook/testing/integration/introduction#5b-web) to download ChromeDriver.
+
+2. Add `chromedriver` to your PATH by modifying your `.bash_profile` or `.zshrc`:
+
+```
+export PATH=${PATH}:/Users/me/folder_containing_chromedriver/
+```
+
+3. Verify you can start `chromedriver`:
+
+```
+chromedriver --port=4444
+```
+
+If you get the error "'chromedriver' cannot be opened because it is from an unidentified developer.", run the following command with your path to the `chromedriver` executable:
+
+```
+xattr -d com.apple.quarantine ~/path/to/chromedriver
+```
+
+## Running a test
+
+- To run all integration tets: `dart run integration_test/run_tests.dart`
+- To run a single integration test: `dart run integration_test/run_tests.dart --target=integration_test/test/my_test.dart`
+
+### Special flags:
+
+- `--test-app-uri`: to speed up local development, you can pass in a vm service uri from a Dart or Flutter 
+app running on your local machine. This saves the cost of spinning up a new test app for each test run. To 
+do this, pass the vm service uri using the `--test-app-uri=some-uri` run flag.
+- `--enable_experiments`: enables experiments for DevTools within the integration test environment
+- `--headless`: this will run the integration test on the 'web-server' device instead of the 'chrome' device, meaning you will not be able to see the integration test run in Chrome when running locally. 

packages/devtools_app/integration_test/run_tests.dart

@@ -0,0 +1,54 @@
+// Copyright 2022 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:collection/collection.dart';
+
+import 'test_infra/_run_test.dart';
+
+// To run this test, run the following from `devtools_app/`:
+// `dart run integration_test/run_test.dart`
+//
+// Arguments that may be passed to this command:
+// --target=<path/to/test.dart> - this will run a single test at the path
+//    provided.
+// --test-app-uri=<some vm service uri> - this will connect DevTools to the app
+//    you specify instead of spinning up a test app inside
+//    [runFlutterIntegrationTest].
+// --enable-experiments - this will run the DevTools integration tests with
+//    DevTools experiments enabled (see feature_flags.dart)
+// --headless - this will run the integration test on the 'web-server' device
+//    instead of the 'chrome' device, meaning you will not be able to see the
+//    integration test run in Chrome when running locally.
+
+void main(List<String> args) async {
+  final targetProvided =
+      args.firstWhereOrNull((arg) => arg.startsWith(TestArgs.testTargetArg)) !=
+          null;
+  if (targetProvided) {
+    // Run the single test at this path.
+    final testRunnerArgs = TestArgs(args);
+    await runFlutterIntegrationTest(testRunnerArgs);
+  } else {
+    // Ran all tests since a target test was not provided.
+    const testSuffix = '_test.dart';
+
+    // TODO(kenz): if we end up having several subdirectories under
+    // `integration_test/test`, we could allow the directory to be modified with
+    // an argument (e.g. --directory=integration_test/test/performance).
+    final testDirectory = Directory('integration_test/test');
+    final testFiles = testDirectory
+        .listSync()
+        .where((testFile) => testFile.path.endsWith(testSuffix));
+    for (final testFile in testFiles) {
+      final testTarget = testFile.path;
+      final testRunnerArgs = TestArgs([
+        ...args,
+        '${TestArgs.testTargetArg}$testTarget',
+      ]);
+      await runFlutterIntegrationTest(testRunnerArgs);
+    }
+  }
+}

packages/devtools_app/integration_test/test_infra/_run_test.dart

@@ -0,0 +1,150 @@
+// Copyright 2022 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:collection/collection.dart';
+
+import 'io_utils.dart';
+import 'test_app_driver.dart';
+
+bool _debugTestScript = false;
+
+Future<void> runFlutterIntegrationTest(
+  TestArgs testRunnerArgs, {
+  String testAppPath = 'test/test_infra/fixtures/flutter_app',
+}) async {
+  TestFlutterApp? testApp;
+  late String testAppUri;
+
+  final bool shouldCreateTestApp = testRunnerArgs.testAppUri == null;
+  if (shouldCreateTestApp) {
+    // Create the test app and start it.
+    // TODO(kenz): support running Dart CLI test apps from here too.
+    testApp = TestFlutterApp(appPath: testAppPath);
+    await testApp.start();
+    testAppUri = testApp.vmServiceUri.toString();
+  } else {
+    testAppUri = testRunnerArgs.testAppUri!;
+  }
+
+  // TODO(kenz): do we need to start chromedriver in headless mode?
+  // Start chrome driver before running the flutter integration test.
+  final chromedriver = ChromeDriver();
+  await chromedriver.start();
+
+  // Run the flutter integration test.
+  final testRunner = TestRunner();
+  await testRunner.run(
+    testRunnerArgs.testTarget,
+    enableExperiments: testRunnerArgs.enableExperiments,
+    headless: testRunnerArgs.headless,
+    testAppArguments: {
+      'service_uri': testAppUri,
+    },
+  );
+
+  if (shouldCreateTestApp) {
+    _debugLog('killing the test app');
+    await testApp?.killGracefully();
+  }
+
+  _debugLog('cancelling stream subscriptions');
+  await testRunner.cancelAllStreamSubscriptions();
+  await chromedriver.cancelAllStreamSubscriptions();
+
+  _debugLog('killing the chromedriver process');
+  chromedriver.kill();
+}
+
+class ChromeDriver with IOMixin {
+  late final Process _process;
+
+  // TODO(kenz): add error messaging if the chromedriver executable is not
+  // found. We can also consider using web installers directly in this script:
+  // https://github.com/flutter/flutter/wiki/Running-Flutter-Driver-tests-with-Web#web-installers-repo.
+  Future<void> start() async {
+    _debugLog('starting the chromedriver process');
+    _process = await Process.start(
+      'chromedriver',
+      [
+        '--port=4444',
+      ],
+    );
+    listenToProcessOutput(_process);
+  }
+
+  void kill() {
+    _process.kill();
+  }
+}
+
+class TestRunner with IOMixin {
+  Future<void> run(
+    String testTarget, {
+    bool headless = false,
+    bool enableExperiments = false,
+    Map<String, Object> testAppArguments = const <String, Object>{},
+  }) async {
+    _debugLog('starting the flutter drive process');
+    final process = await Process.start(
+      'flutter',
+      [
+        'drive',
+        '--profile',
+        '--driver=test_driver/integration_test.dart',
+        '--target=$testTarget',
+        '-d',
+        headless ? 'web-server' : 'chrome',
+        if (testAppArguments.isNotEmpty)
+          '--dart-define=test_args=${jsonEncode(testAppArguments)}',
+        if (enableExperiments) '--dart-define=enable_experiments=true',
+      ],
+    );
+    listenToProcessOutput(process);
+
+    await process.exitCode;
+    process.kill();
+    _debugLog('flutter drive process has exited');
+  }
+}
+
+void _debugLog(String log) {
+  if (_debugTestScript) {
+    print(log);
+  }
+}
+
+// TODO(https://github.com/flutter/devtools/issues/4970): use package:args to
+// parse these arguments.
+class TestArgs {
+  TestArgs(List<String> args) {
+    final argWithTestTarget =
+        args.firstWhereOrNull((arg) => arg.startsWith(testTargetArg));
+    final target = argWithTestTarget?.substring(testTargetArg.length);
+    assert(
+      target != null,
+      'Please specify a test target (e.g. --target=path/to/test.dart',
+    );
+    testTarget = target!;
+
+    final argWithTestAppUri =
+        args.firstWhereOrNull((arg) => arg.startsWith(testAppArg));
+    testAppUri = argWithTestAppUri?.substring(testAppArg.length);
+
+    enableExperiments = args.contains(enableExperimentsArg);
+    headless = args.contains(headlessArg);
+  }
+
+  static const testTargetArg = '--target=';
+  static const testAppArg = '--test-app-uri=';
+  static const enableExperimentsArg = '--enable-experiments';
+  static const headlessArg = '--headless';
+
+  late final String testTarget;
+  late final String? testAppUri;
+  late final bool enableExperiments;
+  late final bool headless;
+}