firebase_test_lab_command.dart

// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:io' as io;

import 'package:file/file.dart';
import 'package:uuid/uuid.dart';

import 'common/core.dart';
import 'common/gradle.dart';
import 'common/output_utils.dart';
import 'common/package_looping_command.dart';
import 'common/plugin_utils.dart';
import 'common/repository_package.dart';

const int _exitGcloudAuthFailed = 3;

/// A command to run tests via Firebase test lab.
class FirebaseTestLabCommand extends PackageLoopingCommand {
  /// Creates an instance of the test runner command.
  FirebaseTestLabCommand(
    super.packagesDir, {
    super.processRunner,
    super.platform,
  }) {
    argParser.addOption(
      _gCloudProjectArg,
      help: 'The Firebase project name.',
    );
    argParser.addOption(_gCloudServiceKeyArg,
        help: 'The path to the service key for gcloud authentication.\n'
            'If not provided, setup will be skipped, so testing will fail '
            'unless gcloud is already configured.');
    argParser.addOption('test-run-id',
        defaultsTo: const Uuid().v4(),
        help:
            'Optional string to append to the results path, to avoid conflicts. '
            'Randomly chosen on each invocation if none is provided. '
            'The default shown here is just an example.');
    argParser.addOption('build-id',
        defaultsTo:
            io.Platform.environment['CIRRUS_BUILD_ID'] ?? 'unknown_build',
        help:
            'Optional string to append to the results path, to avoid conflicts. '
            r'Defaults to $CIRRUS_BUILD_ID if that is set.');
    argParser.addMultiOption('device',
        splitCommas: false,
        defaultsTo: <String>[
          'model=walleye,version=26',
          'model=redfin,version=30'
        ],
        help:
            'Device model(s) to test. See https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run for more info');
    argParser.addOption(_gCloudResultsBucketArg, mandatory: true);
    argParser.addOption(
      kEnableExperiment,
      defaultsTo: '',
      help: 'Enables the given Dart SDK experiments.',
    );
  }

  static const String _gCloudServiceKeyArg = 'service-key';
  static const String _gCloudProjectArg = 'project';
  static const String _gCloudResultsBucketArg = 'results-bucket';

  @override
  final String name = 'firebase-test-lab';

  @override
  final String description = 'Runs the instrumentation tests of the example '
      'apps on Firebase Test Lab.\n\n'
      'Runs tests in test_instrumentation folder using the '
      'instrumentation_test package.';

  bool _firebaseProjectConfigured = false;

  Future<void> _configureFirebaseProject() async {
    if (_firebaseProjectConfigured) {
      return;
    }

    final String serviceKey = getStringArg(_gCloudServiceKeyArg);
    if (serviceKey.isEmpty) {
      print(
          'No --$_gCloudServiceKeyArg provided; skipping gcloud authorization');
    } else {
      final io.ProcessResult result = await processRunner.run(
        'gcloud',
        <String>[
          'auth',
          'activate-service-account',
          '--key-file=$serviceKey',
        ],
        logOnError: true,
      );
      if (result.exitCode != 0) {
        printError('Unable to activate gcloud account.');
        throw ToolExit(_exitGcloudAuthFailed);
      }
    }
    final String project = getStringArg(_gCloudProjectArg);
    if (project.isEmpty) {
      print('No --$_gCloudProjectArg provided; skipping gcloud config');
    } else {
      final int exitCode = await processRunner.runAndStream('gcloud', <String>[
        'config',
        'set',
        'project',
        project,
      ]);
      print('');
      if (exitCode == 0) {
        print('Firebase project configured.');
      } else {
        logWarning(
            'Warning: gcloud config set returned a non-zero exit code. Continuing anyway.');
      }
    }
    _firebaseProjectConfigured = true;
  }

  @override
  Future<PackageResult> runForPackage(RepositoryPackage package) async {
    final List<PackageResult> results = <PackageResult>[];
    for (final RepositoryPackage example in package.getExamples()) {
      results.add(await _runForExample(example, package: package));
    }

    // If all results skipped, report skip overall.
    if (results
        .every((PackageResult result) => result.state == RunState.skipped)) {
      return PackageResult.skip('No examples support Android.');
    }
    // Otherwise, report failure if there were any failures.
    final List<String> allErrors = results
        .map((PackageResult result) =>
            result.state == RunState.failed ? result.details : <String>[])
        .expand((List<String> list) => list)
        .toList();
    return allErrors.isEmpty
        ? PackageResult.success()
        : PackageResult.fail(allErrors);
  }

  /// Runs the test for the given example of [package].
  Future<PackageResult> _runForExample(
    RepositoryPackage example, {
    required RepositoryPackage package,
  }) async {
    final Directory androidDirectory =
        example.platformDirectory(FlutterPlatform.android);
    if (!androidDirectory.existsSync()) {
      return PackageResult.skip(
          '${example.displayName} does not support Android.');
    }

    final Directory uiTestDirectory = androidDirectory
        .childDirectory('app')
        .childDirectory('src')
        .childDirectory('androidTest');
    if (!uiTestDirectory.existsSync()) {
      printError('No androidTest directory found.');
      if (isFlutterPlugin(package)) {
        return PackageResult.fail(
            <String>['No tests ran (use --exclude if this is intentional).']);
      } else {
        return PackageResult.skip(
            '${example.displayName} has no native Android tests.');
      }
    }

    // Ensure that the Dart integration tests will be run, not just native UI
    // tests.
    if (!await _testsContainDartIntegrationTestRunner(uiTestDirectory)) {
      printError('No integration_test runner found. '
          'See the integration_test package README for setup instructions.');
      return PackageResult.fail(<String>['No integration_test runner.']);
    }

    // Ensures that gradle wrapper exists
    final GradleProject project = GradleProject(example,
        processRunner: processRunner, platform: platform);
    if (!await _ensureGradleWrapperExists(project)) {
      return PackageResult.fail(<String>['Unable to build example apk']);
    }

    await _configureFirebaseProject();

    if (!await _runGradle(project, 'app:assembleAndroidTest')) {
      return PackageResult.fail(<String>['Unable to assemble androidTest']);
    }

    final List<String> errors = <String>[];

    // Used within the loop to ensure a unique GCS output location for each
    // test file's run.
    int resultsCounter = 0;
    for (final File test in _findIntegrationTestFiles(example)) {
      final String testName =
          getRelativePosixPath(test, from: package.directory);
      print('Testing $testName...');
      if (!await _runGradle(project, 'app:assembleDebug', testFile: test)) {
        printError('Could not build $testName');
        errors.add('$testName failed to build');
        continue;
      }
      final String buildId = getStringArg('build-id');
      final String testRunId = getStringArg('test-run-id');
      final String resultsDir =
          'plugins_android_test/${package.displayName}/$buildId/$testRunId/'
          '${example.directory.basename}/${resultsCounter++}/';

      // Automatically retry failures; there is significant flake with these
      // tests whose cause isn't yet understood, and having to re-run the
      // entire shard for a flake in any one test is extremely slow. This should
      // be removed once the root cause of the flake is understood.
      // See https://github.com/flutter/flutter/issues/95063
      const int maxRetries = 2;
      bool passing = false;
      for (int i = 1; i <= maxRetries && !passing; ++i) {
        if (i > 1) {
          logWarning('$testName failed on attempt ${i - 1}. Retrying...');
        }
        passing = await _runFirebaseTest(example, test, resultsDir: resultsDir);
      }
      if (!passing) {
        printError('Test failure for $testName after $maxRetries attempts');
        errors.add('$testName failed tests');
      }
    }

    if (errors.isEmpty && resultsCounter == 0) {
      printError('No integration tests were run.');
      errors.add('No tests ran (use --exclude if this is intentional).');
    }

    return errors.isEmpty
        ? PackageResult.success()
        : PackageResult.fail(errors);
  }

  /// Checks that Gradle has been configured for [project], and if not runs a
  /// Flutter build to generate it.
  ///
  /// Returns true if either gradlew was already present, or the build succeeds.
  Future<bool> _ensureGradleWrapperExists(GradleProject project) async {
    if (!project.isConfigured()) {
      print('Running flutter build apk...');
      final String experiment = getStringArg(kEnableExperiment);
      final int exitCode = await processRunner.runAndStream(
          flutterCommand,
          <String>[
            'build',
            'apk',
            '--config-only',
            if (experiment.isNotEmpty) '--enable-experiment=$experiment',
          ],
          workingDir: project.androidDirectory);

      if (exitCode != 0) {
        return false;
      }
    }
    return true;
  }

  /// Runs [test] from [example] as a Firebase Test Lab test, returning true if
  /// the test passed.
  ///
  /// [resultsDir] should be a unique-to-the-test-run directory to store the
  /// results on the server.
  Future<bool> _runFirebaseTest(
    RepositoryPackage example,
    File test, {
    required String resultsDir,
  }) async {
    final List<String> args = <String>[
      'firebase',
      'test',
      'android',
      'run',
      '--type',
      'instrumentation',
      '--app',
      'build/app/outputs/apk/debug/app-debug.apk',
      '--test',
      'build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk',
      '--timeout',
      '7m',
      '--results-bucket=gs://${getStringArg(_gCloudResultsBucketArg)}',
      '--results-dir=$resultsDir',
      for (final String device in getStringListArg('device')) ...<String>[
        '--device',
        device
      ],
    ];
    final int exitCode = await processRunner.runAndStream('gcloud', args,
        workingDir: example.directory);

    return exitCode == 0;
  }

  /// Builds [target] using Gradle in the given [project]. Assumes Gradle is
  /// already configured.
  ///
  /// [testFile] optionally does the Flutter build with the given test file as
  /// the build target.
  ///
  /// Returns true if the command succeeds.
  Future<bool> _runGradle(
    GradleProject project,
    String target, {
    File? testFile,
  }) async {
    final String experiment = getStringArg(kEnableExperiment);
    final String? extraOptions = experiment.isNotEmpty
        ? Uri.encodeComponent('--enable-experiment=$experiment')
        : null;

    final int exitCode = await project.runCommand(
      target,
      arguments: <String>[
        '-Pverbose=true',
        if (testFile != null) '-Ptarget=${testFile.path}',
        if (extraOptions != null) '-Pextra-front-end-options=$extraOptions',
        if (extraOptions != null) '-Pextra-gen-snapshot-options=$extraOptions',
      ],
    );

    if (exitCode != 0) {
      return false;
    }
    return true;
  }

  /// Finds and returns all integration test files for [example].
  Iterable<File> _findIntegrationTestFiles(RepositoryPackage example) sync* {
    final Directory integrationTestDir =
        example.directory.childDirectory('integration_test');

    if (!integrationTestDir.existsSync()) {
      return;
    }

    yield* integrationTestDir
        .listSync(recursive: true)
        .where((FileSystemEntity file) =>
            file is File && file.basename.endsWith('_test.dart'))
        .cast<File>();
  }

  /// Returns true if any of the test files in [uiTestDirectory] contain the
  /// annotation that means that the test will reports the results of running
  /// the Dart integration tests.
  Future<bool> _testsContainDartIntegrationTestRunner(
      Directory uiTestDirectory) async {
    return uiTestDirectory
        .list(recursive: true, followLinks: false)
        .where((FileSystemEntity entity) => entity is File)
        .cast<File>()
        .any((File file) {
      if (file.basename.endsWith('.java')) {
        return file
            .readAsStringSync()
            .contains('@RunWith(FlutterTestRunner.class)');
      } else if (file.basename.endsWith('.kt')) {
        return file
            .readAsStringSync()
            .contains('@RunWith(FlutterTestRunner::class)');
      }
      return false;
    });
  }
}