Merge branch 'flutter:main' into main

.ci.yaml

@@ -12,13 +12,19 @@ platform_properties:
   linux:
     properties:
       os: Linux
+      device_type: "none"
   mac:
     properties:
       os: "Mac-12|Mac-13"
       cpu: x86
+  mac_arm64:
+    properties:
+      os: "Mac-12|Mac-13"
+      cpu: arm64
   windows:
     properties:
       os: Windows
+      device_type: "none"
 
 targets:
   - name: Linux Cocoon
@@ -59,6 +65,10 @@ targets:
     properties:
       script: cipd_packages/doxygen/tool/build.sh
       cipd_name: flutter/doxygen/linux-amd64
+      dependencies: >-
+        [
+          {"dependency": "cmake", "version": "build_id:8787856497187628321"}
+        ]
     runIf:
       - cipd_packages/doxygen/**
       - .ci.yaml
@@ -73,17 +83,56 @@ targets:
       - cipd_packages/codesign/**
       - .ci.yaml
 
-  - name: Mac arm_codesign
+  - name: Mac_arm64 codesign
     recipe: cocoon/cipd
     properties:
       script: cipd_packages/codesign/tool/build.sh
       cipd_name: flutter/codesign/mac-arm64
-      cpu: arm64
       device_type: none
     runIf:
       - cipd_packages/codesign/**
       - .ci.yaml
 
+  - name: Mac ruby
+    bringup: true
+    recipe: cocoon/cipd
+    timeout: 60
+    properties:
+      script: cipd_packages/ruby/tools/build.sh
+      cipd_name: flutter/ruby/mac-amd64
+      device_os: iOS
+      contexts: >-
+        [
+          "osx_sdk_devicelab"
+        ]
+      $flutter/osx_sdk : >-
+        {
+          "sdk_version": "14e300c"
+        }
+    runIf:
+      - cipd_packages/ruby/**
+      - .ci.yaml
+
+  - name: Mac_arm64 ruby
+    bringup: true
+    recipe: cocoon/cipd
+    timeout: 60
+    properties:
+      script: cipd_packages/ruby/tools/build.sh
+      cipd_name: flutter/ruby/mac-arm64
+      device_os: iOS
+      contexts: >-
+        [
+          "osx_sdk_devicelab"
+        ]
+      $flutter/osx_sdk : >-
+        {
+          "sdk_version": "14e300c"
+        }
+    runIf:
+      - cipd_packages/ruby/**
+      - .ci.yaml
+
   - name: Linux ci_yaml roller
     recipe: infra/ci_yaml
     properties:

CODEOWNERS

@@ -38,3 +38,4 @@ auto_submit                                                                 @ric
 cipd_packages/codesign/**                                                   @XilaiZhang
 cipd_packages/device_doctor/**                                              @yusuf-goog
 cipd_packages/doxygen/**                                                    @gspencergoog
+cipd_packages/ruby/**                                                       @godofredoc

app_dart/Dockerfile

@@ -4,7 +4,7 @@
 
 
 # Dart Docker official images can be found here: https://hub.docker.com/_/dart
-FROM dart:beta@sha256:f07d175c1d56a41b61d6ed099a6a79221ddc747eeccdbecd1e59dd324a371142
+FROM dart:beta@sha256:81eff69e362531665e108a78564e3393bc6213ba60622f3d5ea628d72d532b09
 
 WORKDIR /app
 

app_dart/integration_test/data/cocoon_config.json

@@ -1 +1 @@
-{"targets":[{"name":"Linux Cocoon","properties":{"add_recipes_cq":"true"},"recipe":"cocoon/cocoon"},{"name":"Linux device_doctor","properties":{"script":"cipd_packages/device_doctor/tool/build.sh","cipd_name":"flutter/device_doctor/linux-amd64"},"runIf":["cipd_packages/device_doctor/**",".ci.yaml"],"recipe":"cocoon/cipd"},{"name":"Mac device_doctor","properties":{"script":"cipd_packages/device_doctor/tool/build.sh","cipd_name":"flutter/device_doctor/mac-amd64","device_type":"none"},"runIf":["cipd_packages/device_doctor/**",".ci.yaml"],"recipe":"cocoon/cipd"},{"name":"Windows device_doctor","properties":{"script":"cipd_packages\\device_doctor\\tool\\build.bat","cipd_name":"flutter/device_doctor/windows-amd64"},"runIf":["cipd_packages/device_doctor/**",".ci.yaml"],"recipe":"cocoon/cipd"},{"name":"Linux doxygen","properties":{"script":"cipd_packages/doxygen/tool/build.sh","cipd_name":"flutter/doxygen/linux-amd64"},"runIf":["cipd_packages/doxygen/**",".ci.yaml"],"recipe":"cocoon/cipd"},{"name":"Mac codesign","properties":{"script":"cipd_packages/codesign/tool/build.sh","cipd_name":"flutter/codesign/mac-amd64","device_type":"none"},"runIf":["cipd_packages/codesign/**",".ci.yaml"],"recipe":"cocoon/cipd"},{"name":"Mac arm_codesign","properties":{"script":"cipd_packages/codesign/tool/build.sh","cipd_name":"flutter/codesign/mac-arm64","cpu":"arm64","device_type":"none"},"runIf":["cipd_packages/codesign/**",".ci.yaml"],"recipe":"cocoon/cipd"},{"name":"Linux ci_yaml roller","properties":{"add_recipes_cq":"true","backfill":"false"},"recipe":"infra/ci_yaml"}],"enabledBranches":["main"],"platformProperties":{"linux":{"properties":{"os":"Linux"}},"mac":{"properties":{"os":"Mac-12|Mac-13","cpu":"x86"}},"windows":{"properties":{"os":"Windows"}}}}
+{"targets":[{"name":"Linux Cocoon","properties":{"add_recipes_cq":"true"},"recipe":"cocoon/cocoon"},{"name":"Linux device_doctor","properties":{"script":"cipd_packages/device_doctor/tool/build.sh","cipd_name":"flutter/device_doctor/linux-amd64"},"runIf":["cipd_packages/device_doctor/**",".ci.yaml"],"recipe":"cocoon/cipd"},{"name":"Mac device_doctor","properties":{"script":"cipd_packages/device_doctor/tool/build.sh","cipd_name":"flutter/device_doctor/mac-amd64","device_type":"none"},"runIf":["cipd_packages/device_doctor/**",".ci.yaml"],"recipe":"cocoon/cipd"},{"name":"Windows device_doctor","properties":{"script":"cipd_packages\\device_doctor\\tool\\build.bat","cipd_name":"flutter/device_doctor/windows-amd64"},"runIf":["cipd_packages/device_doctor/**",".ci.yaml"],"recipe":"cocoon/cipd"},{"name":"Linux doxygen","properties":{"script":"cipd_packages/doxygen/tool/build.sh","cipd_name":"flutter/doxygen/linux-amd64","dependencies":"[\n  {\"dependency\": \"cmake\", \"version\": \"build_id:8787856497187628321\"}\n]"},"runIf":["cipd_packages/doxygen/**",".ci.yaml"],"recipe":"cocoon/cipd"},{"name":"Mac codesign","properties":{"script":"cipd_packages/codesign/tool/build.sh","cipd_name":"flutter/codesign/mac-amd64","device_type":"none"},"runIf":["cipd_packages/codesign/**",".ci.yaml"],"recipe":"cocoon/cipd"},{"name":"Mac_arm64 codesign","properties":{"script":"cipd_packages/codesign/tool/build.sh","cipd_name":"flutter/codesign/mac-arm64","device_type":"none"},"runIf":["cipd_packages/codesign/**",".ci.yaml"],"recipe":"cocoon/cipd"},{"name":"Mac ruby","bringup":true,"timeout":60,"properties":{"script":"cipd_packages/ruby/tools/build.sh","cipd_name":"flutter/ruby/mac-amd64","device_os":"iOS","contexts":"[\n  \"osx_sdk_devicelab\"\n]","$flutter/osx_sdk":"{\n  \"sdk_version\": \"14e300c\"\n}"},"runIf":["cipd_packages/ruby/**",".ci.yaml"],"recipe":"cocoon/cipd"},{"name":"Mac_arm64 ruby","bringup":true,"timeout":60,"properties":{"script":"cipd_packages/ruby/tools/build.sh","cipd_name":"flutter/ruby/mac-arm64","device_os":"iOS","contexts":"[\n  \"osx_sdk_devicelab\"\n]","$flutter/osx_sdk":"{\n  \"sdk_version\": \"14e300c\"\n}"},"runIf":["cipd_packages/ruby/**",".ci.yaml"],"recipe":"cocoon/cipd"},{"name":"Linux ci_yaml roller","properties":{"add_recipes_cq":"true","backfill":"false"},"recipe":"infra/ci_yaml"}],"enabledBranches":["main"],"platformProperties":{"linux":{"properties":{"os":"Linux","device_type":"none"}},"mac":{"properties":{"os":"Mac-12|Mac-13","cpu":"x86"}},"mac_arm64":{"properties":{"os":"Mac-12|Mac-13","cpu":"arm64"}},"windows":{"properties":{"os":"Windows","device_type":"none"}}}}

app_dart/lib/src/service/github_checks_service.dart

@@ -85,27 +85,38 @@ class GithubChecksService {
       );
       return false;
     }
-    final github.CheckRun checkRun = await githubChecksUtil.getCheckRun(
-      config,
-      slug,
-      buildPushMessage.userData['check_run_id'] as int?,
-    );
     github.CheckRunStatus status = statusForResult(build!.status);
+    // Only `id` and `name` in the CheckRun are needed.
+    // Instead of making an API call to get the details of each check run, we
+    // generate the check run with only necessary info.
+    final github.CheckRun checkRun = github.CheckRun.fromJson({
+      'id': buildPushMessage.userData['check_run_id'] as int?,
+      'status': status,
+      'check_suite': const {'id': null},
+      'started_at': build.createdTimestamp.toString(),
+      'conclusion': null,
+      'name': build.buildParameters!['builder_name'],
+    });
     github.CheckRunConclusion? conclusion =
         (buildPushMessage.build!.result != null) ? conclusionForResult(buildPushMessage.build!.result) : null;
-    // Do not override url for completed status.
-    final String? url = status == github.CheckRunStatus.completed ? checkRun.detailsUrl : buildPushMessage.build!.url;
+    final String? url = buildPushMessage.build!.url;
     github.CheckRunOutput? output;
     // If status has completed with failure then provide more details.
     if (taskFailed(buildPushMessage)) {
       if (rescheduled) {
         status = github.CheckRunStatus.queued;
         conclusion = null;
+        output = github.CheckRunOutput(
+          title: checkRun.name!,
+          summary: 'Note: this is an auto rerun. The timestamp above is based on the first attempt of this check run.',
+        );
       } else {
         final Build buildbucketBuild =
             await luciBuildService.getBuildById(buildPushMessage.build!.id, fields: 'id,builder,summaryMarkdown');
-        output =
-            github.CheckRunOutput(title: checkRun.name!, summary: getGithubSummary(buildbucketBuild.summaryMarkdown));
+        output = github.CheckRunOutput(
+          title: checkRun.name!,
+          summary: getGithubSummary(buildbucketBuild.summaryMarkdown),
+        );
         log.fine('Updating check run with output: [$output]');
       }
     }

app_dart/lib/src/service/scheduler.dart

@@ -142,14 +142,10 @@ class Scheduler {
         policy = GuaranteedPolicy();
       }
       final int? priority = await policy.triggerPriority(task: task, datastore: datastore);
-      // Skip scheduling `bringup: true` targets. They may be newly created and
-      // their corresponding LUCI builder configs may not be ready yet considering we
-      // disabled the `ci_yaml roller` backfill. Existing `bringup: true` targets
-      // can rely on backfiller to get tasks scheduled and executed.
-      if (priority != null && !target.value.bringup) {
+      if (_shouldSchedule(priority, target.value.bringup, policy)) {
         // Mark task as in progress to ensure it isn't scheduled over
         task.status = Task.statusInProgress;
-        toBeScheduled.add(Tuple<Target, Task, int>(target, task, priority));
+        toBeScheduled.add(Tuple<Target, Task, int>(target, task, priority!));
       }
     }
 
@@ -169,6 +165,22 @@ class Scheduler {
     await _uploadToBigQuery(commit);
   }
 
+  /// Checks whether the task should be scheduled.
+  ///
+  /// Skips scheduling `bringup: true` targets for BatchPolicy. For BatchPolicy,
+  /// targets may be newly created and their corresponding LUCI builder configs may
+  /// not be ready yet considering we disabled the `ci_yaml roller` backfill.
+  /// Existing `bringup: true` targets can rely on backfiller to get tasks scheduled and executed.
+  bool _shouldSchedule(int? priority, bool bringup, SchedulerPolicy policy) {
+    if (priority == null) {
+      return false;
+    }
+    if (bringup) {
+      return policy is! BatchPolicy;
+    }
+    return true;
+  }
+
   /// Schedule all builds in batch requests instead of a single request.
   ///
   /// Each batch request contains [Config.batchSize] builds to be scheduled.

app_dart/pubspec.yaml

@@ -17,25 +17,25 @@ dependencies:
   corsac_jwt: 1.0.0-nullsafety.1
   crypto: 3.0.3
   dbcrypt: 2.0.0
-  file: 6.1.4
+  file: 7.0.0
   fixnum: 1.1.0
   gcloud: 0.8.11
   github: 9.19.0
   googleapis: 11.4.0
   googleapis_auth: 1.4.1
-  gql: 1.0.1-alpha+1691943394579
+  gql: 1.0.1-alpha+1696717343881
   graphql: 5.2.0-beta.6
   grpc: 3.2.4
   http: 1.1.0
   json_annotation: 4.8.1
   logging: 1.2.0
-  meta: 1.10.0
+  meta: 1.11.0
   mime: 1.0.4
   mutex: 3.1.0
   neat_cache: 2.0.3
   path: 1.8.3
-  process: 4.2.4
-  process_runner: 4.1.2
+  process: 5.0.1
+  process_runner: 4.1.4
   protobuf: 2.1.0
   retry: ^3.1.2
   truncate: 3.0.1
@@ -49,7 +49,7 @@ dev_dependencies:
   json_serializable: 6.7.1
   mockito: 5.4.2
   platform: 3.1.3
-  test: 1.24.7
+  test: 1.24.8
 
 builders:
   json_serializable: 3.3.0

app_dart/test/service/github_checks_service_test.dart

@@ -50,7 +50,7 @@ void main() {
     );
     checkRun = github.CheckRun.fromJson(
       jsonDecode(
-        '{"name": "Cocoon", "id": 123, "external_id": "678", "status": "completed", "started_at": "2020-05-10T02:49:31Z", "head_sha": "the_sha", "check_suite": {"id": 456}}',
+        '{"name": "Linux Coverage", "id": 123, "external_id": "678", "status": "completed", "started_at": "2020-05-10T02:49:31Z", "head_sha": "the_sha", "check_suite": {"id": 456}}',
       ) as Map<String, dynamic>,
     );
     final Map<String, github.CheckRun> checkRuns = <String, github.CheckRun>{'Cocoon': checkRun};
@@ -113,26 +113,25 @@ void main() {
       );
       final push_message.BuildPushMessage buildPushMessage = push_message.BuildPushMessage.fromJson(
         jsonDecode(
-          buildPushMessageJsonTemplate('{\\"check_run_id\\": 1,'
+          buildPushMessageJsonTemplate('{\\"check_run_id\\": 123,'
               '\\"repo_owner\\": \\"flutter\\",'
               '\\"repo_name\\": \\"cocoon\\"}'),
         ) as Map<String, dynamic>,
       );
       await githubChecksService.updateCheckStatus(buildPushMessage, mockLuciBuildService, slug);
-      expect(
-        verify(
-          mockGithubChecksUtil.updateCheckRun(
-            any,
-            any,
-            captureAny,
-            status: anyNamed('status'),
-            conclusion: anyNamed('conclusion'),
-            detailsUrl: anyNamed('detailsUrl'),
-            output: anyNamed('output'),
-          ),
-        ).captured,
-        <github.CheckRun>[checkRun],
-      );
+      final github.CheckRun checkRunCaptured = await verify(
+        mockGithubChecksUtil.updateCheckRun(
+          any,
+          any,
+          captureAny,
+          status: anyNamed('status'),
+          conclusion: anyNamed('conclusion'),
+          detailsUrl: anyNamed('detailsUrl'),
+          output: anyNamed('output'),
+        ),
+      ).captured.first;
+      expect(checkRunCaptured.id, checkRun.id);
+      expect(checkRunCaptured.name, checkRun.name);
     });
     test('Should rerun a failed task for a roller account', () async {
       when(mockGithubChecksUtil.getCheckRun(any, any, any)).thenAnswer((_) async => checkRun);

app_dart/test/service/scheduler_test.dart

@@ -408,7 +408,7 @@ targets:
         expect(db.values.values.whereType<Task>().where((Task task) => task.status == Task.statusInProgress).length, 1);
       });
 
-      test('skip scheduling bringup true targets', () async {
+      test('skip scheduling bringup true targets for BatchPolicy', () async {
         final PullRequest mergedPr = generatePullRequest();
 
         httpClient = MockClient((http.Request request) async {
@@ -423,7 +423,6 @@ targets:
   - name: Linux B
     scheduler: cocoon
     bringup: true
-
           ''',
               200,
             );
@@ -439,6 +438,37 @@ targets:
         expect(taskB.status, Task.statusNew);
       });
 
+      test('schedule bringup true targets for GuaranteedPolicy', () async {
+        final PullRequest mergedPr = generatePullRequest(repo: 'engine', branch: 'main');
+
+        httpClient = MockClient((http.Request request) async {
+          if (request.url.path.contains('.ci.yaml')) {
+            return http.Response(
+              '''
+enabled_branches:
+  - main
+targets:
+  - name: Linux A
+    scheduler: cocoon
+  - name: Linux B
+    scheduler: cocoon
+    bringup: true
+
+          ''',
+              200,
+            );
+          }
+          throw Exception('Failed to find ${request.url.path}');
+        });
+        await scheduler.addPullRequest(mergedPr);
+        expect(db.values.values.whereType<Commit>().length, 1);
+        expect(db.values.values.whereType<Task>().length, 2);
+        final Task taskA = db.values.values.whereType<Task>().where((task) => task.name == 'Linux A').single;
+        final Task taskB = db.values.values.whereType<Task>().where((task) => task.name == 'Linux B').single;
+        expect(taskA.status, Task.statusInProgress);
+        expect(taskB.status, Task.statusInProgress);
+      });
+
       test('does not schedule tasks against non-merged PRs', () async {
         final PullRequest notMergedPr = generatePullRequest(merged: false);
         await scheduler.addPullRequest(notMergedPr);

auto_submit/Dockerfile

@@ -4,7 +4,7 @@
 
 
 # Dart Docker official images can be found here: https://hub.docker.com/_/dart
-FROM dart:beta@sha256:f07d175c1d56a41b61d6ed099a6a79221ddc747eeccdbecd1e59dd324a371142
+FROM dart:beta@sha256:81eff69e362531665e108a78564e3393bc6213ba60622f3d5ea628d72d532b09
 
 WORKDIR /app
 

auto_submit/pubspec.yaml

@@ -20,7 +20,7 @@ dependencies:
   gql: 1.0.1-alpha+1691943394579
   http: 1.1.0
   json_annotation: 4.8.1
-  meta: 1.10.0
+  meta: 1.11.0
   neat_cache: 2.0.3
   shelf: 1.4.1
   shelf_router: 1.1.4
@@ -35,7 +35,7 @@ dev_dependencies:
   json_serializable: 6.7.1
   flutter_lints: 2.0.3
   mockito: 5.4.2
-  test: 1.24.7
+  test: 1.24.8
 
 builders:
   json_serializable: 3.3.0

cipd_packages/codesign/pubspec.yaml

@@ -8,15 +8,15 @@ environment:
 
 dev_dependencies:
   lints: 2.1.1
-  test: 1.24.7
+  test: 1.24.8
 dependencies:
-  archive: 3.4.5
+  archive: 3.4.6
   args: 2.4.2
   crypto: 3.0.3
   fake_async: 1.3.1
   file: 7.0.0
   flutter_lints: 2.0.3
   logging: 1.2.0
-  meta: 1.10.0
+  meta: 1.11.0
   platform: 3.1.3
   process: 5.0.1