Merge branch 'flutter:main' into main

app_dart/lib/cocoon_service.dart

@@ -44,6 +44,7 @@ export 'src/service/buildbucket.dart';
 export 'src/service/branch_service.dart';
 export 'src/service/cache_service.dart';
 export 'src/service/config.dart';
+export 'src/service/firestore.dart';
 export 'src/service/gerrit_service.dart';
 export 'src/service/github_checks_service.dart';
 export 'src/service/luci_build_service.dart';

app_dart/lib/src/service/access_client_provider.dart

@@ -9,7 +9,32 @@ class AccessClientProvider {
   /// Returns an OAuth 2.0 authenticated access client for the device lab service account.
   Future<Client> createAccessClient({
     List<String> scopes = const <String>['https://www.googleapis.com/auth/cloud-platform'],
+    Client? baseClient,
   }) async {
-    return clientViaApplicationDefaultCredentials(scopes: scopes);
+    return clientViaApplicationDefaultCredentials(scopes: scopes, baseClient: baseClient);
+  }
+}
+
+/// Creates a Firestore base client for none (default) database.
+///
+/// A default header is required for none (default) Firestore API calls.
+/// Both `project_id` and `database_id` are required.
+///
+/// https://firebase.google.com/docs/firestore/manage-databases#access_a_named_database_with_a_client_library
+class FirestoreBaseClient extends BaseClient {
+  FirestoreBaseClient({
+    required this.projectId,
+    required this.databaseId,
+  });
+  final String databaseId;
+  final String projectId;
+  final Client client = Client();
+  @override
+  Future<StreamedResponse> send(BaseRequest request) {
+    final Map<String, String> defaultHeaders = <String, String>{
+      'x-goog-request-params': 'project_id=$projectId&database_id=$databaseId',
+    };
+    request.headers.addAll(defaultHeaders);
+    return client.send(request);
   }
 }

app_dart/lib/src/service/config.dart

@@ -93,6 +93,12 @@ class Config {
   /// Default properties when rerunning a prod build.
   static const Map<String, Object> defaultProperties = <String, Object>{'force_upload': true};
 
+  /// GCP project ID.
+  static const String flutterGcpProjectId = 'flutter-dashboard';
+
+  // GCP Firestore native database ID.
+  static const String flutterGcpFirestoreDatabase = 'cocoon';
+
   @visibleForTesting
   static const Duration configCacheTtl = Duration(hours: 12);
 
@@ -413,6 +419,11 @@ class Config {
     );
   }
 
+  Future<FirestoreService> createFirestoreService() async {
+    final AccessClientProvider accessClientProvider = AccessClientProvider();
+    return FirestoreService(accessClientProvider);
+  }
+
   Future<BigqueryService> createBigQueryService() async {
     final AccessClientProvider accessClientProvider = AccessClientProvider();
     return BigqueryService(accessClientProvider);

app_dart/lib/src/service/firestore.dart

@@ -0,0 +1,98 @@
+// Copyright 2020 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:googleapis/firestore/v1.dart';
+import 'package:http/http.dart';
+
+import '../model/appengine/commit.dart';
+import '../model/appengine/task.dart';
+import '../model/ci_yaml/target.dart';
+import 'access_client_provider.dart';
+import 'config.dart';
+
+const String kDatabase = 'projects/${Config.flutterGcpProjectId}/databases/${Config.flutterGcpFirestoreDatabase}';
+
+class FirestoreService {
+  const FirestoreService(this.accessClientProvider);
+
+  /// AccessClientProvider for OAuth 2.0 authenticated access client
+  final AccessClientProvider accessClientProvider;
+
+  /// Return a [ProjectsDatabasesDocumentsResource] with an authenticated [client]
+  Future<ProjectsDatabasesDocumentsResource> documentResource() async {
+    final Client client = await accessClientProvider.createAccessClient(
+      scopes: const <String>[FirestoreApi.datastoreScope],
+      baseClient: FirestoreBaseClient(
+        projectId: Config.flutterGcpProjectId,
+        databaseId: Config.flutterGcpFirestoreDatabase,
+      ),
+    );
+    return FirestoreApi(client).projects.databases.documents;
+  }
+
+  /// Writes [writes] to Firestore within a transaction.
+  ///
+  /// This is an atomic operation: either all writes succeed or all writes fail.
+  Future<CommitResponse> writeViaTransaction(List<Write> writes) async {
+    final ProjectsDatabasesDocumentsResource databasesDocumentsResource = await documentResource();
+    final BeginTransactionRequest beginTransactionRequest =
+        BeginTransactionRequest(options: TransactionOptions(readWrite: ReadWrite()));
+    final BeginTransactionResponse beginTransactionResponse =
+        await databasesDocumentsResource.beginTransaction(beginTransactionRequest, kDatabase);
+    final CommitRequest commitRequest =
+        CommitRequest(transaction: beginTransactionResponse.transaction, writes: writes);
+    return databasesDocumentsResource.commit(commitRequest, kDatabase);
+  }
+}
+
+/// Generates task documents based on targets.
+List<Document> targetsToTaskDocuments(Commit commit, List<Target> targets) {
+  final Iterable<Document> iterableDocuments = targets.map(
+    (Target target) => Document(
+      name: '$kDatabase/documents/tasks/${commit.sha}_${target.value.name}_1',
+      fields: <String, Value>{
+        'builderNumber': Value(integerValue: null),
+        'createTimestamp': Value(integerValue: commit.timestamp!.toString()),
+        'endTimestamp': Value(integerValue: '0'),
+        'bringup': Value(booleanValue: target.value.bringup),
+        'name': Value(stringValue: target.value.name.toString()),
+        'startTimestamp': Value(integerValue: '0'),
+        'status': Value(stringValue: Task.statusNew),
+        'testFlaky': Value(booleanValue: false),
+        'commitSha': Value(stringValue: commit.sha),
+      },
+    ),
+  );
+  return iterableDocuments.toList();
+}
+
+/// Generates commit document based on datastore commit data model.
+Document commitToCommitDocument(Commit commit) {
+  return Document(
+    name: '$kDatabase/documents/commits/${commit.sha}',
+    fields: <String, Value>{
+      'avatar': Value(stringValue: commit.authorAvatarUrl),
+      'branch': Value(stringValue: commit.branch),
+      'createTimestamp': Value(integerValue: commit.timestamp.toString()),
+      'author': Value(stringValue: commit.author),
+      'message': Value(stringValue: commit.message),
+      'repositoryPath': Value(stringValue: commit.repository),
+      'sha': Value(stringValue: commit.sha),
+    },
+  );
+}
+
+/// Creates a list of [Write] based on documents.
+List<Write> documentsToWrites(List<Document> documents) {
+  return documents
+      .map(
+        (Document document) => Write(
+          update: document,
+          currentDocument: Precondition(exists: false),
+        ),
+      )
+      .toList();
+}

app_dart/lib/src/service/scheduler.dart

@@ -13,6 +13,7 @@ import 'package:github/github.dart' as github;
 import 'package:github/github.dart';
 import 'package:github/hooks.dart';
 import 'package:googleapis/bigquery/v2.dart';
+import 'package:googleapis/firestore/v1.dart';
 import 'package:retry/retry.dart';
 import 'package:truncate/truncate.dart';
 import 'package:yaml/yaml.dart';
@@ -31,6 +32,7 @@ import '../service/logging.dart';
 import 'cache_service.dart';
 import 'config.dart';
 import 'datastore.dart';
+import 'firestore.dart';
 import 'github_checks_service.dart';
 import 'github_service.dart';
 import 'luci_build_service.dart';
@@ -163,6 +165,16 @@ class Scheduler {
 
     await _batchScheduleBuilds(commit, toBeScheduled);
     await _uploadToBigQuery(commit);
+    final Document commitDocument = commitToCommitDocument(commit);
+    final List<Document> taskDocuments = targetsToTaskDocuments(commit, initialTargets);
+    final List<Write> writes = documentsToWrites([...taskDocuments, commitDocument]);
+    final FirestoreService firestoreService = await config.createFirestoreService();
+    // TODO(keyonghan): remove try catch logic after validated to work.
+    try {
+      await firestoreService.writeViaTransaction(writes);
+    } catch (error) {
+      log.warning('Failed to add to Firestore: $error');
+    }
   }
 
   /// Schedule all builds in batch requests instead of a single request.

app_dart/test/request_handlers/github/webhook_subscription_test.dart

@@ -38,6 +38,7 @@ void main() {
   late FakeGerritService gerritService;
   late MockCommitService commitService;
   late MockGitHub gitHubClient;
+  late MockFirestoreService mockFirestoreService;
   late MockGithubChecksUtil mockGithubChecksUtil;
   late MockGithubChecksService mockGithubChecksService;
   late MockIssuesService issuesService;
@@ -53,6 +54,7 @@ void main() {
   setUp(() {
     request = FakeHttpRequest();
     db = FakeDatastoreDB();
+    mockFirestoreService = MockFirestoreService();
     gitHubClient = MockGitHub();
     githubService = FakeGithubService();
     commitService = MockCommitService();
@@ -62,6 +64,7 @@ void main() {
       dbValue: db,
       githubClient: gitHubClient,
       githubService: githubService,
+      firestoreService: mockFirestoreService,
       githubOAuthTokenValue: 'githubOAuthKey',
       missingTestsPullRequestMessageValue: 'missingTestPullRequestMessage',
       releaseBranchPullRequestMessageValue: 'releaseBranchPullRequestMessage',

app_dart/test/request_handlers/vacuum_github_commits_test.dart

@@ -29,6 +29,7 @@ void main() {
     late FakeDatastoreDB db;
     FakeScheduler scheduler;
     late ApiRequestHandlerTester tester;
+    late MockFirestoreService mockFirestoreService;
     late VacuumGithubCommits handler;
 
     late List<String> githubCommits;
@@ -69,6 +70,7 @@ void main() {
       final MockRepositoriesService repositories = MockRepositoriesService();
       final FakeGithubService githubService = FakeGithubService();
       final MockTabledataResource tabledataResourceApi = MockTabledataResource();
+      mockFirestoreService = MockFirestoreService();
       when(tabledataResourceApi.insertAll(any, any, any, any)).thenAnswer((_) async {
         return TableDataInsertAllResponse();
       });
@@ -78,6 +80,7 @@ void main() {
       config = FakeConfig(
         tabledataResource: tabledataResourceApi,
         githubService: githubService,
+        firestoreService: mockFirestoreService,
         dbValue: db,
         supportedBranchesValue: <String>[
           'master',

app_dart/test/service/firestore_test.dart

@@ -0,0 +1,59 @@
+// Copyright 2020 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:cocoon_service/src/model/appengine/commit.dart';
+import 'package:cocoon_service/src/model/appengine/task.dart';
+import 'package:cocoon_service/src/model/ci_yaml/target.dart';
+import 'package:cocoon_service/src/service/firestore.dart';
+
+import 'package:googleapis/firestore/v1.dart';
+import 'package:test/test.dart';
+
+import '../src/utilities/entity_generators.dart';
+
+void main() {
+  test('creates task documents correctly from targets', () async {
+    final Commit commit = generateCommit(1);
+    final List<Target> targets = <Target>[
+      generateTarget(1, platform: 'Mac'),
+      generateTarget(2, platform: 'Linux'),
+    ];
+    final List<Document> taskDocuments = targetsToTaskDocuments(commit, targets);
+    expect(taskDocuments.length, 2);
+    expect(taskDocuments[0].name, '$kDatabase/documents/tasks/${commit.sha}_${targets[0].value.name}_1');
+    expect(taskDocuments[0].fields!['builderNumber']!.integerValue, null);
+    expect(taskDocuments[0].fields!['createTimestamp']!.integerValue, commit.timestamp.toString());
+    expect(taskDocuments[0].fields!['endTimestamp']!.integerValue, '0');
+    expect(taskDocuments[0].fields!['bringup']!.booleanValue, false);
+    expect(taskDocuments[0].fields!['name']!.stringValue, targets[0].value.name);
+    expect(taskDocuments[0].fields!['startTimestamp']!.integerValue, '0');
+    expect(taskDocuments[0].fields!['status']!.stringValue, Task.statusNew);
+    expect(taskDocuments[0].fields!['testFlaky']!.booleanValue, false);
+    expect(taskDocuments[0].fields!['commitSha']!.stringValue, commit.sha);
+  });
+
+  test('creates commit document correctly from commit data model', () async {
+    final Commit commit = generateCommit(1);
+    final Document commitDocument = commitToCommitDocument(commit);
+    expect(commitDocument.name, '$kDatabase/documents/commits/${commit.sha}');
+    expect(commitDocument.fields!['avatar']!.stringValue, commit.authorAvatarUrl);
+    expect(commitDocument.fields!['branch']!.stringValue, commit.branch);
+    expect(commitDocument.fields!['createTimestamp']!.integerValue, commit.timestamp.toString());
+    expect(commitDocument.fields!['author']!.stringValue, commit.author);
+    expect(commitDocument.fields!['message']!.stringValue, commit.message);
+    expect(commitDocument.fields!['repositoryPath']!.stringValue, commit.repository);
+    expect(commitDocument.fields!['sha']!.stringValue, commit.sha);
+  });
+
+  test('creates writes correctly from documents', () async {
+    final List<Document> documents = <Document>[
+      Document(name: 'd1', fields: <String, Value>{'key1': Value(stringValue: 'value1')}),
+      Document(name: 'd2', fields: <String, Value>{'key1': Value(stringValue: 'value2')}),
+    ];
+    final List<Write> writes = documentsToWrites(documents);
+    expect(writes.length, documents.length);
+    expect(writes[0].update, documents[0]);
+    expect(writes[0].currentDocument!.exists, false);
+  });
+}