cocoon scheduler dart project

app_dart/integration_test/validate_all_ci_configs_test.dart

@@ -4,6 +4,7 @@
 
 import 'dart:io' as io;
 
+import 'package:cocoon_scheduler/scheduler.dart';
 import 'package:cocoon_service/cocoon_service.dart';
 import 'package:test/test.dart';
 import 'package:yaml/yaml.dart';

app_dart/lib/src/model/appengine/task.dart

@@ -2,12 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'package:cocoon_scheduler/scheduler.dart' as pb;
 import 'package:gcloud/db.dart';
 import 'package:json_annotation/json_annotation.dart';
 import 'package:meta/meta.dart';
 
 import '../../service/luci.dart';
-import '../proto/internal/scheduler.pb.dart' as pb;
 import 'commit.dart';
 import 'key_converter.dart';
 

app_dart/lib/src/model/proto/protos.dart

@@ -6,6 +6,5 @@ export 'internal/build_status_response.pb.dart';
 export 'internal/commit.pb.dart';
 export 'internal/commit_status.pb.dart';
 export 'internal/key.pb.dart';
-export 'internal/scheduler.pb.dart';
 export 'internal/stage.pb.dart';
 export 'internal/task.pb.dart';

app_dart/lib/src/service/luci.dart

@@ -6,14 +6,14 @@ import 'dart:async';
 import 'dart:math';
 
 import 'package:appengine/appengine.dart';
+import 'package:cocoon_scheduler/scheduler.dart';
 import 'package:github/github.dart';
 import 'package:json_annotation/json_annotation.dart';
 import 'package:meta/meta.dart';
 import 'package:retry/retry.dart';
 
 import '../model/appengine/task.dart';
 import '../model/luci/buildbucket.dart';
-import '../model/proto/internal/scheduler.pb.dart';
 import '../request_handling/api_request_handler.dart';
 import 'buildbucket.dart';
 import 'config.dart';

app_dart/lib/src/service/scheduler.dart

@@ -6,7 +6,7 @@ import 'dart:io';
 import 'dart:typed_data';
 
 import 'package:appengine/appengine.dart';
-import 'package:cocoon_service/src/model/luci/buildbucket.dart';
+import 'package:cocoon_scheduler/scheduler.dart';
 import 'package:gcloud/db.dart';
 import 'package:github/github.dart' as github;
 import 'package:googleapis/bigquery/v2.dart';
@@ -21,7 +21,7 @@ import '../foundation/utils.dart';
 import '../model/appengine/commit.dart';
 import '../model/appengine/task.dart';
 import '../model/github/checks.dart';
-import '../model/proto/protos.dart' show SchedulerConfig, Target;
+import '../model/luci/buildbucket.dart';
 import '../request_handling/exceptions.dart';
 import 'cache_service.dart';
 import 'config.dart';
@@ -434,57 +434,3 @@ class Scheduler {
     }
   }
 }
-
-/// Load [yamlConfig] to [SchedulerConfig] and validate the dependency graph.
-SchedulerConfig schedulerConfigFromYaml(YamlMap yamlConfig) {
-  final SchedulerConfig config = SchedulerConfig();
-  config.mergeFromProto3Json(yamlConfig);
-  _validateSchedulerConfig(config);
-
-  return config;
-}
-
-void _validateSchedulerConfig(SchedulerConfig schedulerConfig) {
-  if (schedulerConfig.targets.isEmpty) {
-    throw const FormatException('Scheduler config must have at least 1 target');
-  }
-
-  if (schedulerConfig.enabledBranches.isEmpty) {
-    throw const FormatException('Scheduler config must have at least 1 enabled branch');
-  }
-
-  final Map<String, List<Target>> targetGraph = <String, List<Target>>{};
-  final List<String> exceptions = <String>[];
-  // Construct [targetGraph]. With a one scan approach, cycles in the graph
-  // cannot exist as it only works forward.
-  for (final Target target in schedulerConfig.targets) {
-    if (targetGraph.containsKey(target.name)) {
-      exceptions.add('ERROR: ${target.name} already exists in graph');
-    } else {
-      targetGraph[target.name] = <Target>[];
-      // Add edges
-      if (target.dependencies.isNotEmpty) {
-        if (target.dependencies.length != 1) {
-          exceptions
-              .add('ERROR: ${target.name} has multiple dependencies which is not supported. Use only one dependency');
-        } else {
-          if (target.dependencies.first == target.name) {
-            exceptions.add('ERROR: ${target.name} cannot depend on itself');
-          } else if (targetGraph.containsKey(target.dependencies.first)) {
-            targetGraph[target.dependencies.first].add(target);
-          } else {
-            exceptions.add('ERROR: ${target.name} depends on ${target.dependencies.first} which does not exist');
-          }
-        }
-      }
-    }
-  }
-  _checkExceptions(exceptions);
-}
-
-void _checkExceptions(List<String> exceptions) {
-  if (exceptions.isNotEmpty) {
-    final String fullException = exceptions.reduce((String exception, _) => exception + '\n');
-    throw FormatException(fullException);
-  }
-}

app_dart/pubspec.lock

@@ -141,6 +141,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "1.0.1"
+  cocoon_scheduler:
+    dependency: "direct main"
+    description:
+      path: "../scheduler"
+      relative: true
+    source: path
+    version: "1.0.0"
   code_builder:
     dependency: transitive
     description:
@@ -695,7 +702,7 @@ packages:
     source: hosted
     version: "0.0.5"
   yaml:
-    dependency: "direct main"
+    dependency: transitive
     description:
       name: yaml
       url: "https://pub.dartlang.org"

app_dart/pubspec.yaml

@@ -12,6 +12,8 @@ environment:
 
 dependencies:
   appengine: ^0.11.0
+  cocoon_scheduler:
+    path: ../scheduler
   collection: ^1.14.11
   corsac_jwt: ^0.2.2
   crypto: ^2.0.6
@@ -31,7 +33,6 @@ dependencies:
   neat_cache: ^1.0.1
   protobuf: ^1.0.0
   truncate: ^2.1.2
-  yaml: ^2.1.16
 
 dev_dependencies:
   build_runner: ^1.0.0

app_dart/test/service/scheduler_test.dart

@@ -4,7 +4,6 @@
 
 import 'dart:convert';
 
-import 'package:cocoon_service/protos.dart' show SchedulerConfig, SchedulerSystem, Target;
 import 'package:cocoon_service/src/model/appengine/commit.dart';
 import 'package:cocoon_service/src/model/appengine/task.dart';
 import 'package:cocoon_service/src/model/github/checks.dart' as cocoon_github;
@@ -20,7 +19,6 @@ import 'package:github/github.dart';
 import 'package:googleapis/bigquery/v2.dart';
 import 'package:mockito/mockito.dart';
 import 'package:test/test.dart';
-import 'package:yaml/yaml.dart';
 
 import '../model/github/checks_test_data.dart';
 import '../src/datastore/fake_config.dart';
@@ -382,179 +380,6 @@ targets:
         expect(retryRequest.builderId.builder, 'Linux A');
       });
     });
-
-    group('scheduler config', () {
-      test('constructs graph with one target', () {
-        final YamlMap singleTargetConfig = loadYaml('''
-enabled_branches:
-  - master
-targets:
-  - name: A
-    builder: builderA
-    properties:
-      test: abc
-      ''') as YamlMap;
-        final SchedulerConfig schedulerConfig = schedulerConfigFromYaml(singleTargetConfig);
-        expect(schedulerConfig.enabledBranches, <String>['master']);
-        expect(schedulerConfig.targets.length, 1);
-        final Target target = schedulerConfig.targets.first;
-        expect(target.bringup, false);
-        expect(target.name, 'A');
-        expect(target.properties, <String, String>{
-          'test': 'abc',
-        });
-        expect(target.builder, 'builderA');
-        expect(target.scheduler, SchedulerSystem.cocoon);
-        expect(target.testbed, 'linux-vm');
-        expect(target.timeout, 30);
-      });
-
-      test('throws exception when non-existent scheduler is given', () {
-        final YamlMap targetWithNonexistentScheduler = loadYaml('''
-enabled_branches:
-  - master
-targets:
-  - name: A
-    scheduler: dashatar
-      ''') as YamlMap;
-        expect(() => schedulerConfigFromYaml(targetWithNonexistentScheduler), throwsA(isA<FormatException>()));
-      });
-
-      test('constructs graph with dependency chain', () {
-        final YamlMap dependentTargetConfig = loadYaml('''
-enabled_branches:
-  - master
-targets:
-  - name: A
-  - name: B
-    dependencies:
-      - A
-  - name: C
-    dependencies:
-      - B
-      ''') as YamlMap;
-        final SchedulerConfig schedulerConfig = schedulerConfigFromYaml(dependentTargetConfig);
-        expect(schedulerConfig.targets.length, 3);
-        final Target a = schedulerConfig.targets.first;
-        final Target b = schedulerConfig.targets[1];
-        final Target c = schedulerConfig.targets[2];
-        expect(a.name, 'A');
-        expect(b.name, 'B');
-        expect(b.dependencies, <String>['A']);
-        expect(c.name, 'C');
-        expect(c.dependencies, <String>['B']);
-      });
-
-      test('constructs graph with parent with two dependents', () {
-        final YamlMap twoDependentTargetConfig = loadYaml('''
-enabled_branches:
-  - master
-targets:
-  - name: A
-  - name: B1
-    dependencies:
-      - A
-  - name: B2
-    dependencies:
-      - A
-      ''') as YamlMap;
-        final SchedulerConfig schedulerConfig = schedulerConfigFromYaml(twoDependentTargetConfig);
-        expect(schedulerConfig.targets.length, 3);
-        final Target a = schedulerConfig.targets.first;
-        final Target b1 = schedulerConfig.targets[1];
-        final Target b2 = schedulerConfig.targets[2];
-        expect(a.name, 'A');
-        expect(b1.name, 'B1');
-        expect(b1.dependencies, <String>['A']);
-        expect(b2.name, 'B2');
-        expect(b2.dependencies, <String>['A']);
-      });
-
-      test('fails when there are cyclic targets', () {
-        final YamlMap configWithCycle = loadYaml('''
-enabled_branches:
-  - master
-targets:
-  - name: A
-    dependencies:
-      - B
-  - name: B
-    dependencies:
-      - A
-      ''') as YamlMap;
-        expect(
-            () => schedulerConfigFromYaml(configWithCycle),
-            throwsA(
-              isA<FormatException>().having(
-                (FormatException e) => e.toString(),
-                'message',
-                contains('ERROR: A depends on B which does not exist'),
-              ),
-            ));
-      });
-
-      test('fails when there are duplicate targets', () {
-        final YamlMap configWithDuplicateTargets = loadYaml('''
-enabled_branches:
-  - master
-targets:
-  - name: A
-  - name: A
-      ''') as YamlMap;
-        expect(
-            () => schedulerConfigFromYaml(configWithDuplicateTargets),
-            throwsA(
-              isA<FormatException>().having(
-                (FormatException e) => e.toString(),
-                'message',
-                contains('ERROR: A already exists in graph'),
-              ),
-            ));
-      });
-
-      test('fails when there are multiple dependencies', () {
-        final YamlMap configWithMultipleDependencies = loadYaml('''
-enabled_branches:
-  - master
-targets:
-  - name: A
-  - name: B
-  - name: C
-    dependencies:
-      - A
-      - B
-      ''') as YamlMap;
-        expect(
-            () => schedulerConfigFromYaml(configWithMultipleDependencies),
-            throwsA(
-              isA<FormatException>().having(
-                (FormatException e) => e.toString(),
-                'message',
-                contains('ERROR: C has multiple dependencies which is not supported. Use only one dependency'),
-              ),
-            ));
-      });
-
-      test('fails when dependency does not exist', () {
-        final YamlMap configWithMissingTarget = loadYaml('''
-enabled_branches:
-  - master
-targets:
-  - name: A
-    dependencies:
-      - B
-      ''') as YamlMap;
-        expect(
-            () => schedulerConfigFromYaml(configWithMissingTarget),
-            throwsA(
-              isA<FormatException>().having(
-                (FormatException e) => e.toString(),
-                'message',
-                contains('ERROR: A depends on B which does not exist'),
-              ),
-            ));
-      });
-    });
   });
 }
 

app_dart/test/src/service/fake_scheduler.dart

@@ -2,9 +2,9 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'package:cocoon_scheduler/models/scheduler.pb.dart';
 import 'package:cocoon_service/src/foundation/github_checks_util.dart';
 import 'package:cocoon_service/src/model/appengine/commit.dart';
-import 'package:cocoon_service/src/model/proto/internal/scheduler.pb.dart';
 import 'package:cocoon_service/src/service/buildbucket.dart';
 import 'package:cocoon_service/src/service/cache_service.dart';
 import 'package:cocoon_service/src/service/config.dart';

scheduler/.gitignore

@@ -0,0 +1,6 @@
+# Files and directories created by pub.
+.dart_tool/
+.packages
+
+# Conventional directory for build output.
+build/

scheduler/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 1.0.0
+
+- Initial version.

scheduler/README.md

@@ -0,0 +1,3 @@
+# Scheduler
+
+Logic for Flutter infrastructure tasks being scheduled on physical devices.