Merge pull request #1351 from dart-lang/sinc_sdk

Calculate “since” dart sdk mapping

Author: pq

pubspec.yaml

@@ -20,13 +20,10 @@ dependencies:
 dev_dependencies:
   cli_util: ^0.1.2
   dart_style: ^1.1.0
-  github:
-    git:
-      # todo(pq): update when 4.0.1 is released (https://github.com/DirectMyFile/github.dart/issues/128)
-      url: git://github.com/DirectMyFile/github.dart.git
-      ref: 0a04ae19e06f95b8afe018d666eea96585869492
+  github: ^4.0.1
   grinder: ^0.8.0
   markdown: ^2.0.0
   matcher: ^0.12.0
   path: ^1.2.0
+  pub_semver: ^1.4.2
   test: ^1.0.0

tool/crawl.dart

@@ -6,9 +6,11 @@ import 'dart:io';
 
 import 'package:analyzer/src/lint/config.dart';
 import 'package:analyzer/src/lint/registry.dart';
+import 'package:github/server.dart';
 import 'package:http/http.dart' as http;
 import 'package:linter/src/analyzer.dart';
 import 'package:linter/src/rules.dart';
+import 'package:pub_semver/pub_semver.dart';
 import 'package:yaml/yaml.dart';
 
 const _allPathSuffix = '/example/all.yaml';
@@ -48,6 +50,33 @@ Future<List<String>> get stagehandRules async =>
 Future<int> get latestMinor async =>
     _latestMinor ??= await _readLatestMinorVersion();
 
+List<String> _sdkTags;
+
+Future<List<String>> get sdkTags async => _sdkTags ??= await _fetchSdkTags();
+
+/// We don't care about SDKs previous to this bottom.
+final Version bottomDartSdk = new Version(2, 0, 0);
+
+Future<List<String>> _fetchSdkTags() {
+  var github = createGitHubClient();
+  var slug = RepositorySlug('dart-lang', 'sdk');
+
+  return github.repositories.listTags(slug).map((t) => t.name).where((t) {
+    // Filter on numeric release tags.
+    if (!t.startsWith(new RegExp(r'\d+'))) {
+      return false;
+    }
+
+    // Filter on bottom.
+    try {
+      var version = Version.parse(t);
+      return version.compareTo(bottomDartSdk) >= 0;
+    } on FormatException {
+      return false;
+    }
+  }).toList();
+}
+
 Iterable<LintRule> _registeredLints;
 
 Iterable<LintRule> get registeredLints {
@@ -58,10 +87,27 @@ Iterable<LintRule> get registeredLints {
   return _registeredLints;
 }
 
-Future<String> findSinceLinter(LintRule lint) async {
+Future<String> findSinceDartSdk(String linterVersion) async =>
+    await dartSdkForLinter(linterVersion);
+
+Future<String> dartSdkForLinter(String version) async {
+  var sdkVersions = <String>[];
+  var sdks = await sdkTags;
+  for (var sdk in sdks) {
+    var linterVersion = await linterForDartSdk(sdk);
+    if (linterVersion == version) {
+      sdkVersions.add(sdk);
+    }
+  }
+
+  sdkVersions.sort();
+  return sdkVersions.isNotEmpty ? sdkVersions.first : null;
+}
+
+Future<String> findSinceLinter(String lint) async {
   // History recorded in `all.yaml` starts in minor 31.
   var rules_31 = await rulesForVersion(31);
-  if (rules_31.contains(lint.name)) {
+  if (rules_31.contains(lint)) {
     var version = await _crawlForVersion(lint);
     if (version != null) {
       return version;
@@ -72,7 +118,7 @@ Future<String> findSinceLinter(LintRule lint) async {
   for (var minor = 31; minor <= latest; ++minor) {
     var rules = await rulesForVersion(minor);
     if (rules != null) {
-      if (rules.contains(lint.name)) {
+      if (rules.contains(lint)) {
         return '0.1.$minor';
       }
     }
@@ -87,12 +133,12 @@ Future<int> _readLatestMinorVersion() async {
   return int.parse(pubspec['version'].split('.').last);
 }
 
-Future<String> _crawlForVersion(LintRule lint) async {
+Future<String> _crawlForVersion(String lint) async {
   var client = new http.Client();
   for (int minor = 1; minor < 31; ++minor) {
     var version = '0.1.$minor';
-    var req = await client
-        .get('$_rulePathPrefix/$version/lib/src/rules/${lint.name}.dart');
+    var req =
+        await client.get('$_rulePathPrefix/$version/lib/src/rules/$lint.dart');
     if (req.statusCode == 200) {
       return version;
     }
@@ -109,6 +155,36 @@ Future<List<String>> rulesForVersion(int minor) async {
   return null;
 }
 
+Map<String, String> _dartSdkToLinterMap = <String, String>{};
+
+Future<String> linterForDartSdk(String sdk) async =>
+    _dartSdkToLinterMap[sdk] ??= await _fetchLinterForVersion(sdk);
+
+Future<String> _fetchLinterForVersion(String version) async {
+  var deps = await _fetchDEPSforVersion(version);
+  if (deps != null) {
+    for (var line in deps.split('\n')) {
+      if (line.trim().startsWith('"lint')) {
+        // "linter_tag": "0.1.59",
+        var split = line.trim().split('"linter_tag":');
+        if (split.length == 2) {
+          //  "0.1.59",
+          return split[1].split('"')[1];
+        }
+      }
+    }
+  }
+  return null;
+}
+
+Future<String> _fetchDEPSforVersion(String version) async {
+  var client = new http.Client();
+  //https://raw.githubusercontent.com/dart-lang/sdk/2.1.0-dev.1.0/DEPS
+  var req = await client
+      .get('https://raw.githubusercontent.com/dart-lang/sdk/$version/DEPS');
+  return req.body;
+}
+
 Future<LintConfig> _fetchConfig(String url) async {
   var client = new http.Client();
   var req = await client.get(url);
@@ -117,6 +193,10 @@ Future<LintConfig> _fetchConfig(String url) async {
 
 Future<List<String>> _fetchRules(String optionsUrl) async {
   var config = await _fetchConfig(optionsUrl);
+  if (config == null) {
+    print('no config found for: $optionsUrl (SKIPPED)');
+    return <String>[];
+  }
   var rules = <String>[];
   for (var ruleConfig in config.ruleConfigs) {
     rules.add(ruleConfig.name);

tool/scorecard.dart

@@ -40,12 +40,13 @@ void printAll(ScoreCard scorecard) {
 
 void printMarkdownTable(ScoreCard scorecard) {
   print(
-      '| name | since | fix | flutter user | flutter repo | pedantic | stagehand | status | bug refs |');
+      '| name | linter | dart sdk | fix | flutter user | flutter repo | pedantic | stagehand | status | bug refs |');
   print(
-      '| :--- | :--- | :---: | :---:| :---: | :---: | :---: | :---: | :--- |');
+      '| :--- | :--- | :--- | :---: | :---:| :---: | :---: | :---: | :---: | :--- |');
   scorecard.forEach((lint) {
     var sb = StringBuffer('| `${lint.name}` |');
     sb.write(' ${lint.since.sinceLinter} |');
+    sb.write(' ${lint.since.sinceDartSdk} |');
     sb.write('${lint.hasFix ? " $bulb" : ""} |');
     sb.write('${lint.ruleSets.contains('flutter') ? " $checkMark" : ""} |');
     sb.write(

tool/since.dart

@@ -4,17 +4,28 @@
 
 import 'dart:io';
 
+import 'package:pub_semver/pub_semver.dart';
 import 'package:yaml/yaml.dart';
 
 import 'crawl.dart';
 
 main() async {
-  // Uncomment to (re)generate since.yaml contents.
+// Uncomment to (re)generate since/linter.yaml contents.
 //  for (var lint in registeredLints) {
 //    var since = await findSinceLinter(lint);
 //    if (since != null) {
 //      print('${lint.name}: $since');
 //    }
+//  }
+
+// Uncomment to (re)generate since/dart_sdk.yaml contents.
+//  var tags = await sdkTags;
+//  for (var tag in sdkTags)) {
+//    var version = await fetchLinterForVersion(tag);
+//    if (version.startsWith('@')) {
+//      version = version.substring(1);
+//    }
+//    print('$tag: $version');
 //  }
 
   await sinceMap.then((m) => m.entries.forEach(print));
@@ -26,22 +37,96 @@ Future<Map<String, SinceInfo>> get sinceMap async =>
     _sinceMap ??= await _getSinceInfo();
 
 Future<Map<String, SinceInfo>> _getSinceInfo() async {
-  var cache = await new File('tool/since.yaml').readAsString();
-  YamlMap yaml = loadYamlNode(cache);
-  Map<String, SinceInfo> sinceMap = <String, SinceInfo>{};
-  for (var lint in registeredLints) {
-    var linterVersion = yaml[lint.name];
-    sinceMap[lint.name] =
-        new SinceInfo(linterVersion ?? await findSinceLinter(lint));
+  var linterCache = await new File('tool/since/linter.yaml').readAsString();
+  YamlMap linterVersionCache = loadYamlNode(linterCache);
+
+  var sinceMap = <String, SinceInfo>{};
+  for (var lint in registeredLints.map((l) => l.name)) {
+    var linterVersion = linterVersionCache[lint];
+    sinceMap[lint] = new SinceInfo(
+        sinceLinter: linterVersion ?? await findSinceLinter(lint),
+        sinceDartSdk: await _sinceSdkForLinter(linterVersion));
   }
   return sinceMap;
 }
 
+Map<String, String> _dartSdkMap;
+
+Future<Map<String, String>> get dartSdkMap async {
+  if (_dartSdkMap == null) {
+    var dartSdkCache =
+        await new File('tool/since/dart_sdk.yaml').readAsString();
+    YamlMap yamlMap = loadYamlNode(dartSdkCache);
+    _dartSdkMap =
+        yamlMap.map((k, v) => new MapEntry(k.toString(), v.toString()));
+
+    var sdks = await sdkTags;
+    for (var sdk in sdks) {
+      if (!_dartSdkMap.containsKey(sdk)) {
+        var linterVersion = await linterForDartSdk(sdk);
+        _dartSdkMap[sdk] = linterVersion;
+        print('fetched...');
+        print('$sdk : $linterVersion');
+        print('(consider caching in tool/dart_sdk.yaml)');
+      }
+    }
+  }
+  return _dartSdkMap;
+}
+
+Version earliestLinterInDart2 = Version.parse('0.1.58');
+
+Future<String> _sinceSdkForLinter(String linterVersionString) async {
+  if (linterVersionString == null) {
+    return null;
+  }
+
+  var linterVersion = Version.parse(linterVersionString);
+  if (linterVersion.compareTo(earliestLinterInDart2) < 0) {
+    return bottomDartSdk.toString();
+  }
+
+  var sdkVersions = <String>[];
+  var sdkCache = await dartSdkMap;
+  for (var sdkEntry in sdkCache.entries) {
+    if (Version.parse(sdkEntry.value) == linterVersion) {
+      sdkVersions.add(sdkEntry.key);
+    }
+  }
+  if (sdkVersions.isEmpty) {
+    var nextLinter = await _nextLinterVersion(linterVersion);
+    return _sinceSdkForLinter(nextLinter);
+  }
+
+  sdkVersions.sort();
+  return sdkVersions.first;
+}
+
+Future<String> _nextLinterVersion(Version linterVersion) async {
+  for (String version in await linterVersions) {
+    if (Version.parse(version).compareTo(linterVersion) > 0) {
+      return version;
+    }
+  }
+  return null;
+}
+
+List<String> _linterVersions;
+Future<List<String>> get linterVersions async {
+  if (_linterVersions == null) {
+    _linterVersions = <String>[];
+    for (var minor = 0; minor <= await latestMinor; ++minor) {
+      _linterVersions.add('0.1.$minor');
+    }
+  }
+  return _linterVersions;
+}
+
 class SinceInfo {
-  // todo (pq): add sinceSdk
-  String sinceLinter;
-  SinceInfo(this.sinceLinter);
+  final String sinceLinter;
+  final String sinceDartSdk;
+  SinceInfo({this.sinceLinter, this.sinceDartSdk});
 
   @override
-  String toString() => 'linter: $sinceLinter';
+  String toString() => 'linter: $sinceLinter | sdk: $sinceDartSdk';
 }