diff --git a/build/vs_toolchain.py b/build/vs_toolchain.py index 9625a1d5ab..9f419a1785 100644 --- a/build/vs_toolchain.py +++ b/build/vs_toolchain.py @@ -32,7 +32,7 @@ script_dir = os.path.dirname(os.path.realpath(__file__)) json_data_file = os.path.join(script_dir, 'win_toolchain.json') -sys.path.insert(0, os.path.join(script_dir)) +sys.path.insert(0, os.path.join(script_dir, '..', 'tools')) # VS versions are listed in descending order of priority (highest first). MSVS_VERSIONS = collections.OrderedDict([ diff --git a/tools/buildtools/update.py b/tools/buildtools/update.py new file mode 100644 index 0000000000..831863a8c6 --- /dev/null +++ b/tools/buildtools/update.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# +# Copyright 2017 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Pulls down tools required to build flutter.""" + +import os +import subprocess +import sys + +SRC_ROOT = (os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +BUILDTOOLS = os.path.join(SRC_ROOT, 'buildtools') +TOOLS_BUILDTOOLS = os.path.join(SRC_ROOT, 'tools', 'buildtools') + +sys.path.insert(0, os.path.join(SRC_ROOT, 'tools')) +import find_depot_tools + +DEPOT_PATH = find_depot_tools.add_depot_tools_to_path() + + +def Update(): + path = os.path.join(BUILDTOOLS, 'update.sh') + return subprocess.call([ + '/bin/bash', path, '--ninja', '--gn', '--clang'], cwd=SRC_ROOT) + + +def UpdateOnWindows(): + sha1_file = os.path.join(TOOLS_BUILDTOOLS, 'win', 'gn.exe.sha1') + output_dir = os.path.join(BUILDTOOLS, 'win', 'gn.exe') + downloader_script = os.path.join(DEPOT_PATH, 'download_from_google_storage.py') + download_cmd = [ + 'python', + downloader_script, + '--no_auth', + '--no_resume', + '--quiet', + '--platform=win*', + '--bucket', + 'chromium-gn', + '-s', + sha1_file, + '-o', + output_dir + ] + return subprocess.call(download_cmd) + + +def main(argv): + if sys.platform.startswith('win'): + return UpdateOnWindows() + return Update() + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/tools/buildtools/win/gn.exe.sha1 b/tools/buildtools/win/gn.exe.sha1 new file mode 100644 index 0000000000..0ef132f1fb --- /dev/null +++ b/tools/buildtools/win/gn.exe.sha1 @@ -0,0 +1 @@ +e93779cab57d5f36100faa4d88524a1e33be7b0f diff --git a/tools/dart/create_updated_flutter_deps.py b/tools/dart/create_updated_flutter_deps.py new file mode 100755 index 0000000000..838488c63c --- /dev/null +++ b/tools/dart/create_updated_flutter_deps.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +# +# Copyright 2017 The Dart project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Usage: tools/dart/create_updated_flutter_deps.py [-d dart/DEPS] [-f flutter/DEPS] +# +# This script parses existing flutter DEPS file, identifies all 'dart_' prefixed +# dependencies, looks up revision from dart DEPS file, updates those dependencies +# and rewrites flutter DEPS file. + +import argparse +import os +import sys + +DART_SCRIPT_DIR = os.path.dirname(sys.argv[0]) +DART_ROOT = os.path.realpath(os.path.join(DART_SCRIPT_DIR, '../../third_party/dart')) +FLUTTER_ROOT = os.path.realpath(os.path.join(DART_SCRIPT_DIR, '../../flutter')) + +class VarImpl(object): + def __init__(self, local_scope): + self._local_scope = local_scope + + def Lookup(self, var_name): + """Implements the Var syntax.""" + if var_name in self._local_scope.get("vars", {}): + return self._local_scope["vars"][var_name] + if var_name == 'host_os': + return 'linux' # assume some default value + if var_name == 'host_cpu': + return 'x64' # assume some default value + raise Exception("Var is not defined: %s" % var_name) + + +def ParseDepsFile(deps_file): + local_scope = {} + var = VarImpl(local_scope) + global_scope = { + 'Var': var.Lookup, + 'deps_os': {}, + } + # Read the content. + with open(deps_file, 'r') as fp: + deps_content = fp.read() + + # Eval the content. + exec(deps_content, global_scope, local_scope) + + return (local_scope.get('vars', {}), local_scope.get('deps', {})) + +def ParseArgs(args): + args = args[1:] + parser = argparse.ArgumentParser( + description='A script to generate updated dart dependencies for flutter DEPS.') + parser.add_argument('--dart_deps', '-d', + type=str, + help='Dart DEPS file.', + default=os.path.join(DART_ROOT, 'DEPS')) + parser.add_argument('--flutter_deps', '-f', + type=str, + help='Flutter DEPS file.', + default=os.path.join(FLUTTER_ROOT, 'DEPS')) + return parser.parse_args(args) + +def Main(argv): + args = ParseArgs(argv) + (new_vars, new_deps) = ParseDepsFile(args.dart_deps) + (old_vars, old_deps) = ParseDepsFile(args.flutter_deps) + + updated_vars = {} + + # Collect updated dependencies + for (k,v) in sorted(old_vars.items()): + if k not in ('dart_revision', 'dart_git') and k.startswith('dart_'): + dart_key = k[len('dart_'):] + if dart_key in new_vars: + updated_revision = new_vars[dart_key].lstrip('@') if dart_key in new_vars else v + updated_vars[k] = updated_revision + + # Write updated DEPS file to a side + updatedfilename = args.flutter_deps + ".new" + updatedfile = open(updatedfilename, "w") + file = open(args.flutter_deps) + lines = file.readlines() + i = 0 + while i < len(lines): + updatedfile.write(lines[i]) + if lines[i].startswith(" 'dart_revision':"): + i = i + 2 + updatedfile.writelines([ + '\n', + ' # WARNING: DO NOT EDIT MANUALLY\n', + ' # The lines between blank lines above and below are generated by a script. See create_updated_flutter_deps.py\n']) + while i < len(lines) and len(lines[i].strip()) > 0: + i = i + 1 + for (k, v) in sorted(updated_vars.items()): + updatedfile.write(" '%s': '%s',\n" % (k, v)) + updatedfile.write('\n') + + elif lines[i].startswith(" # WARNING: Unused Dart dependencies"): + updatedfile.write('\n') + i = i + 1 + while i < len(lines) and (lines[i].startswith(" # WARNING: end of dart dependencies") == 0): + i = i + 1 + for (k, v) in sorted(old_deps.items()): + if (k.startswith('src/third_party/dart/')): + for (dart_k, dart_v) in (list(new_deps.items())): + dart_k_suffix = dart_k[len('sdk/') if dart_k.startswith('sdk/') else 0:] + if (k.endswith(dart_k_suffix)): + if (isinstance(dart_v, str)): + updated_value = dart_v.replace(new_vars["dart_git"], "Var('dart_git') + '/") + updated_value = updated_value.replace(old_vars["chromium_git"], "Var('chromium_git') + '") + + plain_v = dart_k[dart_k.rfind('/') + 1:] + # This dependency has to be special-cased here because the + # repository name is not the same as the directory name. + if plain_v == "quiver": + plain_v = "quiver-dart" + if ('dart_' + plain_v + '_tag' in updated_vars): + updated_value = updated_value[:updated_value.rfind('@')] + "' + '@' + Var('dart_" + plain_v + "_tag')" + elif ('dart_' + plain_v + '_rev' in updated_vars): + updated_value = updated_value[:updated_value.rfind('@')] + "' + '@' + Var('dart_" + plain_v + "_rev')" + else: + updated_value = updated_value + "'" + else: + # Non-string values(dicts) copy verbatim, keeping them sorted + # to ensure stable ordering of items. + updated_value = dict(sorted(dart_v.items())) + + updatedfile.write(" '%s':\n %s,\n\n" % (k, updated_value)) + break + updatedfile.write(lines[i]) + i = i + 1 + + # Rename updated DEPS file into a new DEPS file + os.remove(args.flutter_deps) + os.rename(updatedfilename, args.flutter_deps) + + return 0 + +if __name__ == '__main__': + sys.exit(Main(sys.argv)) diff --git a/tools/dart/get_dart_roll_revisions.py b/tools/dart/get_dart_roll_revisions.py new file mode 100644 index 0000000000..64f5431504 --- /dev/null +++ b/tools/dart/get_dart_roll_revisions.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +# +# Copyright 2019 The Dart project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +# +# This tool is used to get a list of Dart SDK revisions which have been rolled +# into the flutter/engine repository on GitHub. Revisions are printed in reverse +# chronological order along with the merge date and link to the PR created for +# the roll. +# +# This tool requires the following setup to work: +# - Run `pip install PyGithub` to install PyGithub (github) +# - Set the GITHUB_API_KEY environment variable to a Github personal access +# token (see https://github.com/settings/tokens). + +from github import Github +import argparse +import os + +DART_REVISION_PATCH_STR = "+ 'dart_revision'" +DART_SDK_ROLL = 'Roll src/third_party/dart' +GITHUB_API_KEY = os.getenv('GITHUB_API_KEY') + +# Control codes for coloured terminal output. +CGREEN = '\033[92m' +CEND = '\033[0m' + +def get_revision_from_patch(patch): + revision_line_list = [x for x in patch.splitlines() + if DART_REVISION_PATCH_STR in x] + assert(len(revision_line_list) == 1) + revision_line = revision_line_list[0] + return revision_line.split()[-1][:-1].replace("'","") + + +def print_output(revision, pull): + msg = CGREEN + 'SDK Revision: ' + CEND + msg += revision + CGREEN + ' Merged At: ' + CEND + msg += str(pull.merged_at) + CGREEN + ' PR: ' + CEND + pull.html_url + print(msg) + + +def main(): + parser = argparse.ArgumentParser(description='Get Dart SDK revisions which ' + + 'have been rolled into flutter/engine from ' + + 'GitHub.') + parser.add_argument('--github-api-key', help='The GitHub API key to be used ' + + 'for querying the flutter/engine pull requests. Defaults' + + ' to the "GITHUB_API_KEY" environment variable if this ' + + 'option is not provided.') + parser.add_argument('--max-revisions', help='The maximum number of revisions ' + + 'of Dart SDKs which have been rolled into flutter/engine ' + + 'to return (default: 10).', default=10, type=int) + + args = parser.parse_args() + + github_api_key = args.github_api_key + if not github_api_key: + github_api_key = GITHUB_API_KEY + + max_revisions = args.max_revisions + revision_count = 0 + + github = Github(github_api_key) + github_engine_repo = github.get_repo('flutter/engine') + pulls = github_engine_repo.get_pulls(state='closed', + sort='created', + direction='desc') + + for pull in pulls: + if DART_SDK_ROLL in pull.title and pull.merged: + # Get the last commit from the PR. Automated SDK rolls shouldn't have many + # commits in their PRs, so this shouldn't be too expensive. + commit = [c for c in pull.get_commits()][-1] + for f in commit.files: + if f.filename == 'DEPS': + print_output(get_revision_from_patch(f.patch), pull) + revision_count += 1 + if revision_count == max_revisions: + return + break + + +if __name__ == '__main__': + main() diff --git a/build/find_depot_tools.py b/tools/find_depot_tools.py similarity index 100% rename from build/find_depot_tools.py rename to tools/find_depot_tools.py diff --git a/tools/remove_stale_pyc_files.py b/tools/remove_stale_pyc_files.py new file mode 100755 index 0000000000..3dec7c3b87 --- /dev/null +++ b/tools/remove_stale_pyc_files.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +# +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import sys + + +def RemoveAllStalePycFiles(base_dir): + """Scan directories for old .pyc files without a .py file and delete them.""" + for dirname, _, filenames in os.walk(base_dir): + if '.svn' in dirname or '.git' in dirname: + continue + for filename in filenames: + root, ext = os.path.splitext(filename) + if ext != '.pyc': + continue + + pyc_path = os.path.join(dirname, filename) + py_path = os.path.join(dirname, root + '.py') + + try: + if not os.path.exists(py_path): + os.remove(pyc_path) + except OSError: + # Wrap OS calls in try/except in case another process touched this file. + pass + + try: + os.removedirs(dirname) + except OSError: + # Wrap OS calls in try/except in case another process touched this dir. + pass + + +if __name__ == '__main__': + for path in sys.argv[1:]: + RemoveAllStalePycFiles(path)