From 959dbf1ca66c7d824fdc968e59789f816da8797a Mon Sep 17 00:00:00 2001 From: "Brian R. Bondy" Date: Sun, 28 Aug 2016 16:05:23 -0400 Subject: [PATCH] Add cibuild script Auditors: @aekeus --- .gitignore | 3 ++ tools/cibuild.py | 82 ++++++++++++++++++++++++++++++++++++++ tools/lib/__init__.py | 0 tools/lib/github.py | 76 ++++++++++++++++++++++++++++++++++++ tools/upload.py | 91 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 252 insertions(+) create mode 100755 tools/cibuild.py create mode 100644 tools/lib/__init__.py create mode 100644 tools/lib/github.py create mode 100755 tools/upload.py diff --git a/.gitignore b/.gitignore index f830f5e6021..8715d9349f3 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,9 @@ pids *.pid *.seed +# Python +*.pyc + # local vagrant configuration files .vagrant diff --git a/tools/cibuild.py b/tools/cibuild.py new file mode 100755 index 00000000000..80a200e93e0 --- /dev/null +++ b/tools/cibuild.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python + +import os +import subprocess +import sys +import os.path + +UPSTREAM_ELECTRON = '1.3.3' +SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) +TARGET_ARCH= os.environ['TARGET_ARCH'] if os.environ.has_key('TARGET_ARCH') else 'x64' +os.environ['npm_config_arch'] = TARGET_ARCH + +def execute(argv, env=os.environ): + print ' '.join(argv) + try: + output = subprocess.check_output(argv, stderr=subprocess.STDOUT, env=env) + print output + return output + except subprocess.CalledProcessError as e: + print e.output + raise e + + +def write_npmrc(): + data = 'runtime = electron\n' \ + 'target = %s\n' \ + 'target_arch = %s\n' \ + 'disturl = https://atom.io/download/atom-shell\n' % (UPSTREAM_ELECTRON, TARGET_ARCH) + f = open('.npmrc','wb') + f.write(data) + f.close() + + +def run_script(script, args=[]): + sys.stderr.write('\nRunning ' + script +'\n') + sys.stderr.flush() + script = os.path.join(SOURCE_ROOT, 'tools', script) + subprocess.check_call([sys.executable, script] + args) + + +PLATFORM = { + 'cygwin': 'win32', + 'darwin': 'darwin', + 'linux2': 'linux', + 'win32': 'win32', +}[sys.platform] + +is_linux = PLATFORM == 'linux' +is_windows = PLATFORM == 'win32' +is_darwin = PLATFORM == 'darwin' + +deps = [] +if is_darwin: + deps = ['GITHUB_TOKEN', 'CHANNEL', 'IDENTIFIER'] +elif is_windows: + deps = ['GITHUB_TOKEN', 'CHANNEL', 'CERT_PASSWORD', 'TARGET_ARCH'] +else: + deps = ['GITHUB_TOKEN', 'CHANNEL'] + +if any(not os.environ.has_key(v) for v in deps): + print 'Missing some environment variables', deps + sys.exit(1) + +execute(['git', 'pull']) +execute(['rm', '-Rf', 'node_modules']) +execute(['rm', '-Rf', os.path.join(os.path.expanduser("~"), '.electron')]) + +write_npmrc() + +npm = 'npm.cmd' if is_windows else 'npm' +execute([npm, 'install']) + +if is_darwin: + execute(['node', './tools/electronBuilderHack.js']) +# For whatever reason on linux pstinstall webpack isn't running +elif is_linux: + execute([npm, 'run', 'webpack']) + +execute([npm, 'run', 'build-package']) +execute([npm, 'run', 'build-installer']) + +run_script('upload.py') diff --git a/tools/lib/__init__.py b/tools/lib/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tools/lib/github.py b/tools/lib/github.py new file mode 100644 index 00000000000..def7be585e8 --- /dev/null +++ b/tools/lib/github.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python + +import json +import os +import re +import sys + +REQUESTS_DIR = os.path.abspath(os.path.join(__file__, '..', '..', '..', + 'vendor', 'requests')) +sys.path.append(os.path.join(REQUESTS_DIR, 'build', 'lib')) +sys.path.append(os.path.join(REQUESTS_DIR, 'build', 'lib.linux-x86_64-2.7')) +import requests + +GITHUB_URL = 'https://api.github.com' +GITHUB_UPLOAD_ASSET_URL = 'https://uploads.github.com' + +class GitHub: + def __init__(self, access_token): + self._authorization = 'token %s' % access_token + + pattern = '^/repos/{0}/{0}/releases/{1}/assets$'.format('[^/]+', '[0-9]+') + self._releases_upload_api_pattern = re.compile(pattern) + + def __getattr__(self, attr): + return _Callable(self, '/%s' % attr) + + def send(self, method, path, **kw): + if not 'headers' in kw: + kw['headers'] = dict() + headers = kw['headers'] + headers['Authorization'] = self._authorization + headers['Accept'] = 'application/vnd.github.manifold-preview' + + # Switch to a different domain for the releases uploading API. + if self._releases_upload_api_pattern.match(path): + url = '%s%s' % (GITHUB_UPLOAD_ASSET_URL, path) + else: + url = '%s%s' % (GITHUB_URL, path) + # Data are sent in JSON format. + if 'data' in kw: + kw['data'] = json.dumps(kw['data']) + + r = getattr(requests, method)(url, **kw).json() + if 'message' in r: + raise Exception(json.dumps(r, indent=2, separators=(',', ': '))) + return r + + +class _Executable: + def __init__(self, gh, method, path): + self._gh = gh + self._method = method + self._path = path + + def __call__(self, **kw): + return self._gh.send(self._method, self._path, **kw) + + +class _Callable(object): + def __init__(self, gh, name): + self._gh = gh + self._name = name + + def __call__(self, *args): + if len(args) == 0: + return self + + name = '%s/%s' % (self._name, '/'.join([str(arg) for arg in args])) + return _Callable(self._gh, name) + + def __getattr__(self, attr): + if attr in ['get', 'put', 'post', 'patch', 'delete']: + return _Executable(self._gh, attr, self._name) + + name = '%s/%s' % (self._name, attr) + return _Callable(self._gh, name) diff --git a/tools/upload.py b/tools/upload.py new file mode 100755 index 00000000000..cff823a41d7 --- /dev/null +++ b/tools/upload.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python + +import json +import os +from lib.github import GitHub +import requests +from requests.packages.urllib3.exceptions import InsecureRequestWarning +requests.packages.urllib3.disable_warnings(InsecureRequestWarning) + +version = json.load(open('package.json'))['version'] +BROWSER_LAPTOP_REPO = 'brave/browser-laptop' +RELEASE_NAME = 'PRE (DO NOT DOWNLOAD UNLESS YOU ARE TESTING ' \ + 'THIS RELEASE CANDIDATE) Dev Channel Beta' + +def main(): + github = GitHub(auth_token()) + releases = github.repos(BROWSER_LAPTOP_REPO).releases.get() + tag_exists = False + for release in releases: + if not release['draft'] and release['tag_name'] == version: + tag_exists = True + break + release = create_or_get_release_draft(github, releases, version, + tag_exists) + for f in get_files_to_upload(): + upload_browser_laptop(github,release, f) + + +def get_files_to_upload(): + matches = [] + for root, dirnames, filenames in os.walk('dist'): + for filename in filenames: + matches.append(os.path.join(root, filename)) + return matches + + +def upload_browser_laptop(github, release, file_path): + filename = os.path.basename(file_path) + try: + for asset in release['assets']: + if asset['name'] == filename: + github.repos(BROWSER_LAPTOP_REPO).releases.assets(asset['id']).delete() + except Exception: + pass + + # Upload the file. + with open(file_path, 'rb') as f: + upload_io_to_github(github, release, + filename, f, 'application/octet-stream') + + +def create_release_draft(github, tag): + name = '{0} {1}'.format(RELEASE_NAME, tag) + # TODO: Parse release notes from CHANGELOG.md + body = '(placeholder)' + if body == '': + sys.stderr.write('Quit due to empty release note.\n') + sys.exit(1) + data = dict(tag_name=tag, name=name, body=body, draft=True) + r = github.repos(BROWSER_LAPTOP_REPO).releases.post(data=data) + return r + + +def create_or_get_release_draft(github, releases, tag, tag_exists): + # Search for existing draft. + for release in releases: + if release['draft']: + return release + + if tag_exists: + tag = 'do-not-publish-me' + return create_release_draft(github, tag) + +def upload_io_to_github(github, release, name, io, content_type): + params = {'name': name} + headers = {'Content-Type': content_type} + github.repos(BROWSER_LAPTOP_REPO).releases(release['id']).assets.post( + params=params, headers=headers, data=io, verify=False) + + +def auth_token(): + token = os.environ['GITHUB_TOKEN'] + message = ('Error: Please set the $GITHUB_TOKEN ' + 'environment variable, which is your personal token') + assert token, message + return token + + +if __name__ == '__main__': + import sys + sys.exit(main())