From a5e6fdbaaa5fd08f8b0372b3e87987e54990cda8 Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 23 Feb 2018 11:54:16 +0100 Subject: [PATCH 1/2] unpin dev requirements pinning strict requirements is okay for application deploymens to do, but not for user environments, which is how we are using dev-requirements in CONTRIBUTING.md --- dev-requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 7af53f4f3e..6b7e7d6682 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,3 @@ -PyGithub==1.35 -python-dateutil==2.6.1 -ruamel.yaml==0.15.35 +PyGithub>=1 +python-dateutil>=2 +ruamel.yaml>=0.15 From 6aa7c23ffbb05bc2731c41b71bab66cbc713f68d Mon Sep 17 00:00:00 2001 From: Min RK Date: Fri, 23 Feb 2018 12:07:24 +0100 Subject: [PATCH 2/2] get chartpress from standalone repo removes build.py --- .travis.yml | 2 +- CONTRIBUTING.md | 8 +- build.py | 193 ------------------------------------------- ci/deploy.sh | 2 +- dev-requirements.txt | 2 +- 5 files changed, 7 insertions(+), 200 deletions(-) delete mode 100755 build.py diff --git a/.travis.yml b/.travis.yml index 575b82dd49..8307de31b6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ install: - . ci/minikube.env - ./ci/install.sh script: -- ./build.py --commit-range ${TRAVIS_COMMIT_RANGE} +- chartpress --commit-range ${TRAVIS_COMMIT_RANGE} - ./ci/test.sh deploy: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0fd1996749..62ff06c5a8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,9 +26,9 @@ development. python3 -m venv . pip install -r dev-requirements.txt ``` - 6. Now run `build.py` to build the requisite docker images inside minikube: + 6. Now run `chartpress` to build the requisite docker images inside minikube: ```bash - ./build.py + chartpress ``` This will build the docker images inside minikube & modify @@ -164,9 +164,9 @@ the DockerHub registry, open an issue and ping one of the core devs. If you have all of this, you can then: 1. Check out latest master of [z2jh](https://github.com/jupyterhub/zero-to-jupyterhub-k8s) -2. Run `./build.py --tag --push --publish-chart`. +2. Run `chartpress --tag --push --publish-chart`. * For example, to relase `v0.5`, you would run - `./build.py --tag v0.5 --push --publish-chart`. + `chartpress --tag v0.5 --push --publish-chart`. Note the `v` before version. 3. This will also modify the files `Chart.yaml` and `values.yaml`. Commit these changes. diff --git a/build.py b/build.py deleted file mode 100755 index 66f0c43437..0000000000 --- a/build.py +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import os -import subprocess -import shutil -from tempfile import TemporaryDirectory - -from ruamel.yaml import YAML - -# use safe roundtrip yaml loader -yaml = YAML(typ='rt') -yaml.indent(mapping=2, offset=2, sequence=4) - -def last_modified_commit(*paths, **kwargs): - return subprocess.check_output([ - 'git', - 'log', - '-n', '1', - '--pretty=format:%h', - *paths - ], **kwargs).decode('utf-8') - -def last_modified_date(*paths, **kwargs): - return subprocess.check_output([ - 'git', - 'log', - '-n', '1', - '--pretty=format:%cd', - '--date=iso', - *paths - ], **kwargs).decode('utf-8') - -def path_touched(*paths, commit_range): - return subprocess.check_output([ - 'git', 'diff', '--name-only', commit_range, *paths - ]).decode('utf-8').strip() != '' - - -def render_build_args(options, ns): - """Get docker build args dict, rendering any templated args.""" - build_args = options.get('buildArgs', {}) - for key, value in build_args.items(): - build_args[key] = value.format(**ns) - return build_args - -def build_image(image_path, image_spec, build_args): - cmd = ['docker', 'build', '-t', image_spec, image_path] - - for k, v in build_args.items(): - cmd += ['--build-arg', '{}={}'.format(k, v)] - subprocess.check_call(cmd) - -def build_images(prefix, images, tag=None, commit_range=None, push=False): - value_modifications = {} - for name, options in images.items(): - image_path = os.path.join('images', name) - paths = options.get('paths', []) + [image_path] - last_commit = last_modified_commit(*paths) - if tag is None: - tag = last_commit - image_name = prefix + name - image_spec = '{}:{}'.format(image_name, tag) - value_modifications[options['valuesPath']] = { - 'name': image_name, - 'tag': tag - } - - if commit_range and not path_touched(*paths, commit_range=commit_range): - print(f"Skipping {name}, not touched in {commit_range}") - continue - - template_namespace = { - 'LAST_COMMIT': last_commit, - 'TAG': tag, - } - - build_args = render_build_args(options, template_namespace) - build_image(image_path, image_spec, build_args) - - if push: - subprocess.check_call([ - 'docker', 'push', image_spec - ]) - return value_modifications - -def build_values(name, values_mods): - """Update name/values.yaml with modifications""" - - values_file = os.path.join(name, 'values.yaml') - - with open(values_file) as f: - values = yaml.load(f) - - for key, value in values_mods.items(): - parts = key.split('.') - mod_obj = values - for p in parts: - mod_obj = mod_obj[p] - mod_obj.update(value) - - - with open(values_file, 'w') as f: - yaml.dump(values, f) - - -def build_chart(name, version=None, paths=None): - """Update chart with specified version or last-modified commit in path(s)""" - chart_file = os.path.join(name, 'Chart.yaml') - with open(chart_file) as f: - chart = yaml.load(f) - - if version is None: - if paths is None: - paths = ['.'] - commit = last_modified_commit(*paths) - version = chart['version'].split('-')[0] + '-' + commit - - chart['version'] = version - - with open(chart_file, 'w') as f: - yaml.dump(chart, f) - - -def publish_pages(name, paths, git_repo, published_repo): - """publish helm chart index to github pages""" - version = last_modified_commit(*paths) - checkout_dir = '{}-{}'.format(name, version) - subprocess.check_call([ - 'git', 'clone', '--no-checkout', - 'git@github.com:{}'.format(git_repo), checkout_dir], - ) - subprocess.check_call(['git', 'checkout', 'gh-pages'], cwd=checkout_dir) - - # package the latest version into a temporary directory - # and run helm repo index with --merge to update index.yaml - # without refreshing all of the timestamps - with TemporaryDirectory() as td: - subprocess.check_call([ - 'helm', 'package', name, - '--destination', td + '/', - ]) - - subprocess.check_call([ - 'helm', 'repo', 'index', td, - '--url', published_repo, - '--merge', os.path.join(checkout_dir, 'index.yaml'), - ]) - - # equivalent to `cp td/* checkout/` - # copies new helm chart and updated index.yaml - for f in os.listdir(td): - shutil.copy2( - os.path.join(td, f), - os.path.join(checkout_dir, f) - ) - subprocess.check_call(['git', 'add', '.'], cwd=checkout_dir) - subprocess.check_call([ - 'git', - 'commit', - '-m', '[{}] Automatic update for commit {}'.format(name, version) - ], cwd=checkout_dir) - subprocess.check_call( - ['git', 'push', 'origin', 'gh-pages'], - cwd=checkout_dir, - ) - - -def main(): - with open('chartpress.yaml') as f: - config = yaml.load(f) - - argparser = argparse.ArgumentParser() - - argparser.add_argument('--commit-range', help='Range of commits to consider when building images') - argparser.add_argument('--push', action='store_true') - argparser.add_argument('--publish-chart', action='store_true') - argparser.add_argument('--tag', default=None, help='Use this tag for images & charts') - - args = argparser.parse_args() - - for chart in config['charts']: - value_mods = build_images(chart['imagePrefix'], chart['images'], args.tag, args.commit_range, args.push) - build_values(chart['name'], value_mods) - chart_paths = ['.'] + chart.get('paths', []) - build_chart(chart['name'], paths=chart_paths, version=args.tag) - if args.publish_chart: - publish_pages(chart['name'], - paths=chart_paths, - git_repo=chart['repo']['git'], - published_repo=chart['repo']['published'], - ) - -main() diff --git a/ci/deploy.sh b/ci/deploy.sh index 148f23bd7f..da3ee3841f 100755 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -5,4 +5,4 @@ set -x chmod 0400 travis docker login -u "${DOCKER_USERNAME}" -p "${DOCKER_PASSWORD}" export GIT_SSH_COMMAND="ssh -i ${PWD}/travis" -./build.py --commit-range "${TRAVIS_COMMIT_RANGE}" --push --publish-chart +chartpress --commit-range "${TRAVIS_COMMIT_RANGE}" --push --publish-chart diff --git a/dev-requirements.txt b/dev-requirements.txt index 6b7e7d6682..e30897c5a8 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,3 @@ PyGithub>=1 python-dateutil>=2 -ruamel.yaml>=0.15 +https://github.com/jupyterhub/chartpress/archive/master.tar.gz