From 57067d986efa76318574657928a4035f1c47b96f Mon Sep 17 00:00:00 2001 From: Allison Reinheimer Moore <allison.moore@10gen.com> Date: Wed, 3 Jun 2020 12:03:20 -0400 Subject: [PATCH] DOP-1160: adds deploy stage to world builder, updates list of projects --- tools/world-builder/build.py | 89 ++++++++++++++++++++----------- tools/world-builder/projects.json | 83 ++++++++++++++++++++-------- 2 files changed, 119 insertions(+), 53 deletions(-) diff --git a/tools/world-builder/build.py b/tools/world-builder/build.py index 305525ef7..4eb1d91cd 100755 --- a/tools/world-builder/build.py +++ b/tools/world-builder/build.py @@ -7,67 +7,81 @@ import yaml from typing import Any, Dict, Optional, Set, List -PAT_URL = re.compile('.+/(.+).git') +PAT_URL = re.compile(".+/(.+).git") class BuildTask: - def __init__(self, name: str, branches: List[str], build_dir: str, build_command: str) -> None: + def __init__( + self, + name: str, + branches: List[str], + build_dir: str, + build_command: str, + deploy_command: str, + ) -> None: self.name = name self.branches = branches self.build_dir = build_dir self.build_command = build_command - self.built_flag_file = os.path.join(self.build_dir, '.built') + self.deploy_command = deploy_command + self.built_flag_file = os.path.join(self.build_dir, ".built") def build(self) -> None: - subprocess.check_call(['git', 'clean', '-xfd'], cwd=self.build_dir) + subprocess.check_call(["git", "clean", "-xfd"], cwd=self.build_dir) # Some of our properties die if these directories do not exist - for path in ('build', 'build/master', 'build/public', 'build/public/master'): + for path in ("build", "build/master", "build/public", "build/public/master"): try: os.mkdir(os.path.join(self.build_dir, path)) except OSError: pass for branch in self.branches: - print('Building ' + self.build_dir) - subprocess.check_call(['git', 'checkout', '-q', branch], cwd=self.build_dir) - subprocess.check_call(['git', 'pull', '--ff-only', '-q'], cwd=self.build_dir) + print("Building " + self.build_dir) + subprocess.check_call(["git", "checkout", "-q", branch], cwd=self.build_dir) + subprocess.check_call( + ["git", "pull", "--ff-only", "-q"], cwd=self.build_dir + ) subprocess.check_call(self.build_command, shell=True, cwd=self.build_dir) + def deploy(self) -> None: + for branch in self.branches: + print("Deploying " + self.build_dir + branch) + subprocess.check_call(["git", "checkout", "-q", branch], cwd=self.build_dir) + subprocess.check_call(self.deploy_command, shell=True, cwd=self.build_dir) + def load_branches_from_yaml(path: str) -> List[str]: - with open(path, 'r') as f: + with open(path, "r") as f: loaded = yaml.safe_load(f) - return loaded['git']['branches']['published'] + return loaded["git"]["branches"]["published"] def initialize_project(project: Dict[str, Any]) -> BuildTask: - git_url = project['git'] - project_name = project.get('name', None) + git_url = project["git"] + project_name = project.get("name", None) if not project_name: match = PAT_URL.match(git_url) if not match: - raise ValueError('Cannot determine project name: {}'.format(git_url)) + raise ValueError("Cannot determine project name: {}".format(git_url)) project_name = match.group(1) - branches = project['branches'] + branches = project["branches"] if isinstance(branches, str): branches = load_branches_from_yaml(branches) - output_dir = os.path.join('build', project_name) + output_dir = os.path.join("build", project_name) if not os.path.isdir(output_dir): - print('Cloning {}'.format(git_url)) - subprocess.check_call(['git', 'clone', '-q', git_url, output_dir]) + print("Cloning {}".format(git_url)) + subprocess.check_call(["git", "clone", "-q", git_url, output_dir]) return BuildTask( - project_name, - branches, - output_dir, - project['build']) + project_name, branches, output_dir, project["build"], project["deploy"] + ) def main(path: str, projects: Optional[Set[str]]) -> None: - with open(path, 'r') as f: + with open(path, "r") as f: data = json.load(f) tasks = [] @@ -76,24 +90,37 @@ def main(path: str, projects: Optional[Set[str]]) -> None: if projects is None or task.name in projects: tasks.append(task) - errors = [] # type: List[str] + build_errors = [] # type: List[str] for task in tasks: try: task.build() except subprocess.SubprocessError: - errors.append(task.build_dir) - print('Failed to build in {}'.format(task.build_dir)) + build_errors.append(task.build_dir) + print("Failed to build in {}".format(task.build_dir)) + tasks.remove(task) # so that we don't deploy the unbuilt thing + + if build_errors: + print("\nFailed to build:") + for d in build_errors: + print(" " + d) + + deploy_errors = [] # type: List[str] + for task in tasks: + try: + task.deploy() + except subprocess.SubprocessError: + deploy_errors.append(task.build) - if errors: - print('\nFailed to build:') - for d in errors: - print(' ' + d) + if deploy_errors: + print("\nFailed to deploy:") + for d in deploy_errors: + print(" " + d) -if __name__ == '__main__': +if __name__ == "__main__": if len(sys.argv) > 1: projects = set(sys.argv[1:]) # type: Optional[Set[str]] else: projects = None - main('projects.json', projects) + main("projects.json", projects) diff --git a/tools/world-builder/projects.json b/tools/world-builder/projects.json index 139eaebeb..550659cbd 100644 --- a/tools/world-builder/projects.json +++ b/tools/world-builder/projects.json @@ -2,79 +2,118 @@ { "git": "https://github.com/mongodb/docs-bi-connector.git", "branches": "../../data/bi-connector-published-branches.yaml", - "build": "mkdir -p build/master && make publish" + "build": "make publish", + "deploy": "make deploy" }, { "git": "https://github.com/mongodb/docs-spark-connector.git", "branches": "../../data/spark-connector-published-branches.yaml", - "build": "giza make publish" + "build": "make publish", + "deploy": "make deploy" }, { "git": "https://github.com/mongodb/docs.git", "branches": "../../data/manual-published-branches.yaml", - "build": "make publish" + "build": "make publish", + "deploy": "make deploy" }, { "name": "mms-onprem", "git": "git@github.com:10gen/mms-docs.git", "branches": "../../data/mms-published-branches.yaml", - "build": "giza make publish-onprem" + "build": "make publish-onprem", + "deploy": "make deploy-opsmgr" }, { "name": "mms-cloud", "git": "git@github.com:10gen/mms-docs.git", "branches": ["master"], - "build": "make publish-cloud" - }, - { - "git": "git@github.com:10gen/baas-docs.git", - "branches": "../../data/stitch-published-branches.yaml", - "build": "make publish" + "build": "make publish-cloud", + "deploy": "make deploy-cloud" }, { "git": "https://github.com/mongodb/docs-compass.git", "branches": "../../data/compass-published-branches.yaml", - "build": "make publish" + "build": "make publish", + "deploy": "make deploy" }, { "git": "git@github.com:10gen/docs-charts.git", "branches": "../../data/charts-published-branches.yaml", - "build": "make publish" + "build": "make publish", + "deploy": "make deploy" }, { "git": "git@github.com:10gen/cloud-docs.git", "branches": ["master"], - "build": "make publish" + "build": "make publish", + "deploy": "make deploy" }, { "git": "https://github.com/mongodb/docs-php-library.git", "branches": "../../data/php-library-published-branches.yaml", - "build": "make publish" + "build": "make publish", + "deploy": "make deploy" }, { "git": "https://github.com/mongodb/docs-ruby.git", "branches": "../../data/ruby-driver-published-branches.yaml", - "build": "make publish" + "build": "make publish", + "deploy": "make deploy" }, { "git": "git@github.com:10gen/docs-mongoid.git", "branches": "../../data/mongoid-published-branches.yaml", - "build": "make publish" + "build": "make publish", + "deploy": "make deploy" }, { "git": "git@github.com:mongodb/docs-meta.git", "branches": ["master"], - "build": "make publish" + "build": "make publish", + "deploy": "make deploy" }, { - "git": "git@github.com:mongodb/docs-ecosystem.git", + "name": "guides", + "git": "git@github.com:10gen/docs-tutorials.git", "branches": ["master"], - "build": "make publish" + "build": "make publish", + "deploy": "make deploy" }, { - "git": "https://github.com/mongodb/docs-landing.git", + "git": "git@github.com:mongodb/docs-commandline-tools.git", "branches": ["master"], - "build": "", - "docs-tools": false + "build": "make publish", + "deploy": "make deploy" + }, + { + "git": "git@github.com:10gen/docs-kafka-connector.git", + "branches": "../../data/kafka-connector-published-branches.yaml", + "build": "make publish", + "deploy": "make deploy" + }, + { + "git": "git@github.com:10gen/docs-mongocli.git", + "branches": "../../data/mongocli-published-branches.yaml", + "build": "make publish", + "deploy": "make deploy" + }, + { + "git": "git@github.com:mongodb/docs-mongodb-shell.git", + "branches": ["master"], + "build": "make publish", + "deploy": "make deploy" + }, + { + "git": "git@github.com:10gen/docs-k8s-operator.git", + "branches": "../../data/kubernetes-operator-published-branches.yaml", + "build": "make publish", + "deploy": "make deploy" + }, + { + "git": "git@github.com:10gen/cloud-docs-osb.git", + "branches": "../../data/atlas-open-service-broker-published-branches.yaml", + "build": "make publish", + "deploy": "make deploy" } ]