From ef5acd2da604841296376e0b7c99bc72f7a66fcd Mon Sep 17 00:00:00 2001 From: iameskild Date: Mon, 7 Mar 2022 14:03:13 +0100 Subject: [PATCH 01/12] make the commit of the terraform rendering optional --- qhub/provider/cicd/github.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/qhub/provider/cicd/github.py b/qhub/provider/cicd/github.py index 8f60ce59c7..124ae2974d 100644 --- a/qhub/provider/cicd/github.py +++ b/qhub/provider/cicd/github.py @@ -245,9 +245,11 @@ def gen_qhub_ops(config): }, ) - job1 = GHA_job_id( - name="qhub", runs_on_="ubuntu-latest", steps=[step1, step2, step3, step4, step5] - ) + gha_steps = [step1, step2, step3, step4] + if os.environ.get("QHUB_PREVENT_COMMIT_RENDER", "no") != "yes": + gha_steps.append(step5) + + job1 = GHA_job_id(name="qhub", runs_on_="ubuntu-latest", steps=gha_steps) jobs = GHA_jobs(__root__={"build": job1}) return QhubOps( From c13ddbc6e8fce41d1191b55d74676e4d5f41b12d Mon Sep 17 00:00:00 2001 From: iameskild Date: Mon, 7 Mar 2022 14:14:00 +0100 Subject: [PATCH 02/12] make the commit of the terraform rendering optional, gitlab --- qhub/provider/cicd/gitlab.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/qhub/provider/cicd/gitlab.py b/qhub/provider/cicd/gitlab.py index c63982a3a4..fae6aacff4 100644 --- a/qhub/provider/cicd/gitlab.py +++ b/qhub/provider/cicd/gitlab.py @@ -1,3 +1,4 @@ +import os from typing import Optional, Dict, List, Union from pydantic import BaseModel, Field @@ -53,6 +54,9 @@ def gen_gitlab_ci(config): f"git checkout {branch}", f"{pip_install}", "qhub deploy --config qhub-config.yaml --disable-prompt --skip-remote-state-provision", + ] + + commit_render_script = [ "git config user.email 'qhub@quansight.com'", "git config user.name 'gitlab ci'", "git add .", @@ -60,6 +64,9 @@ def gen_gitlab_ci(config): f"git push origin {branch})", ] + if os.environ.get("QHUB_PREVENT_COMMIT_RENDER", "no") != "yes": + script += commit_render_script + rules = [ GLCI_rules( if_=f"$CI_COMMIT_BRANCH == '{branch}'", From 8f5a839fdfb64e8cae5614f77fa44f277143f84d Mon Sep 17 00:00:00 2001 From: iameskild Date: Mon, 7 Mar 2022 17:30:34 +0100 Subject: [PATCH 03/12] Add commit_render to qhub_config schema --- qhub/provider/cicd/github.py | 3 ++- qhub/provider/cicd/gitlab.py | 4 ++-- qhub/schema.py | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/qhub/provider/cicd/github.py b/qhub/provider/cicd/github.py index 124ae2974d..94cfb930f9 100644 --- a/qhub/provider/cicd/github.py +++ b/qhub/provider/cicd/github.py @@ -215,6 +215,7 @@ def gen_qhub_ops(config): env_vars = gha_env_vars(config) branch = config["ci_cd"]["branch"] + commit_render = config["ci_cd"]["commit_render"] qhub_version = config["qhub_version"] push = GHA_on_extras(branches=[branch], paths=["qhub-config.yaml"]) @@ -246,7 +247,7 @@ def gen_qhub_ops(config): ) gha_steps = [step1, step2, step3, step4] - if os.environ.get("QHUB_PREVENT_COMMIT_RENDER", "no") != "yes": + if commit_render: gha_steps.append(step5) job1 = GHA_job_id(name="qhub", runs_on_="ubuntu-latest", steps=gha_steps) diff --git a/qhub/provider/cicd/gitlab.py b/qhub/provider/cicd/gitlab.py index fae6aacff4..75a5e444c8 100644 --- a/qhub/provider/cicd/gitlab.py +++ b/qhub/provider/cicd/gitlab.py @@ -1,4 +1,3 @@ -import os from typing import Optional, Dict, List, Union from pydantic import BaseModel, Field @@ -42,6 +41,7 @@ class GLCI(BaseModel): def gen_gitlab_ci(config): branch = config["ci_cd"]["branch"] + commit_render = config["ci_cd"]["commit_render"] before_script = config["ci_cd"].get("before_script") after_script = config["ci_cd"].get("after_script") pip_install = pip_install_qhub(config["qhub_version"]) @@ -64,7 +64,7 @@ def gen_gitlab_ci(config): f"git push origin {branch})", ] - if os.environ.get("QHUB_PREVENT_COMMIT_RENDER", "no") != "yes": + if commit_render: script += commit_render_script rules = [ diff --git a/qhub/schema.py b/qhub/schema.py index d9c8fceea4..091641ef7b 100644 --- a/qhub/schema.py +++ b/qhub/schema.py @@ -3,7 +3,7 @@ from abc import ABC import pydantic -from pydantic import validator, root_validator +from pydantic import validator, root_validator, Field from qhub.utils import namestr_regex from .version import rounded_ver_parse, __version__ @@ -54,6 +54,7 @@ class Config: class CICD(Base): type: CiEnum branch: str + commit_render: typing.Optional[bool] = Field(default=True) before_script: typing.Optional[typing.List[str]] after_script: typing.Optional[typing.List[str]] From a617c46d57ad728eb4bd137c250563949689f3ad Mon Sep 17 00:00:00 2001 From: iameskild Date: Mon, 7 Mar 2022 17:34:36 +0100 Subject: [PATCH 04/12] Make commit_render non-optional --- qhub/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qhub/schema.py b/qhub/schema.py index 091641ef7b..cb9a63442b 100644 --- a/qhub/schema.py +++ b/qhub/schema.py @@ -54,7 +54,7 @@ class Config: class CICD(Base): type: CiEnum branch: str - commit_render: typing.Optional[bool] = Field(default=True) + commit_render: bool = Field(default=True) before_script: typing.Optional[typing.List[str]] after_script: typing.Optional[typing.List[str]] From 156abb08bd25127b83acebfb5355d735e2e7d95f Mon Sep 17 00:00:00 2001 From: iameskild Date: Mon, 7 Mar 2022 17:43:28 +0100 Subject: [PATCH 05/12] Update initialize.py --- qhub/initialize.py | 6 +++++- qhub/schema.py | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/qhub/initialize.py b/qhub/initialize.py index ed505769c5..4f136f0dc5 100644 --- a/qhub/initialize.py +++ b/qhub/initialize.py @@ -65,7 +65,11 @@ }, } -CICD_CONFIGURATION = {"type": "PLACEHOLDER", "branch": "main"} +CICD_CONFIGURATION = { + "type": "PLACEHOLDER", + "branch": "main", + "commit_render": True, +} AUTH_PASSWORD = { "type": "password", diff --git a/qhub/schema.py b/qhub/schema.py index cb9a63442b..213920a55b 100644 --- a/qhub/schema.py +++ b/qhub/schema.py @@ -3,7 +3,7 @@ from abc import ABC import pydantic -from pydantic import validator, root_validator, Field +from pydantic import validator, root_validator from qhub.utils import namestr_regex from .version import rounded_ver_parse, __version__ @@ -54,7 +54,7 @@ class Config: class CICD(Base): type: CiEnum branch: str - commit_render: bool = Field(default=True) + commit_render: bool before_script: typing.Optional[typing.List[str]] after_script: typing.Optional[typing.List[str]] From 1321d7a21f7a5ce4390df356efd9167a7787cffa Mon Sep 17 00:00:00 2001 From: iameskild Date: Mon, 7 Mar 2022 17:51:29 +0100 Subject: [PATCH 06/12] Set default correctly --- qhub/initialize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qhub/initialize.py b/qhub/initialize.py index 4f136f0dc5..4898976e27 100644 --- a/qhub/initialize.py +++ b/qhub/initialize.py @@ -259,7 +259,7 @@ def render_config( config["provider"] = cloud_provider if ci_provider is not None and ci_provider != "none": - config["ci_cd"] = {"type": ci_provider, "branch": "main"} + config["ci_cd"] = {"type": ci_provider, "branch": "main", "commit_render": True} if terraform_state is not None: config["terraform_state"] = {"type": terraform_state} From 6df3e9dcd491b17c41309781ea6d644ccd15d50b Mon Sep 17 00:00:00 2001 From: iameskild Date: Mon, 7 Mar 2022 17:54:31 +0100 Subject: [PATCH 07/12] Remove unused dict --- qhub/initialize.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/qhub/initialize.py b/qhub/initialize.py index 4898976e27..afaa987968 100644 --- a/qhub/initialize.py +++ b/qhub/initialize.py @@ -65,12 +65,6 @@ }, } -CICD_CONFIGURATION = { - "type": "PLACEHOLDER", - "branch": "main", - "commit_render": True, -} - AUTH_PASSWORD = { "type": "password", } From b13880337e73d9557f6fb5a31604ebb4f34955d3 Mon Sep 17 00:00:00 2001 From: iameskild Date: Mon, 7 Mar 2022 18:44:33 +0100 Subject: [PATCH 08/12] Add env var to overwrite commit_render --- qhub/initialize.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/qhub/initialize.py b/qhub/initialize.py index afaa987968..583b8aa765 100644 --- a/qhub/initialize.py +++ b/qhub/initialize.py @@ -65,6 +65,12 @@ }, } +CICD_CONFIGURATION = { + "type": "PLACEHOLDER", + "branch": "main", + "commit_render": True, +} + AUTH_PASSWORD = { "type": "password", } @@ -253,7 +259,10 @@ def render_config( config["provider"] = cloud_provider if ci_provider is not None and ci_provider != "none": - config["ci_cd"] = {"type": ci_provider, "branch": "main", "commit_render": True} + config["ci_cd"] = CICD_CONFIGURATION.copy() + config["ci_cd"]["type"] = ci_provider + if os.environ.get("QHUB__cicd__commit_render") == "False": + config["ci_cd"]["commit_render"] = False if terraform_state is not None: config["terraform_state"] = {"type": terraform_state} From d35654daff44f75762537d17a7c0f2cd93d9104c Mon Sep 17 00:00:00 2001 From: iameskild Date: Tue, 8 Mar 2022 07:23:33 +0100 Subject: [PATCH 09/12] Make commit_render optional in the schema --- qhub/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qhub/schema.py b/qhub/schema.py index 213920a55b..f37a6df4e7 100644 --- a/qhub/schema.py +++ b/qhub/schema.py @@ -54,7 +54,7 @@ class Config: class CICD(Base): type: CiEnum branch: str - commit_render: bool + commit_render: typing.Optional[bool] before_script: typing.Optional[typing.List[str]] after_script: typing.Optional[typing.List[str]] From 172a58f84ea12704cf8375254d4b20c0ce1aed52 Mon Sep 17 00:00:00 2001 From: iameskild Date: Tue, 8 Mar 2022 07:24:40 +0100 Subject: [PATCH 10/12] Add default value of True --- qhub/schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qhub/schema.py b/qhub/schema.py index f37a6df4e7..091641ef7b 100644 --- a/qhub/schema.py +++ b/qhub/schema.py @@ -3,7 +3,7 @@ from abc import ABC import pydantic -from pydantic import validator, root_validator +from pydantic import validator, root_validator, Field from qhub.utils import namestr_regex from .version import rounded_ver_parse, __version__ @@ -54,7 +54,7 @@ class Config: class CICD(Base): type: CiEnum branch: str - commit_render: typing.Optional[bool] + commit_render: typing.Optional[bool] = Field(default=True) before_script: typing.Optional[typing.List[str]] after_script: typing.Optional[typing.List[str]] From da0b782365e248729dcd145ea37f63fefd99135e Mon Sep 17 00:00:00 2001 From: eskild <42120229+iameskild@users.noreply.github.com> Date: Tue, 8 Mar 2022 21:37:22 +0100 Subject: [PATCH 11/12] Update initialize.py --- qhub/initialize.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/qhub/initialize.py b/qhub/initialize.py index 583b8aa765..f35dc30397 100644 --- a/qhub/initialize.py +++ b/qhub/initialize.py @@ -261,8 +261,6 @@ def render_config( if ci_provider is not None and ci_provider != "none": config["ci_cd"] = CICD_CONFIGURATION.copy() config["ci_cd"]["type"] = ci_provider - if os.environ.get("QHUB__cicd__commit_render") == "False": - config["ci_cd"]["commit_render"] = False if terraform_state is not None: config["terraform_state"] = {"type": terraform_state} From 0113356603bc9fb6ff9ca25d33f2e26d045be303 Mon Sep 17 00:00:00 2001 From: eskild <42120229+iameskild@users.noreply.github.com> Date: Tue, 8 Mar 2022 21:38:06 +0100 Subject: [PATCH 12/12] Update schema.py --- qhub/schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qhub/schema.py b/qhub/schema.py index 091641ef7b..074968d2a3 100644 --- a/qhub/schema.py +++ b/qhub/schema.py @@ -3,7 +3,7 @@ from abc import ABC import pydantic -from pydantic import validator, root_validator, Field +from pydantic import validator, root_validator from qhub.utils import namestr_regex from .version import rounded_ver_parse, __version__ @@ -54,7 +54,7 @@ class Config: class CICD(Base): type: CiEnum branch: str - commit_render: typing.Optional[bool] = Field(default=True) + commit_render: typing.Optional[bool] = True before_script: typing.Optional[typing.List[str]] after_script: typing.Optional[typing.List[str]]