Updates/fixes for rendering CICD workflows (#1086)

* Clean up GHA schema

* Rename

* Remove extra braces, minor fixes

* Fix env ref

* Add quotes

* gitlab-ci.yaml -> yml

* Modify gitlab-ci rule

* Fix

qhub/provider/cicd/github.py

@@ -126,7 +126,7 @@ def gha_env_vars(config):
 
 class GHA_on_extras(BaseModel):
     branches: List[str]
-    path: List[str]
+    paths: List[str]
 
 
 class GHA_on(BaseModel):
@@ -171,7 +171,7 @@ class GHA(BaseModel):
     name: str
     on: GHA_on
     env: Optional[Dict[str, str]]
-    jobs: List[GHA_jobs]
+    jobs: GHA_jobs
 
 
 class QhubOps(GHA):
@@ -193,7 +193,7 @@ def checkout_image_step():
         uses="actions/checkout@master",
         with_={
             "token": GHA_job_steps_extras(
-                __root__="{{ '${{ secrets.REPOSITORY_ACCESS_TOKEN }}' }}"
+                __root__="${{ secrets.REPOSITORY_ACCESS_TOKEN }}"
             )
         },
     )
@@ -217,7 +217,7 @@ def gen_qhub_ops(config):
     branch = config["ci_cd"]["branch"]
     qhub_version = config["qhub_version"]
 
-    push = GHA_on_extras(branches=[branch], path=["qhub-config.yaml"])
+    push = GHA_on_extras(branches=[branch], paths=["qhub-config.yaml"])
     on = GHA_on(__root__={"push": push})
 
     step1 = checkout_image_step()
@@ -235,20 +235,20 @@ def gen_qhub_ops(config):
             "git config user.email 'qhub@quansight.com' ; "
             "git config user.name 'github action' ; "
             "git add . ; "
-            "git diff --quiet && git diff --staged --quiet || (git commit -m '${COMMIT_MSG}') ; "
+            "git diff --quiet && git diff --staged --quiet || (git commit -m '${{ env.COMMIT_MSG }}') ; "
             f"git push origin {branch}"
         ),
         env={
             "COMMIT_MSG": GHA_job_steps_extras(
-                __root__="qhub-config.yaml automated commit: {{ '${{ github.sha }}' }}"
+                __root__="qhub-config.yaml automated commit: ${{ github.sha }}"
             )
         },
     )
 
     job1 = GHA_job_id(
         name="qhub", runs_on_="ubuntu-latest", steps=[step1, step2, step3, step4, step5]
     )
-    jobs = [GHA_jobs(__root__={"build": job1})]
+    jobs = GHA_jobs(__root__={"build": job1})
 
     return QhubOps(
         name="qhub auto update",
@@ -270,20 +270,18 @@ def gen_qhub_linter(config):
     branch = config["ci_cd"]["branch"]
     qhub_version = config["qhub_version"]
 
-    pull_request = GHA_on_extras(branches=[branch], path=["qhub-config.yaml"])
+    pull_request = GHA_on_extras(branches=[branch], paths=["qhub-config.yaml"])
     on = GHA_on(__root__={"pull_request": pull_request})
 
     step1 = checkout_image_step()
     step2 = setup_python_step()
     step3 = install_qhub_step(qhub_version)
 
     step4_envs = {
-        "PR_NUMBER": GHA_job_steps_extras(
-            __root__="{{ '${{ github.event.number }}' }}"
-        ),
-        "REPO_NAME": GHA_job_steps_extras(__root__="{{ '${{ github.repository }}' }}"),
+        "PR_NUMBER": GHA_job_steps_extras(__root__="${{ github.event.number }}"),
+        "REPO_NAME": GHA_job_steps_extras(__root__="${{ github.repository }}"),
         "GITHUB_TOKEN": GHA_job_steps_extras(
-            __root__="{{ '${{ secrets.REPOSITORY_ACCESS_TOKEN }}' }}"
+            __root__="${{ secrets.REPOSITORY_ACCESS_TOKEN }}"
         ),
     }
 
@@ -296,7 +294,11 @@ def gen_qhub_linter(config):
     job1 = GHA_job_id(
         name="qhub", runs_on_="ubuntu-latest", steps=[step1, step2, step3, step4]
     )
-    jobs = [GHA_jobs(__root__={"qhub-validate": job1})]
+    jobs = GHA_jobs(
+        __root__={
+            "qhub-validate": job1,
+        }
+    )
 
     return QhubLinter(
         name="qhub linter",

qhub/provider/cicd/gitlab.py

@@ -17,8 +17,6 @@ class GLCI_image(BaseModel):
 class GLCI_rules(BaseModel):
     if_: Optional[str] = Field(alias="if")
     changes: Optional[List[str]]
-    # exists:
-    # variables:
 
     class Config:
         allow_population_by_field_name = True
@@ -30,7 +28,7 @@ class GLCI_job(BaseModel):
     before_script: Optional[List[str]]
     after_script: Optional[List[str]]
     script: List[str]
-    rules: GLCI_rules
+    rules: Optional[List[GLCI_rules]]
 
 
 class GLCI(BaseModel):
@@ -51,24 +49,23 @@ def gen_gitlab_ci(config):
         "COMMIT_MSG": "qhub-config.yaml automated commit: {{ '$CI_COMMIT_SHA' }}",
     }
 
-    # if qhub_gh_branch:
-    #     render_vars["QHUB_GH_BRANCH"] = qhub_gh_branch
-    #     pip_install_qhub = f"pip install https://github.com/Quansight/qhub/archive/{qhub_gh_branch}.zip"
-
     script = [
         f"git checkout {branch}",
         f"{pip_install}",
         "qhub deploy --config qhub-config.yaml --disable-prompt --skip-remote-state-provision",
         "git config user.email 'qhub@quansight.com'",
         "git config user.name 'gitlab ci'",
         "git add .",
-        "git diff --quiet && git diff --staged --quiet || (git commit -m '${COMMIT_MSG}'; git push origin {branch})",
+        "git diff --quiet && git diff --staged --quiet || (git commit -m '${COMMIT_MSG}'",
+        f"git push origin {branch})",
     ]
 
-    rules = GLCI_rules(
-        if_=f"$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == '{branch}'",
-        changes=["qhub-config.yaml"],
-    )
+    rules = [
+        GLCI_rules(
+            if_=f"$CI_COMMIT_BRANCH == '{branch}'",
+            changes=["qhub-config.yaml"],
+        )
+    ]
 
     render_qhub = GLCI_job(
         image=f"python:{PYTHON_VERSION}",

qhub/render.py

@@ -184,7 +184,7 @@ def gen_cicd(config):
         cicd_files[gha_dir + "qhub-linter.yaml"] = gen_qhub_linter(config)
 
     elif cicd_provider == "gitlab-ci":
-        cicd_files[".gitlab-ci.yaml"] = gen_gitlab_ci(config)
+        cicd_files[".gitlab-ci.yml"] = gen_gitlab_ci(config)
 
     else:
         raise ValueError(

qhub/utils.py

@@ -352,7 +352,7 @@ def pip_install_qhub(qhub_version: str) -> str:
     # dev branches
     if len(qhub_version.split(".")) > 3 and qhub_gh_branch:
         pip_install = (
-            f"pip install https://github.com/Quansight/qhub.git@{qhub_gh_branch}"
+            f"pip install git+https://github.com/Quansight/qhub.git@{qhub_gh_branch}"
         )
 
     return pip_install