Initial commit for auth and stages workflow

.github/workflows/image.yaml

@@ -8,7 +8,7 @@ on:
 
   push:
     paths:
-      - "qhub/template/{{ cookiecutter.repo_directory }}/image/**"
+      - "qhub/template/image/**"
       - ".github/workflows/image.yaml"
 
     branches: # Only on paths above
@@ -17,9 +17,9 @@ on:
     tags: # Always, regardless of paths above
       - '*'
 
-concurrency:
-  group: ${{ github.ref }}
-  cancel-in-progress: true
+# concurrency:
+#   group: ${{ github.ref }}
+#   cancel-in-progress: true
 
 jobs:
   jupyterlab:
@@ -29,10 +29,8 @@ jobs:
       matrix:
         dockerfile:
           - jupyterlab
-          - jupyterlab-centos
           - jupyterhub
           - dask-worker
-          - dask-worker-centos
           - dask-gateway
     steps:
       - name: 'Checkout Infrastructure'
@@ -77,15 +75,14 @@ jobs:
       - name: Add symlink to cookiecutter dir and tree
         run: |
           cd ./qhub/template
-          ln -s ./\{\{\ cookiecutter.repo_directory\ \}\} cookiecutter
           sudo apt-get install tree
-          tree ./cookiecutter/image
+          tree ./image
 
       - name: Build docker
         uses: docker/build-push-action@v2
         with:
-          context: ./qhub/template/cookiecutter/image
-          file: "./qhub/template/cookiecutter/image/Dockerfile.${{ matrix.dockerfile }}"
+          context: "./qhub/template/image"
+          file: "./qhub/template/image/Dockerfile.${{ matrix.dockerfile }}"
           tags: |
             ${{ steps.meta.outputs.tags }}
             ghcr.io/${{ steps.meta.outputs.tags }}
@@ -96,6 +93,6 @@ jobs:
       - name: Lint Dockerfiles
         uses: jbergstroem/hadolint-gh-action@v1
         with:
-          dockerfile: ./qhub/template/cookiecutter/image/Dockerfile.${{ matrix.dockerfile }}
+          dockerfile: ./qhub/template/image/Dockerfile.${{ matrix.dockerfile }}
           output_format: tty
           error_level: 0

.github/workflows/kubernetes_test.yaml

@@ -130,7 +130,9 @@ jobs:
         run: echo "QHUB_CONFIG_PATH=`realpath ./local-deployment/qhub-config.yaml`" >> "$GITHUB_ENV"
 
       - name: Create example-user
-        run: qhub keycloak --config "${QHUB_CONFIG_PATH}" adduser example-user "${CYPRESS_EXAMPLE_USER_PASSWORD}"
+        run: |
+          qhub keycloak --config "${QHUB_CONFIG_PATH}" adduser example-user "${CYPRESS_EXAMPLE_USER_PASSWORD}"
+          qhub keycloak --config "${QHUB_CONFIG_PATH}" listusers
 
       - name: Sleep to see if it fixes flaky Kubernetes Tests
         run: sleep 60
@@ -153,6 +155,9 @@ jobs:
 
       - name: Deployment Pytests
         run: |
+          export KEYCLOAK_USERNAME=qhub-user
+          export KEYCLOAK_PASSWORD=password
+          qhub keycloak --config "${QHUB_CONFIG_PATH}" adduser $KEYCLOAK_USERNAME $KEYCLOAK_PASSWORD
           pytest tests_deployment/ -v
 
       ### CLEANUP AFTER TESTS

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ repos:
     rev: v2.3.0
     hooks:
     -   id: check-yaml
-        exclude: 'qhub/template/\{\{ cookiecutter\.repo_directory \}\}/(\.github/.*|infrastructure/jupyterhub.yaml)'
+        exclude: 'qhub/template/(\.github/.*|infrastructure/jupyterhub.yaml|stages/07-kubernetes-services/modules/kubernetes/services/(clearml/chart/templates/.*|prefect/chart/templates/.*))'
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
   - repo: https://github.com/psf/black
@@ -14,7 +14,10 @@ repos:
     rev: 3.8.4
     hooks:
     -   id: flake8
-        exclude: 'qhub/template/\{\{ cookiecutter\.repo_directory \}\}/infrastructure/modules/kubernetes/services/conda-store/config/conda_store_config.py'
+        exclude: 'qhub/template/stages/07-kubernetes-services/modules/kubernetes/services/conda-store/config/conda_store_config.py'
+        args: [
+          "--builtins=c"
+        ]
   - repo: https://github.com/hadolint/hadolint.git
     rev: v2.3.0
     hooks:

docs/source/user_guide/experimental.md

@@ -3,38 +3,3 @@
 > NOTE: The features listed below are experimental, proceed with caution.
 
 
-## CentOS `jupyterlab` and `dask-worker` profiles
-
-The default images used during a typical QHub deployment are running on `ubuntu`. If you need your `jupyterlab` or `dask-worker` pods to run on `centOS` instead, simply update the appropriate images in your `qhub-config.yaml`.
-
-```
-...
-default_images:
-  jupyterhub: quansight/qhub-jupyterhub:main
-  jupyterlab: quansight/qhub-jupyterlab-centos:main     <--- here
-  dask_worker: quansight/qhub-dask-worker-centos:main   <--- here
-  dask_gateway: quansight/qhub-dask-gateway:main
-  conda_store: quansight/conda-store-server:v0.3.3
-...
-profiles:
-  jupyterlab:
-  - display_name: Small Instance
-    description: Stable environment with 1 cpu / 4 GB ram
-    default: true
-    kubespawner_override:
-      cpu_limit: 0.25
-      cpu_guarantee: 0.25
-      mem_limit: 2G
-      mem_guarantee: 2G
-      image: quansight/qhub-jupyterlab-centos:main      <--- here
-  dask_worker:
-    Small Worker:
-      worker_cores_limit: 0.5
-      worker_cores: 0.5
-      worker_memory_limit: 2G
-      worker_memory: 2G
-      worker_threads: 1
-      image: quansight/qhub-dask-worker-centos:main     <--- here
-```
-
-Then to get these changes to take hold, simply redeploy QHub.

environment-dev.yaml

@@ -2,7 +2,6 @@ name: qhub-dev
 channels:
   - conda-forge
 dependencies:
-  - cookiecutter ==1.7.2
   - ruamel.yaml
   - cloudflare
   - auth0-python

flake.nix

@@ -14,7 +14,6 @@
       keycloak = (import nixpkgs-keycloak { system = "x86_64-linux"; }).python3Packages.python-keycloak;
 
       propagatedDependencies = [
-        pythonPackages.cookiecutter
         pythonPackages.ruamel-yaml
         pythonPackages.cloudflare
         pythonPackages.auth0-python
@@ -53,7 +52,6 @@
 
         patchPhase = ''
           substituteInPlace setup.py \
-            --replace "cookiecutter==1.7.2" "cookiecutter" \
             --replace "azure-identity==1.6.1" "azure-identity" \
             --replace "azure-mgmt-containerservice==16.2.0" "azure-mgmt-containerservice"
         '';

pyproject.toml

@@ -1,4 +1,3 @@
 [tool.black]
 line-length = 88
 target-version = ['py36', 'py37', 'py38']
-exclude = '/\{\{ cookiecutter.repo_directory \}\}/'

qhub/cli/destroy.py

@@ -13,21 +13,11 @@ def create_destroy_subcommand(subparser):
     subparser = subparser.add_parser("destroy")
     subparser.add_argument("-c", "--config", help="qhub configuration", required=True)
     subparser.add_argument("-o", "--output", default="./", help="output directory")
-    subparser.add_argument(
-        "--skip-remote-state-provision",
-        action="store_true",
-        help="Skip terraform state import and destroy",
-    )
     subparser.add_argument(
         "--disable-render",
         action="store_true",
         help="Disable auto-rendering before destroy",
     )
-    subparser.add_argument(
-        "--full-only",
-        action="store_true",
-        help="Only carry out one full pass instead of targeted sections",
-    )
     subparser.set_defaults(func=handle_destroy)
 
 
@@ -45,8 +35,4 @@ def handle_destroy(args):
     if not args.disable_render:
         render_template(args.output, args.config, force=True)
 
-    destroy_configuration(
-        config,
-        args.skip_remote_state_provision,
-        args.full_only,
-    )
+    destroy_configuration(config)

qhub/cli/keycloak.py

@@ -9,7 +9,9 @@ def create_keycloak_subcommand(subparser):
     subparser = subparser.add_parser("keycloak")
     subparser.add_argument("-c", "--config", help="qhub configuration", required=True)
     subparser.add_argument(
-        "keycloak_action", nargs="+", help="adduser <username> [password]"
+        "keycloak_action",
+        nargs="+",
+        help="`adduser <username> [password]` or `listusers`",
     )
     subparser.set_defaults(func=handle_keycloak)
 

qhub/cli/render.py

@@ -11,6 +11,11 @@ def create_render_subcommand(subparser):
     subparser.add_argument(
         "-c", "--config", help="qhub configuration yaml file", required=True
     )
+    subparser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="simulate rendering files without actually writing or updating any files",
+    )
     subparser.set_defaults(func=handle_render)
 
 
@@ -25,4 +30,4 @@ def handle_render(args):
 
     verify(config)
 
-    render_template(args.output, args.config, force=True)
+    render_template(args.output, args.config, force=True, dry_run=args.dry_run)