External Container Registry

docs/source/admin_guide/troubleshooting.md

@@ -130,12 +130,21 @@ command and open your web browser to `localhost:8000`.
 ```shell
 docker run -p 8000:8000 -it Quansight/qhub-jupyterlab:latest jupyter lab --port 8000 --ip 0.0.0.0
 ```
----
 
-### Useful Kubernetes commands
+### Using a Private AWS ECR Container Registry
+
+By default, images such as the default JupyterLab image specified as `quansight/qhub-jupyterhub:v||QHUB_VERSION||` will be pulled from Docker Hub.
+
+To specify a private AWS ECR (and this technique should work regardless of which cloud your QHub is deployed to), first provide details of the ECR and AWS access keys in `qhub-config.yaml`:
+
+```yaml
+external_container_reg:
+  enabled: true
+  access_key_id: <AWS access key id>
+  secret_access_key: <AWS secret key>
+  extcr_account: 12345678
+  extcr_region: us-west-1
+```
+
+This will mean you can specify private Docker images such as `12345678.dkr.ecr.us-west-1.amazonaws.com/quansight/qhub-jupyterlab:mytag` in your `qhub-config.yaml` file. The AWS key and secret provided must have relevant ecr IAMS permissions to authenticate and read from the ECR container registry.
 
-### Integrations
-#### Prefect
-TODO
-#### Bodo
-TODO

qhub/schema.py

@@ -2,7 +2,7 @@
 import typing
 
 import pydantic
-from pydantic import validator
+from pydantic import validator, root_validator
 from qhub.utils import namestr_regex, check_for_duplicates
 
 
@@ -295,6 +295,43 @@ class CDSDashboards(Base):
     cds_hide_user_dashboard_servers: typing.Optional[bool]
 
 
+# ======== External Container Registry ========
+
+# This allows the user to set a private AWS ECR as a replacement for
+# Docker Hub for some images - those where you provide the full path
+# to the image on the ECR.
+# extcr_account and extcr_region are the AWS account number and region
+# of the ECR respectively. access_key_id and secret_access_key are
+# AWS access keys that should have read access to the ECR.
+
+
+class ExtContainerReg(Base):
+    enabled: bool
+    access_key_id: typing.Optional[str]
+    secret_access_key: typing.Optional[str]
+    extcr_account: typing.Optional[str]
+    extcr_region: typing.Optional[str]
+
+    @root_validator
+    def enabled_must_have_fields(cls, values):
+        if values["enabled"]:
+            for fldname in (
+                "access_key_id",
+                "secret_access_key",
+                "extcr_account",
+                "extcr_region",
+            ):
+                if (
+                    fldname not in values
+                    or values[fldname] is None
+                    or values[fldname].strip() == ""
+                ):
+                    raise ValueError(
+                        f"external_container_reg must contain a non-blank {fldname} when enabled is true"
+                    )
+        return values
+
+
 # ==================== Main ===================
 
 letter_dash_underscore_pydantic = pydantic.constr(regex=namestr_regex)
@@ -314,6 +351,7 @@ class Main(Base):
     prefect: typing.Optional[Prefect]
     cdsdashboards: CDSDashboards
     security: Security
+    external_container_reg: typing.Optional[ExtContainerReg]
     default_images: DefaultImages
     storage: typing.Dict[str, str]
     local: typing.Optional[LocalProvider]

qhub/template/cookiecutter.json

@@ -31,6 +31,7 @@
     "environments": null,
     "storage": null,
     "cdsdashboards": {},
+    "external_container_reg": {},
 
     "local": {
         "node_selectors": {

qhub/template/{{ cookiecutter.repo_directory }}/infrastructure/kubernetes.tf

@@ -209,6 +209,14 @@ module "qhub" {
   forwardauth-jh-client-secret  = random_password.forwardauth-jhsecret.result
   forwardauth-callback-url-path = local.forwardauth-callback-url-path
 
+  extcr_config = {
+    enabled : {{ cookiecutter.external_container_reg.enabled | default(false,true) | jsonify }}
+    access_key_id : "{{ cookiecutter.external_container_reg.access_key_id | default("",true) }}"
+    secret_access_key : "{{ cookiecutter.external_container_reg.secret_access_key | default("",true) }}"
+    extcr_account : "{{ cookiecutter.external_container_reg.extcr_account | default("",true) }}"
+    extcr_region : "{{ cookiecutter.external_container_reg.extcr_region | default("",true) }}"
+  }
+
   depends_on = [
     module.kubernetes-ingress
   ]

qhub/template/{{ cookiecutter.repo_directory }}/infrastructure/modules/kubernetes/services/extcr/main.tf

@@ -0,0 +1,261 @@
+resource "kubernetes_secret" "customer_extcr_key" {
+  metadata {
+    name      = "customer-extcr-key"
+    namespace = var.namespace
+  }
+
+  data = {
+    "access-key-id"     = var.access_key_id
+    "secret-access-key" = var.secret_access_key
+    "extcr-account"     = var.extcr_account
+    "extcr-region"      = var.extcr_region
+  }
+}
+
+resource "kubernetes_manifest" "role_extcr_cred_updater" {
+  provider = kubernetes-alpha
+  manifest = {
+    "apiVersion" = "rbac.authorization.k8s.io/v1"
+    "kind"       = "Role"
+    "metadata" = {
+      "name"      = "extcr-cred-updater"
+      "namespace" = var.namespace
+    }
+    "rules" = [
+      {
+        "apiGroups" = [
+          "",
+        ]
+        "resources" = [
+          "secrets",
+        ]
+        "verbs" = [
+          "get",
+          "create",
+          "delete",
+        ]
+      },
+      {
+        "apiGroups" = [
+          "",
+        ]
+        "resources" = [
+          "serviceaccounts",
+        ]
+        "verbs" = [
+          "get",
+          "patch",
+        ]
+      },
+    ]
+  }
+}
+
+resource "kubernetes_manifest" "serviceaccount_extcr_cred_updater" {
+  provider = kubernetes-alpha
+  manifest = {
+    "apiVersion" = "v1"
+    "kind"       = "ServiceAccount"
+    "metadata" = {
+      "name"      = "extcr-cred-updater"
+      "namespace" = var.namespace
+    }
+  }
+}
+
+resource "kubernetes_manifest" "rolebinding_extcr_cred_updater" {
+  provider = kubernetes-alpha
+  manifest = {
+    "apiVersion" = "rbac.authorization.k8s.io/v1"
+    "kind"       = "RoleBinding"
+    "metadata" = {
+      "name"      = "extcr-cred-updater"
+      "namespace" = var.namespace
+    }
+    "roleRef" = {
+      "apiGroup" = "rbac.authorization.k8s.io"
+      "kind"     = "Role"
+      "name"     = "extcr-cred-updater"
+    }
+    "subjects" = [
+      {
+        "kind" = "ServiceAccount"
+        "name" = "extcr-cred-updater"
+      },
+    ]
+  }
+}
+
+resource "kubernetes_manifest" "job_extcr_cred_updater" {
+  provider = kubernetes-alpha
+  manifest = {
+    "apiVersion" = "batch/v1"
+    "kind"       = "Job"
+    "metadata" = {
+      "name"      = "extcr-cred-updater"
+      "namespace" = var.namespace
+    }
+    "spec" = {
+      "backoffLimit" = 4
+      "template" = {
+        "spec" = {
+          "containers" = [
+            {
+              "command" = [
+                "/bin/sh",
+                "-c",
+                <<-EOT
+                DOCKER_REGISTRY_SERVER=https://$${AWS_ACCOUNT}.dkr.ecr.$${AWS_REGION}.amazonaws.com
+                DOCKER_USER=AWS
+                DOCKER_PASSWORD=`aws ecr get-login --region $${AWS_REGION} --registry-ids $${AWS_ACCOUNT} | cut -d' ' -f6`
+                kubectl delete secret extcrcreds || true
+                kubectl create secret docker-registry extcrcreds \
+                --docker-server=$DOCKER_REGISTRY_SERVER \
+                --docker-username=$DOCKER_USER \
+                --docker-password=$DOCKER_PASSWORD \
+                --docker-email=no@email.local
+                kubectl patch serviceaccount default -p '{"imagePullSecrets":[{"name":"extcrcreds"}]}'
+
+                EOT
+                ,
+              ]
+              "env" = [
+                {
+                  "name" = "AWS_ACCESS_KEY_ID"
+                  "valueFrom" = {
+                    "secretKeyRef" = {
+                      "key"  = "access-key-id"
+                      "name" = "customer-extcr-key"
+                    }
+                  }
+                },
+                {
+                  "name" = "AWS_SECRET_ACCESS_KEY"
+                  "valueFrom" = {
+                    "secretKeyRef" = {
+                      "key"  = "secret-access-key"
+                      "name" = "customer-extcr-key"
+                    }
+                  }
+                },
+                {
+                  "name" = "AWS_ACCOUNT"
+                  "valueFrom" = {
+                    "secretKeyRef" = {
+                      "key"  = "extcr-account"
+                      "name" = "customer-extcr-key"
+                    }
+                  }
+                },
+                {
+                  "name" = "AWS_REGION"
+                  "valueFrom" = {
+                    "secretKeyRef" = {
+                      "key"  = "extcr-region"
+                      "name" = "customer-extcr-key"
+                    }
+                  }
+                },
+              ]
+              "image" = "xynova/aws-kubectl"
+              "name"  = "kubectl"
+            },
+          ]
+          "restartPolicy"                 = "Never"
+          "serviceAccountName"            = "extcr-cred-updater"
+          "terminationGracePeriodSeconds" = 0
+        }
+      }
+    }
+  }
+}
+
+resource "kubernetes_manifest" "cronjob_extcr_cred_updater" {
+  provider = kubernetes-alpha
+  manifest = {
+    "apiVersion" = "batch/v1beta1"
+    "kind"       = "CronJob"
+    "metadata" = {
+      "name"      = "extcr-cred-updater"
+      "namespace" = var.namespace
+    }
+    "spec" = {
+      "failedJobsHistoryLimit" = 1
+      "jobTemplate" = {
+        "spec" = {
+          "backoffLimit" = 4
+          "template" = {
+            "spec" = {
+              "containers" = [
+                {
+                  "command" = [
+                    "/bin/sh",
+                    "-c",
+                    <<-EOT
+                    DOCKER_REGISTRY_SERVER=https://$${AWS_ACCOUNT}.dkr.ecr.$${AWS_REGION}.amazonaws.com
+                    DOCKER_USER=AWS
+                    DOCKER_PASSWORD=`aws ecr get-login --region $${AWS_REGION} --registry-ids $${AWS_ACCOUNT} | cut -d' ' -f6`
+                    kubectl delete secret extcrcreds || true
+                    kubectl create secret docker-registry extcrcreds \
+                    --docker-server=$DOCKER_REGISTRY_SERVER \
+                    --docker-username=$DOCKER_USER \
+                    --docker-password=$DOCKER_PASSWORD \
+                    --docker-email=no@email.local
+                    kubectl patch serviceaccount default -p '{"imagePullSecrets":[{"name":"extcrcreds"}]}'
+                    EOT
+                    ,
+                  ]
+                  "env" = [
+                    {
+                      "name" = "AWS_ACCESS_KEY_ID"
+                      "valueFrom" = {
+                        "secretKeyRef" = {
+                          "key"  = "access-key-id"
+                          "name" = "customer-extcr-key"
+                        }
+                      }
+                    },
+                    {
+                      "name" = "AWS_SECRET_ACCESS_KEY"
+                      "valueFrom" = {
+                        "secretKeyRef" = {
+                          "key"  = "secret-access-key"
+                          "name" = "customer-extcr-key"
+                        }
+                      }
+                    },
+                    {
+                      "name" = "AWS_ACCOUNT"
+                      "valueFrom" = {
+                        "secretKeyRef" = {
+                          "key"  = "extcr-account"
+                          "name" = "customer-extcr-key"
+                        }
+                      }
+                    },
+                    {
+                      "name" = "AWS_REGION"
+                      "valueFrom" = {
+                        "secretKeyRef" = {
+                          "key"  = "extcr-region"
+                          "name" = "customer-extcr-key"
+                        }
+                      }
+                    },
+                  ]
+                  "image" = "xynova/aws-kubectl"
+                  "name"  = "kubectl"
+                },
+              ]
+              "restartPolicy"                 = "Never"
+              "serviceAccountName"            = "extcr-cred-updater"
+              "terminationGracePeriodSeconds" = 0
+            }
+          }
+        }
+      }
+      "schedule"                   = "* */8 * * *"
+      "successfulJobsHistoryLimit" = 1
+    }
+  }
+}

qhub/template/{{ cookiecutter.repo_directory }}/infrastructure/modules/kubernetes/services/extcr/variables.tf

@@ -0,0 +1,24 @@
+variable "namespace" {
+  description = "namespace to deploy extcr"
+  type        = string
+}
+
+variable "access_key_id" {
+  description = "Customer's access key id for external container reg"
+  type        = string
+}
+
+variable "secret_access_key" {
+  description = "Customer's secret access key for external container reg"
+  type        = string
+}
+
+variable "extcr_account" {
+  description = "AWS Account of the external container reg"
+  type        = string
+}
+
+variable "extcr_region" {
+  description = "AWS Region of the external container reg"
+  type        = string
+}