diff --git a/.github/workflows/kubernetes_test.yaml b/.github/workflows/kubernetes_test.yaml index 39c8ef9dcc..56cf6bdbf1 100644 --- a/.github/workflows/kubernetes_test.yaml +++ b/.github/workflows/kubernetes_test.yaml @@ -51,63 +51,36 @@ jobs: run: | conda install -c anaconda pip pip install .[dev] - - name: Download and Install Minikube and Kubectl + - name: Download and Install Kind and Kubectl run: | mkdir -p bin pushd bin - curl -L https://github.com/kubernetes/minikube/releases/download/v1.22.0/minikube-linux-amd64 -o minikube - chmod +x minikube curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubectl chmod +x kubectl echo "$PWD" >> $GITHUB_PATH popd - - name: Start Minikube + - name: Enable docker permissions for user run: | sudo docker ps sudo usermod -aG docker $USER && newgrp docker - minikube start --kubernetes-version=1.19.4 --driver=docker --cpus 4 --memory 12288 --wait=all - - name: Print minikube and kubectl versions - run: | - minikube version - kubectl version - - name: Use minikube docker daemon - run: | - eval $(minikube docker-env) - echo "DOCKER_TLS_VERIFY=$DOCKER_TLS_VERIFY" >> $GITHUB_ENV - echo "DOCKER_HOST=$DOCKER_HOST" >> $GITHUB_ENV - echo "DOCKER_CERT_PATH=$DOCKER_CERT_PATH" >> $GITHUB_ENV - echo "MINIKUBE_ACTIVE_DOCKERD=$MINIKUBE_ACTIVE_DOCKERD" >> $GITHUB_ENV - - name: Print docker connection information - run: | + docker info docker ps - - name: List docker images in minikube - run: | - docker images - name: Get routing table for docker pods run: | ip route - - name: Configure LoadBalancer IPs - run: | - python tests/scripts/minikube-loadbalancer-ip.py - name: Add DNS entry to hosts run: | - sudo echo "192.168.49.100 github-actions.qhub.dev" | sudo tee -a /etc/hosts - - name: Enable Minikube metallb - run: | - minikube addons enable metallb - - name: Basic kubectl checks before deployment - run: | - kubectl get all,cm,secret,ing -A + sudo echo "172.18.1.100 github-actions.qhub.dev" | sudo tee -a /etc/hosts - name: Initialize QHub Cloud run: | mkdir -p local-deployment cd local-deployment qhub init local --project=thisisatest --domain github-actions.qhub.dev --auth-provider=password - # Need smaller profiles on Minikube + # Need smaller profiles on Local Kind sed -i -E 's/(cpu_guarantee):\s+[0-9\.]+/\1: 0.25/g' "qhub-config.yaml" sed -i -E 's/(mem_guarantee):\s+[A-Za-z0-9\.]+/\1: 0.25G/g' "qhub-config.yaml" @@ -194,13 +167,3 @@ jobs: run: | cd local-deployment qhub destroy --config qhub-config.yaml - - - name: Basic kubectl checks after cleanup - if: always() - run: | - kubectl get all,cm,secret,ing -A - - - name: Delete minikube cluster - if: always() - run: | - minikube delete diff --git a/.github/workflows/test-provider.yaml b/.github/workflows/test-provider.yaml index 5044647568..26838773b5 100644 --- a/.github/workflows/test-provider.yaml +++ b/.github/workflows/test-provider.yaml @@ -46,6 +46,7 @@ jobs: - do - gcp - local + - existing cicd: - none - github-actions diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d6cf16ba74..fd02da0a10 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,6 +14,7 @@ repos: - id: check-yaml # jinja2 templates for helm charts exclude: 'qhub/template/stages/07-kubernetes-services/modules/kubernetes/services/(clearml/chart/templates/.*|prefect/chart/templates/.*)' + args: [--allow-multiple-documents] - repo: https://github.com/codespell-project/codespell rev: v2.1.0 diff --git a/qhub/initialize.py b/qhub/initialize.py index 3a36b28c38..c213e38086 100644 --- a/qhub/initialize.py +++ b/qhub/initialize.py @@ -119,6 +119,23 @@ } } +EXISTING = { + "node_selectors": { + "general": { + "key": "kubernetes.io/os", + "value": "linux", + }, + "user": { + "key": "kubernetes.io/os", + "value": "linux", + }, + "worker": { + "key": "kubernetes.io/os", + "value": "linux", + }, + } +} + DIGITAL_OCEAN = { "region": "nyc3", "kubernetes_version": "PLACEHOLDER", @@ -390,6 +407,13 @@ def render_config( set_kubernetes_version(config, kubernetes_version, cloud_provider) if "AWS_DEFAULT_REGION" in os.environ: config["amazon_web_services"]["region"] = os.environ["AWS_DEFAULT_REGION"] + + elif cloud_provider == "existing": + config["theme"]["jupyterhub"][ + "hub_subtitle" + ] = "Autoscaling Compute Environment" + config["existing"] = EXISTING.copy() + elif cloud_provider == "local": config["theme"]["jupyterhub"][ "hub_subtitle" diff --git a/qhub/provider/cicd/github.py b/qhub/provider/cicd/github.py index ee8f09ccbb..a1a961a3cc 100644 --- a/qhub/provider/cicd/github.py +++ b/qhub/provider/cicd/github.py @@ -148,7 +148,7 @@ def gha_env_vars(config): env_vars["DIGITALOCEAN_TOKEN"] = "${{ secrets.DIGITALOCEAN_TOKEN }}" elif config["provider"] == "gcp": env_vars["GOOGLE_CREDENTIALS"] = "${{ secrets.GOOGLE_CREDENTIALS }}" - elif config["provider"] == "local": + elif config["provider"] in ["local", "existing"]: # create mechanism to allow for extra env vars? pass else: diff --git a/qhub/schema.py b/qhub/schema.py index 5838d2b405..ea2d9622f6 100644 --- a/qhub/schema.py +++ b/qhub/schema.py @@ -24,6 +24,7 @@ class TerraformStateEnum(str, enum.Enum): class ProviderEnum(str, enum.Enum): local = "local" + existing = "existing" do = "do" aws = "aws" gcp = "gcp" @@ -311,6 +312,11 @@ class LocalProvider(Base): node_selectors: typing.Dict[str, KeyValueDict] +class ExistingProvider(Base): + kube_context: typing.Optional[str] + node_selectors: typing.Dict[str, KeyValueDict] + + # ================= Theme ================== @@ -488,6 +494,7 @@ class Main(Base): default_images: DefaultImages storage: typing.Dict[str, str] local: typing.Optional[LocalProvider] + existing: typing.Optional[ExistingProvider] google_cloud_platform: typing.Optional[GoogleCloudPlatformProvider] amazon_web_services: typing.Optional[AmazonWebServicesProvider] azure: typing.Optional[AzureProvider] diff --git a/qhub/stages/input_vars.py b/qhub/stages/input_vars.py index 3563922242..ed89ecdfc2 100644 --- a/qhub/stages/input_vars.py +++ b/qhub/stages/input_vars.py @@ -36,7 +36,14 @@ def stage_01_terraform_state(stage_outputs, config): def stage_02_infrastructure(stage_outputs, config): if config["provider"] == "local": - return {"kube_context": config["local"].get("kube_context")} + return { + "kubeconfig_filename": os.path.join( + tempfile.gettempdir(), "QHUB_KUBECONFIG" + ), + "kube_context": config["local"].get("kube_context"), + } + elif config["provider"] == "existing": + return {"kube_context": config["existing"].get("kube_context")} elif config["provider"] == "do": return { "name": config["project_name"], @@ -165,6 +172,8 @@ def _calculate_note_groups(config): group: {"key": "doks.digitalocean.com/node-pool", "value": group} for group in ["general", "user", "worker"] } + elif config["provider"] == "existing": + return config["existing"].get("node_selectors") else: return config["local"]["node_selectors"] diff --git a/qhub/stages/tf_objects.py b/qhub/stages/tf_objects.py index 0f154cc6bd..8792c8d218 100644 --- a/qhub/stages/tf_objects.py +++ b/qhub/stages/tf_objects.py @@ -107,6 +107,16 @@ def QHubTerraformState(directory: str, qhub_config: Dict): container_name=f"{qhub_config['project_name']}-{qhub_config['namespace']}-state", key=f"terraform/{qhub_config['project_name']}-{qhub_config['namespace']}/{directory}", ) + elif qhub_config["provider"] == "existing": + optional_kwargs = {} + if "kube_context" in qhub_config["existing"]: + optional_kwargs["confix_context"] = qhub_config["existing"]["kube_context"] + return TerraformBackend( + "kubernetes", + secret_suffix=f"{qhub_config['project_name']}-{qhub_config['namespace']}-{directory}", + load_config_file=True, + **optional_kwargs, + ) elif qhub_config["provider"] == "local": optional_kwargs = {} if "kube_context" in qhub_config["local"]: diff --git a/qhub/template/stages/02-infrastructure/existing/main.tf b/qhub/template/stages/02-infrastructure/existing/main.tf new file mode 100644 index 0000000000..4383e8294c --- /dev/null +++ b/qhub/template/stages/02-infrastructure/existing/main.tf @@ -0,0 +1,17 @@ +variable "kube_context" { + description = "Optional kubernetes context to use to connect to kubernetes cluster" + type = string +} + +output "kubernetes_credentials" { + description = "Parameters needed to connect to kubernetes cluster locally" + value = { + config_path = pathexpand("~/.kube/config") + config_context = var.kube_context + } +} + +output "kubeconfig_filename" { + description = "filename for qhub kubeconfig" + value = pathexpand("~/.kube/config") +} diff --git a/qhub/template/stages/02-infrastructure/local/main.tf b/qhub/template/stages/02-infrastructure/local/main.tf index 4383e8294c..9fd8bb2618 100644 --- a/qhub/template/stages/02-infrastructure/local/main.tf +++ b/qhub/template/stages/02-infrastructure/local/main.tf @@ -1,17 +1,111 @@ -variable "kube_context" { - description = "Optional kubernetes context to use to connect to kubernetes cluster" - type = string +terraform { + required_providers { + kind = { + source = "kyma-incubator/kind" + version = "0.0.11" + } + docker = { + source = "kreuzwerker/docker" + version = "2.16.0" + } + kubectl = { + source = "gavinbunney/kubectl" + version = ">= 1.7.0" + } + } +} + +provider "kind" { + +} + +provider "docker" { + } -output "kubernetes_credentials" { - description = "Parameters needed to connect to kubernetes cluster locally" - value = { - config_path = pathexpand("~/.kube/config") - config_context = var.kube_context +provider "kubernetes" { + host = kind_cluster.default.endpoint + cluster_ca_certificate = kind_cluster.default.cluster_ca_certificate + client_key = kind_cluster.default.client_key + client_certificate = kind_cluster.default.client_certificate +} + +provider "kubectl" { + load_config_file = false + host = kind_cluster.default.endpoint + cluster_ca_certificate = kind_cluster.default.cluster_ca_certificate + client_key = kind_cluster.default.client_key + client_certificate = kind_cluster.default.client_certificate +} + +resource "kind_cluster" "default" { + name = "test-cluster" + wait_for_ready = true + + kind_config { + kind = "Cluster" + api_version = "kind.x-k8s.io/v1alpha4" + + node { + role = "general" + image = "kindest/node:v1.21.10" + } } } -output "kubeconfig_filename" { - description = "filename for qhub kubeconfig" - value = pathexpand("~/.kube/config") +resource "kubernetes_namespace" "metallb" { + metadata { + name = "metallb-system" + } +} + +data "kubectl_path_documents" "metallb" { + pattern = "${path.module}/metallb.yaml" +} + +resource "kubectl_manifest" "metallb" { + for_each = toset(data.kubectl_path_documents.metallb.documents) + yaml_body = each.value + wait = true + depends_on = [kubernetes_namespace.metallb] +} + +resource "kubectl_manifest" "load-balancer" { + yaml_body = yamlencode({ + apiVersion = "v1" + kind = "ConfigMap" + metadata = { + namespace = kubernetes_namespace.metallb.metadata.0.name + name = "config" + } + data = { + config = yamlencode({ + address-pools = [{ + name = "default" + protocol = "layer2" + addresses = [ + "${local.metallb_ip_min}-${local.metallb_ip_max}" + ] + }] + }) + } + }) + + depends_on = [kubectl_manifest.metallb] +} + +data "docker_network" "kind" { + name = "kind" + + depends_on = [kind_cluster.default] +} + +locals { + metallb_ip_min = cidrhost([ + for network in data.docker_network.kind.ipam_config : network if network.gateway != "" + ][0].subnet, 356) + + metallb_ip_max = cidrhost([ + for network in data.docker_network.kind.ipam_config : network if network.gateway != "" + ][0].subnet, 406) } diff --git a/qhub/template/stages/02-infrastructure/local/metallb.yaml b/qhub/template/stages/02-infrastructure/local/metallb.yaml new file mode 100644 index 0000000000..9d6b6833c8 --- /dev/null +++ b/qhub/template/stages/02-infrastructure/local/metallb.yaml @@ -0,0 +1,480 @@ +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + labels: + app: metallb + name: controller +spec: + allowPrivilegeEscalation: false + allowedCapabilities: [] + allowedHostPaths: [] + defaultAddCapabilities: [] + defaultAllowPrivilegeEscalation: false + fsGroup: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + hostIPC: false + hostNetwork: false + hostPID: false + privileged: false + readOnlyRootFilesystem: true + requiredDropCapabilities: + - ALL + runAsUser: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + seLinux: + rule: RunAsAny + supplementalGroups: + ranges: + - max: 65535 + min: 1 + rule: MustRunAs + volumes: + - configMap + - secret + - emptyDir +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + labels: + app: metallb + name: speaker +spec: + allowPrivilegeEscalation: false + allowedCapabilities: + - NET_RAW + allowedHostPaths: [] + defaultAddCapabilities: [] + defaultAllowPrivilegeEscalation: false + fsGroup: + rule: RunAsAny + hostIPC: false + hostNetwork: true + hostPID: false + hostPorts: + - max: 7472 + min: 7472 + - max: 7946 + min: 7946 + privileged: true + readOnlyRootFilesystem: true + requiredDropCapabilities: + - ALL + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - configMap + - secret + - emptyDir +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: metallb + name: controller + namespace: metallb-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: metallb + name: speaker + namespace: metallb-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: metallb + name: metallb-system:controller +rules: +- apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - '' + resources: + - services/status + verbs: + - update +- apiGroups: + - '' + resources: + - events + verbs: + - create + - patch +- apiGroups: + - policy + resourceNames: + - controller + resources: + - podsecuritypolicies + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: metallb + name: metallb-system:speaker +rules: +- apiGroups: + - '' + resources: + - services + - endpoints + - nodes + verbs: + - get + - list + - watch +- apiGroups: ["discovery.k8s.io"] + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - '' + resources: + - events + verbs: + - create + - patch +- apiGroups: + - policy + resourceNames: + - speaker + resources: + - podsecuritypolicies + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app: metallb + name: config-watcher + namespace: metallb-system +rules: +- apiGroups: + - '' + resources: + - configmaps + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app: metallb + name: pod-lister + namespace: metallb-system +rules: +- apiGroups: + - '' + resources: + - pods + verbs: + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app: metallb + name: controller + namespace: metallb-system +rules: +- apiGroups: + - '' + resources: + - secrets + verbs: + - create +- apiGroups: + - '' + resources: + - secrets + resourceNames: + - memberlist + verbs: + - list +- apiGroups: + - apps + resources: + - deployments + resourceNames: + - controller + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: metallb + name: metallb-system:controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metallb-system:controller +subjects: +- kind: ServiceAccount + name: controller + namespace: metallb-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: metallb + name: metallb-system:speaker +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metallb-system:speaker +subjects: +- kind: ServiceAccount + name: speaker + namespace: metallb-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app: metallb + name: config-watcher + namespace: metallb-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: config-watcher +subjects: +- kind: ServiceAccount + name: controller +- kind: ServiceAccount + name: speaker +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app: metallb + name: pod-lister + namespace: metallb-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pod-lister +subjects: +- kind: ServiceAccount + name: speaker +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app: metallb + name: controller + namespace: metallb-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: controller +subjects: +- kind: ServiceAccount + name: controller +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app: metallb + component: speaker + name: speaker + namespace: metallb-system +spec: + selector: + matchLabels: + app: metallb + component: speaker + template: + metadata: + annotations: + prometheus.io/port: '7472' + prometheus.io/scrape: 'true' + labels: + app: metallb + component: speaker + spec: + containers: + - args: + - --port=7472 + - --config=config + - --log-level=info + env: + - name: METALLB_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: METALLB_HOST + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: METALLB_ML_BIND_ADDR + valueFrom: + fieldRef: + fieldPath: status.podIP + # needed when another software is also using memberlist / port 7946 + # when changing this default you also need to update the container ports definition + # and the PodSecurityPolicy hostPorts definition + #- name: METALLB_ML_BIND_PORT + # value: "7946" + - name: METALLB_ML_LABELS + value: "app=metallb,component=speaker" + - name: METALLB_ML_SECRET_KEY + valueFrom: + secretKeyRef: + name: memberlist + key: secretkey + image: quay.io/metallb/speaker:v0.12.1 + name: speaker + ports: + - containerPort: 7472 + name: monitoring + - containerPort: 7946 + name: memberlist-tcp + - containerPort: 7946 + name: memberlist-udp + protocol: UDP + livenessProbe: + httpGet: + path: /metrics + port: monitoring + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /metrics + port: monitoring + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_RAW + drop: + - ALL + readOnlyRootFilesystem: true + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + serviceAccountName: speaker + terminationGracePeriodSeconds: 2 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: metallb + component: controller + name: controller + namespace: metallb-system +spec: + revisionHistoryLimit: 3 + selector: + matchLabels: + app: metallb + component: controller + template: + metadata: + annotations: + prometheus.io/port: '7472' + prometheus.io/scrape: 'true' + labels: + app: metallb + component: controller + spec: + containers: + - args: + - --port=7472 + - --config=config + - --log-level=info + env: + - name: METALLB_ML_SECRET_NAME + value: memberlist + - name: METALLB_DEPLOYMENT + value: controller + image: quay.io/metallb/controller:v0.12.1 + name: controller + ports: + - containerPort: 7472 + name: monitoring + livenessProbe: + httpGet: + path: /metrics + port: monitoring + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /metrics + port: monitoring + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + readOnlyRootFilesystem: true + nodeSelector: + kubernetes.io/os: linux + securityContext: + runAsNonRoot: true + runAsUser: 65534 + fsGroup: 65534 + serviceAccountName: controller + terminationGracePeriodSeconds: 0 diff --git a/qhub/template/stages/02-infrastructure/local/outputs.tf b/qhub/template/stages/02-infrastructure/local/outputs.tf new file mode 100644 index 0000000000..bb3134b493 --- /dev/null +++ b/qhub/template/stages/02-infrastructure/local/outputs.tf @@ -0,0 +1,20 @@ +output "kubernetes_credentials" { + description = "Parameters needed to connect to kubernetes cluster locally" + sensitive = true + value = { + host = kind_cluster.default.endpoint + cluster_ca_certificate = kind_cluster.default.cluster_ca_certificate + client_key = kind_cluster.default.client_key + client_certificate = kind_cluster.default.client_certificate + } +} + +resource "local_file" "default" { + content = kind_cluster.default.kubeconfig + filename = var.kubeconfig_filename +} + +output "kubeconfig_filename" { + description = "filename for qhub kubeconfig" + value = var.kubeconfig_filename +} diff --git a/qhub/template/stages/02-infrastructure/local/variables.tf b/qhub/template/stages/02-infrastructure/local/variables.tf new file mode 100644 index 0000000000..097bb19598 --- /dev/null +++ b/qhub/template/stages/02-infrastructure/local/variables.tf @@ -0,0 +1,10 @@ +variable "kubeconfig_filename" { + description = "Kubernetes kubeconfig written to filesystem" + type = string + default = null +} + +variable "kube_context" { + description = "Optional kubernetes context to use to connect to kubernetes cluster" + type = string +} diff --git a/qhub/utils.py b/qhub/utils.py index 7691f24915..6dbbc414a5 100644 --- a/qhub/utils.py +++ b/qhub/utils.py @@ -179,7 +179,7 @@ def check_cloud_credentials(config): f"""The environment variables AWS_SECRET_ACCESS_KEY and SPACES_SECRET_ACCESS_KEY must be equal\n See {DO_ENV_DOCS} for more information""" ) - elif config["provider"] == "local": + elif config["provider"] in ["local", "existing"]: pass else: raise ValueError("Cloud Provider configuration not supported")