Merge pull request #429 from yuvipanda/grenoble-1

Refactor GCP terraform code + add  MOEM-IGE hub

.readthedocs.yaml

@@ -0,0 +1,11 @@
+version: 2
+
+conda:
+  environment: docs/environment.yml
+
+build:
+  image: latest
+
+python:
+  version: 3.8
+  install: []

config/hubs/2i2c.cluster.yaml

@@ -6,6 +6,16 @@ gcp:
   project: two-eye-two-see
   cluster: pilot-hubs-cluster
   zone: us-central1-b
+support:
+  config:
+    grafana:
+      ingress:
+        hosts:
+          - grafana.pilot.2i2c.cloud
+        tls:
+          - secretName: grafana-tls
+            hosts:
+              - grafana.pilot.2i2c.cloud
 hubs:
   - name: staging
     domain: staging.pilot.2i2c.cloud

config/hubs/meom-ige.cluster.yaml

@@ -0,0 +1,144 @@
+name: meom-ige
+provider: gcp
+gcp:
+  key: secrets/meom.json
+  project: meom-ige-cnrs
+  cluster: meom-ige-cluster
+  zone: us-central1-b
+hubs:
+  - name: staging
+    domain: staging.meom-ige.2i2c.cloud
+    template: daskhub
+    auth0:
+      connection: github
+    config: &meomConfig
+      basehub:
+        nfsPVC:
+          nfs:
+            # from https://docs.aws.amazon.com/efs/latest/ug/mounting-fs-nfs-mount-settings.html
+            mountOptions:
+            - rsize=1048576
+            - wsize=1048576
+            - timeo=600
+            - soft # We pick soft over hard, so NFS lockups don't lead to hung processes
+            - retrans=2
+            - noresvport
+            serverIP: nfs-server-01
+            baseShareName: /export/home-01/homes/
+        jupyterhub:
+          custom:
+            homepage:
+              templateVars:
+                org:
+                  name: "SWOT Ocean Pangeo Team"
+                  logo_url: https://2i2c.org/media/logo.png
+                  url: https://meom-group.github.io/
+                designed_by:
+                  name: 2i2c
+                  url: https://2i2c.org
+                operated_by:
+                  name: 2i2c
+                  url: https://2i2c.org
+                funded_by:
+                  name: SWOT Ocean Pangeo Team
+                  url: https://meom-group.github.io/
+          singleuser:
+            extraEnv:
+              DATA_BUCKET: gcs://meom-ige-data
+              SCRATCH_BUCKET: 'gcs://meom-ige-scratch/$(JUPYTERHUB_USER)'
+            profileList:
+              # The mem-guarantees are here so k8s doesn't schedule other pods
+              # on these nodes. They need to be just under total allocatable
+              # RAM on a node, not total node capacity
+              - display_name: "Small"
+                description: "~2 CPU, ~8G RAM"
+                kubespawner_override:
+                  mem_limit: 8G
+                  mem_guarantee: 5.5G
+                  node_selector:
+                    node.kubernetes.io/instance-type: e2-standard-2
+              - display_name: "Medium"
+                description: "~8 CPU, ~32G RAM"
+                kubespawner_override:
+                  mem_limit: 32G
+                  mem_guarantee: 25G
+                  node_selector:
+                    node.kubernetes.io/instance-type: e2-standard-8
+              - display_name: "Large"
+                description: "~16 CPU, ~64G RAM"
+                kubespawner_override:
+                  mem_limit: 64G
+                  mem_guarantee: 55G
+                  node_selector:
+                    node.kubernetes.io/instance-type: e2-standard-16
+              - display_name: "Very Large"
+                description: "~32 CPU, ~128G RAM"
+                kubespawner_override:
+                  mem_limit: 128G
+                  mem_guarantee: 115G
+                  node_selector:
+                    node.kubernetes.io/instance-type: e2-standard-32
+              - display_name: "Huge"
+                description: "~64 CPU, ~256G RAM"
+                kubespawner_override:
+                  mem_limit: 256G
+                  mem_guarantee: 230G
+                  node_selector:
+                    node.kubernetes.io/instance-type: n2-standard-64
+            defaultUrl: /lab
+            image:
+              name: pangeo/pangeo-notebook
+              tag: 2021.02.19
+          scheduling:
+            userPlaceholder:
+              enabled: false
+              replicas: 0
+            userScheduler:
+              enabled: false
+          proxy:
+            service:
+              type: LoadBalancer
+            https:
+              enabled: true
+            chp:
+              resources:
+                requests:
+                  # FIXME: We want no guarantees here!!!
+                  # This is lowest possible value
+                  cpu: 0.01
+                  memory: 1Mi
+          hub:
+            resources:
+              requests:
+                # FIXME: We want no guarantees here!!!
+                # This is lowest possible value
+                cpu: 0.01
+                memory: 1Mi
+            config:
+              Authenticator:
+                allowed_users: &users
+                  - roxyboy
+                  - lesommer
+                  - auraoupa
+                  - yuvipanda
+                  - choldgraf
+                  - GeorgianaElena
+                admin_users: *users
+
+            allowNamedServers: true
+            networkPolicy:
+              # FIXME: For dask gateway
+              enabled: false
+            readinessProbe:
+              enabled: false
+      dask-gateway:
+          extraConfig:
+            idle: |
+              # timeout after 30 minutes of inactivity
+              c.KubeClusterConfig.idle_timeout = 1800
+  - name: prod
+    domain: meom-ige.2i2c.cloud
+    template: daskhub
+    auth0:
+      connection: github
+    config: *meomConfig

deployer/__main__.py

@@ -29,6 +29,23 @@ def build(cluster_name):
             cluster.build_image()
 
 
+def deploy_support(cluster_name):
+    """
+    Deploy support components to a cluster
+    """
+
+    # Validate our config with JSON Schema first before continuing
+    validate(cluster_name)
+
+
+    config_file_path = Path(os.getcwd()) / "config/hubs" / f'{cluster_name}.cluster.yaml'
+    with open(config_file_path) as f:
+        cluster = Cluster(yaml.load(f))
+
+    if cluster.support:
+        with cluster.auth():
+            cluster.deploy_support()
+
 def deploy(cluster_name, hub_name, skip_hub_health_test, config_path):
     """
     Deploy one or more hubs in a given cluster
@@ -97,6 +114,7 @@ def main():
     build_parser = subparsers.add_parser("build")
     deploy_parser = subparsers.add_parser("deploy")
     validate_parser = subparsers.add_parser("validate")
+    deploy_support_parser = subparsers.add_parser("deploy-support")
 
     build_parser.add_argument("cluster_name")
 
@@ -107,6 +125,8 @@ def main():
 
     validate_parser.add_argument("cluster_name")
 
+    deploy_support_parser.add_argument("cluster_name")
+
     args = argparser.parse_args()
 
     if args.action == "build":
@@ -115,6 +135,8 @@ def main():
         deploy(args.cluster_name, args.hub_name, args.skip_hub_health_test, args.config_path)
     elif args.action == 'validate':
         validate(args.cluster_name)
+    elif args.action == 'deploy-support':
+        deploy_support(args.cluster_name)
     else:
         # Print help message and exit when no arguments are passed
         # FIXME: Is there a better way to do this?

deployer/hub.py

@@ -29,6 +29,7 @@ def __init__(self, spec):
             Hub(self, hub_yaml)
             for hub_yaml in self.spec['hubs']
         ]
+        self.support = self.spec.get('support', {})
 
     def build_image(self):
         self.ensure_docker_credhelpers()
@@ -77,6 +78,32 @@ def ensure_docker_credhelpers(self):
                 with open(dockercfg_path, 'w') as f:
                     json.dump(config, f, indent=4)
 
+    def deploy_support(self):
+        cert_manager_version = 'v1.3.1'
+
+        print("Provisioning cert-manager...")
+        subprocess.check_call([
+            'helm', 'upgrade', '--install', '--create-namespace',
+            '--namespace', 'cert-manager',
+            'cert-manager', 'jetstack/cert-manager',
+            '--version', cert_manager_version,
+            '--set', 'installCRDs=true'
+        ])
+        print("Done!")
+
+        print("Support charts...")
+
+        with tempfile.NamedTemporaryFile(mode='w') as f:
+            yaml.dump(self.support.get('config', {}), f)
+            f.flush()
+            subprocess.check_call([
+                'helm', 'upgrade', '--install', '--create-namespace',
+                '--namespace', 'support',
+                'support', 'support',
+                '-f', f.name,
+                '--wait'
+            ])
+        print("Done!")
 
     def auth_kubeconfig(self):
         """

docs/conf.py

@@ -62,43 +62,48 @@
 from yaml import safe_load
 import pandas as pd
 from pathlib import Path
-
-# Grab the latest list of clusters defined in pilot-hubs/
-clusters = Path("../config/hubs").glob("*")
-# Add list of repos managed outside pilot-hubs
-hub_list = [{
-    'name': 'University of Toronto',
-    'domain': 'jupyter.utoronto.ca',
-    'id': 'utoronto',
-    'template': 'base-hub ([deployment repo](https://github.com/utoronto-2i2c/jupyterhub-deploy/))'
-}]
-for cluster_info in clusters:
-    if "schema" in cluster_info.name:
-        continue
-    # For each cluster, grab it's YAML w/ the config for each hub
-    yaml = cluster_info.read_text()
-    cluster = safe_load(yaml)
-
-    # For each hub in cluster, grab its metadata and add it to the list
-    for hub in cluster['hubs']:
-        config = hub['config']
-        # Config is sometimes nested
-        if 'basehub' in config:
-            hub_config = config['basehub']['jupyterhub']
-        else:
-            hub_config = config['jupyterhub']
-        # Domain can be a list
-        if isinstance(hub['domain'], list):
-            hub['domain'] = hub['domain'][0]
-
-        hub_list.append({
-            'name': hub_config['custom']['homepage']['templateVars']['org']['name'],
-            'domain': f"[{hub['domain']}](https://{hub['domain']})",
-            "id": hub['name'],
-            "template": hub['template'],
-        })
-df = pd.DataFrame(hub_list)
-path_tmp = Path("tmp")
-path_tmp.mkdir(exist_ok=True)
-path_table = path_tmp / "hub-table.csv"
-df.to_csv(path_table, index=None)
+import subprocess
+
+def render_hubs():
+    # Grab the latest list of clusters defined in pilot-hubs/
+    clusters = Path("../config/hubs").glob("*")
+    # Add list of repos managed outside pilot-hubs
+    hub_list = [{
+        'name': 'University of Toronto',
+        'domain': 'jupyter.utoronto.ca',
+        'id': 'utoronto',
+        'template': 'base-hub ([deployment repo](https://github.com/utoronto-2i2c/jupyterhub-deploy/))'
+    }]
+    for cluster_info in clusters:
+        if "schema" in cluster_info.name:
+            continue
+        # For each cluster, grab it's YAML w/ the config for each hub
+        yaml = cluster_info.read_text()
+        cluster = safe_load(yaml)
+
+        # For each hub in cluster, grab its metadata and add it to the list
+        for hub in cluster['hubs']:
+            config = hub['config']
+            # Config is sometimes nested
+            if 'basehub' in config:
+                hub_config = config['basehub']['jupyterhub']
+            else:
+                hub_config = config['jupyterhub']
+            # Domain can be a list
+            if isinstance(hub['domain'], list):
+                hub['domain'] = hub['domain'][0]
+
+            hub_list.append({
+                'name': hub_config['custom']['homepage']['templateVars']['org']['name'],
+                'domain': f"[{hub['domain']}](https://{hub['domain']})",
+                "id": hub['name'],
+                "template": hub['template'],
+            })
+    df = pd.DataFrame(hub_list)
+    path_tmp = Path("tmp")
+    path_tmp.mkdir(exist_ok=True)
+    path_table = path_tmp / "hub-table.csv"
+    df.to_csv(path_table, index=None)
+
+
+render_hubs()

docs/environment.yml

@@ -0,0 +1,14 @@
+channels:
+- conda-forge
+dependencies:
+- go-terraform-docs
+- pip
+- python=3.8
+- pip:
+  - myst-parser[sphinx,linkify]
+  - sphinx-book-theme
+  - sphinx-panels
+  - sphinx-autobuild
+  - pandas
+  - pyyaml
+  - requests

docs/index.md

@@ -42,6 +42,8 @@ Topic guides go more in-depth on a particular topic.
 topic/config.md
 topic/hub-templates.md
 topic/storage-layer.md
+topic/terraform.md
+topic/cluster-design.md
 ```
 
 ## Reference