From 48f1d4e3a24db1fb73320d15832ef058d1d4bca6 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Tue, 4 May 2021 01:16:06 +0530 Subject: [PATCH 01/12] Add support for `kubeconfig` auth provider --- config/hubs/schema.yaml | 16 +++++++++++- deployer/hub.py | 54 +++++++++++++++++++++++++++-------------- 2 files changed, 51 insertions(+), 19 deletions(-) diff --git a/config/hubs/schema.yaml b/config/hubs/schema.yaml index f7ca33133a..ea2e2fd420 100644 --- a/config/hubs/schema.yaml +++ b/config/hubs/schema.yaml @@ -14,9 +14,23 @@ properties: type: string description: | Cloud provider this cluster is running on. Used to perform - authentication against the cluster. Currently supports gcp. + authentication against the cluster. Currently supports gcp + and raw kubeconfig files. enum: - gcp + - kubeconfig + kubeconfig: + type: object + description: | + Configuration to connect to a cluster purely via a kubeconfig + file. + additionalProperties: false + properties: + file: + type: string + descriptiON: | + Path to kubeconfig file (encrypted with sops) to use for + connecting to the cluster gcp: type: object additionalProperties: false diff --git a/deployer/hub.py b/deployer/hub.py index 40e18bc2c8..2670ec298f 100644 --- a/deployer/hub.py +++ b/deployer/hub.py @@ -37,12 +37,30 @@ def build_image(self): @contextmanager def auth(self): - with tempfile.NamedTemporaryFile() as kubeconfig: - # FIXME: This is dumb - os.environ['KUBECONFIG'] = kubeconfig.name - assert self.spec['provider'] == 'gcp' - + if self.spec['provider'] == 'gcp': yield from self.auth_gcp() + elif self.spec['provider'] == 'kubeconfig': + yield from self.auth_kubeconfig() + else: + raise ValueError(f'Provider {self.spec["provider"]} not supported') + + + def auth_kubeconfig(self): + """ + Context manager for authenticating with just a kubeconfig file + + For the duration of the contextmanager, we: + 1. Decrypt the file specified in kubeconfig.file with sops + 2. Set `KUBECONFIG` env var to our decrypted file path, so applications + we call (primarily helm) will use that as config + """ + config = self.spec['kubeconfig'] + config_path = config['file'] + + with decrypt_file(config_path) as decrypted_key_path: + # FIXME: Unset this after our yield + os.environ['KUBECONFIG'] = decrypted_key_path + yield def auth_gcp(self): config = self.spec['gcp'] @@ -52,23 +70,23 @@ def auth_gcp(self): # Else, it'll just have a `zone` key set. Let's respect either. location = config.get('zone', config.get('region')) cluster = config['cluster'] + with tempfile.NamedTemporaryFile() as kubeconfig: + with decrypt_file(key_path) as decrypted_key_path: + subprocess.check_call([ + 'gcloud', 'auth', + 'activate-service-account', + '--key-file', os.path.abspath(decrypted_key_path) + ]) - with decrypt_file(key_path) as decrypted_key_path: subprocess.check_call([ - 'gcloud', 'auth', - 'activate-service-account', - '--key-file', os.path.abspath(decrypted_key_path) + 'gcloud', 'container', 'clusters', + # --zone works with regions too + f'--zone={location}', + f'--project={project}', + 'get-credentials', cluster ]) - subprocess.check_call([ - 'gcloud', 'container', 'clusters', - # --zone works with regions too - f'--zone={location}', - f'--project={project}', - 'get-credentials', cluster - ]) - - yield + yield class Hub: From 240b7a957d1c3411ca66e44da6d95d716e96ccc4 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Tue, 4 May 2021 01:30:05 +0530 Subject: [PATCH 02/12] Add farallon secret + cluster config Not complete yet --- config/hubs/farallon.cluster.yaml | 164 ++++++++++++++++++ .../basehub/templates/nfs-share-creator.yaml | 2 + secrets/farallon.yaml | 33 ++++ 3 files changed, 199 insertions(+) create mode 100644 config/hubs/farallon.cluster.yaml create mode 100644 secrets/farallon.yaml diff --git a/config/hubs/farallon.cluster.yaml b/config/hubs/farallon.cluster.yaml new file mode 100644 index 0000000000..563869eba2 --- /dev/null +++ b/config/hubs/farallon.cluster.yaml @@ -0,0 +1,164 @@ +name: farallon +provider: kubeconfig +kubeconfig: + file: secrets/farallon.yaml +hubs: + - name: farallon-staging + domain: staging.farallon.2i2c.cloud + template: daskhub + auth0: + connection: github + config: + scratchBucket: + enabled: false + base-hub: + nfsPVC: + nfs: + serverIP: fs-7b129903.efs.us-east-2.amazonaws.com + baseShareName: /homes + shareCreator: + tolerations: + - key: node-role.kubernetes.io/master + operator: "Exists" + effect: "NoSchedule" + jupyterhub: + homepage: + templateVars: + org: + name: Farallon Institute + logo_url: https://2i2c.org/media/logo.png + url: http://www.faralloninstitute.org/ + designed_by: + name: 2i2c + url: https://2i2c.org + operated_by: + name: 2i2c + url: https://2i2c.org + funded_by: + name: Farallon Institute + urL: http://www.faralloninstitute.org/ + profileList: + # The mem-guarantees are here so k8s doesn't schedule other pods + # on these nodes. + - display_name: "Default: m5.xlarge" + description: "~4CPUs & ~15GB RAM" + kubespawner_override: + mem_guarantee: 14G + cpu_guarantee: 3 + node_selector: + hub.jupyter.org/pool-name: notebook-m5-xlarge + - display_name: "Default: m5.2xlarge" + description: "~8CPUs & ~30GB RAM" + kubespawner_override: + mem_guarantee: 28G + cpu_guarantee: 7 + node_selector: + hub.jupyter.org/pool-name: notebook-m5-2xlarge + scheduling: + userPlaceholder: + enabled: false + replicas: 0 + userScheduler: + enabled: false + proxy: + service: + type: LoadBalancer + + chp: + nodeSelector: {} + tolerations: + - key: "node-role.kubernetes.io/master" + effect: "NoSchedule" + traefik: + nodeSelector: {} + tolerations: + - key: "node-role.kubernetes.io/master" + effect: "NoSchedule" + hub: + allowNamedServers: true + networkPolicy: + # FIXME: For dask gateway + enabled: false + readinessProbe: + enabled: false + nodeSelector: {} + tolerations: + - key: "node-role.kubernetes.io/master" + effect: "NoSchedule" + dask-gateway: + traefik: + tolerations: + - key: "node-role.kubernetes.io/master" + effect: "NoSchedule" + controller: + tolerations: + - key: "node-role.kubernetes.io/master" + effect: "NoSchedule" + gateway: + tolerations: + - key: "node-role.kubernetes.io/master" + effect: "NoSchedule" + backend: + scheduler: + extraPodConfig: + nodeSelector: + hub.jupyter.org/pool-name: dask-worker + tolerations: + - key: "k8s.dask.org/dedicated" + operator: "Equal" + value: "worker" + effect: "NoSchedule" + - key: "k8s.dask.org_dedicated" + operator: "Equal" + value: "worker" + effect: "NoSchedule" + worker: + extraPodConfig: + nodeSelector: + hub.jupyter.org/pool-name: dask-worker + tolerations: + - key: "k8s.dask.org/dedicated" + operator: "Equal" + value: "worker" + effect: "NoSchedule" + - key: "k8s.dask.org_dedicated" + operator: "Equal" + value: "worker" + effect: "NoSchedule" + + # TODO: figure out a replacement for userLimits. + extraConfig: + optionHandler: | + from dask_gateway_server.options import Options, Integer, Float, String + def cluster_options(user): + def option_handler(options): + if ":" not in options.image: + raise ValueError("When specifying an image you must also provide a tag") + extra_annotations = { + "hub.jupyter.org/username": user.name, + "prometheus.io/scrape": "true", + "prometheus.io/port": "8787", + } + extra_labels = { + "hub.jupyter.org/username": user.name, + } + return { + "worker_cores_limit": options.worker_cores, + "worker_cores": min(options.worker_cores / 2, 1), + "worker_memory": "%fG" % options.worker_memory, + "image": options.image, + "scheduler_extra_pod_annotations": extra_annotations, + "worker_extra_pod_annotations": extra_annotations, + "scheduler_extra_pod_labels": extra_labels, + "worker_extra_pod_labels": extra_labels, + } + return Options( + Integer("worker_cores", 2, min=1, max=16, label="Worker Cores"), + Float("worker_memory", 4, min=1, max=32, label="Worker Memory (GiB)"), + String("image", default="pangeo/pangeo-notebook:latest", label="Image"), + handler=option_handler, + ) + c.Backend.cluster_options = cluster_options + idle: | + # timeout after 30 minutes of inactivity + c.KubeClusterConfig.idle_timeout = 1800 \ No newline at end of file diff --git a/hub-templates/basehub/templates/nfs-share-creator.yaml b/hub-templates/basehub/templates/nfs-share-creator.yaml index e5522ddf12..c3333ee80a 100644 --- a/hub-templates/basehub/templates/nfs-share-creator.yaml +++ b/hub-templates/basehub/templates/nfs-share-creator.yaml @@ -22,6 +22,8 @@ spec: spec: restartPolicy: Never terminationGracePeriodSeconds: 0 + tolerations: {{ .Values.nfsPVC.shareCreator.tolerations | toJson }} + containers: - name: dummy image: busybox diff --git a/secrets/farallon.yaml b/secrets/farallon.yaml new file mode 100644 index 0000000000..736d6c299b --- /dev/null +++ b/secrets/farallon.yaml @@ -0,0 +1,33 @@ +apiVersion: ENC[AES256_GCM,data:v78=,iv:jtNO2Px/wbm94DieLvsv88uEI5Smf5p4Us6NvSuzLKE=,tag:FaxNM/Z8AXoQQlrSYjJ0CA==,type:str] +clusters: + - cluster: + certificate-authority-data: ENC[AES256_GCM,data:GV7rhSD+g8HdZ/04a/sxS/afQj/MlG4ujy6+ejPj/IrMWFYsxrQi22+Uzj6+3Uvrdpb9XtfKGgeRFHCGysMetxjdg5vjDxnRZswkgbIx2b5bn376Dg2jZev2TheOwJa59H3nVSyj3JSvBiBrvXuNV3CWkTigqz4WtYKAd6bjJC9JPkLoZroHfh0U3gVw28JIgd5Mwu2RYnR6ivcd/2o/jaDbcIJHKZtxWLplBX34Oa1OVKj+fVTdG16gUCzgS00s06szSsuwxoT9IE5mBAuAC8gcBoP6r1wLbnu/QEErEMDXcW6ar72vwjDAUJcJQp1DPvhSB5QOR0sv0r/Xza1MskPRbKrKYEXV+xGCoU9MWk+98V+HK2ssqLx4NgMgYZeRW6hcI0FkpqBzItTfAuI2fpzHkEAfb7h91RnJVMAOwBmh88Wrb0IDWaWaqKcmaG5wEL3hL1ppwIVKQHebN1sEIjVrGK2iCNk4Cvt6YolZL2FiP1FzpeDRbSg+0c0enyriQjejLRWlym4e7+UeR9583kvGNHZzs7aotfqdDQfyjsBg4Mal4v8FGGpEqqkdp0T7CpgxWPOegqGkykewo8Q/M63w9vfL/OSLjSqIRaPBsRwpjjh8fMivNxJDfL5JcOTi7e9/iZBItIqBIMD6v3jfjp+ntm5kCYYJgPPDudYWn6/cheKnB0jGvb9Q02++a7ZaDpM/Lj1GQJp9C5j5paDzwYyHuaAJBMRnU5G9vNAsM6/hmW5OktacVA6FirTL2TaBGPia8PvGtHxP9E57DDPc6quVcGgoEdKGeHVszENfuuYqJjFWClddF8vqihdbWLiq/pER1ru7JN+jGOv61WYBTLF97zTc7g2GWLHEv4UQpSkciY0m2WG3et1U4XzXIQgbwPOUSe36KBmYgPpPceXrTuWRM5n58ax0085fGNpKb6QuX5dG2w8cKto5EgDkRnnfZ4ZLFfZkagmHbiByYGltC3RcmU3VrGXcTN6oNAAy6+Nz9O8ac3m6aht6PAMVoWIWBgsFLdvyMg/X28i38vtSST3c9LYQBgD83cGaRgK6XDZWNOTobYniRB0QftjF9eVUY45azC846KlnpKzgGKG173TS0mILhME2MFW9zmQc+7lZR/FSrYk7HD10/ForqNlmUzC0+arEOxVGIpgpH9FuYxHDoidhQp43/3bdXXkIDLwKFj8DmATEbRLmhjlHuivBYh+z21YH8jyI2kQ6RWca4O0JpKDwLK9DKRn2QjqHfqT5jO4D/78vbH/91faMF+wNhV70UGEbshlQkABUC0KHuZcRZQBXMTXJkWVrChttl4yhDmi8lrSMmd9ajZIjslkFujG9LNpyrPJLqSUczASU5hSnW5h8wpDjWW0Rigol6NV8Jok/CZArenJ4HM7FWw065uNPxnJnfDvLgnub+fWSah2ECF17AA/yprghl+R/rooItyGw7WOMaP5t5b3YqidTvhz7+WrYGSIN5J4IBbcTbUvToVu93IFNv90cPM3ZAP2D/DuFAaGA+u2uEocLkdyR63UmOCoF/osHjbdjdkwvrY/bf7BSj4gUUqps47rqyljy6ytidgZln2Qmr0px71+fT7hKxyZsRW2Clll4L/KAV8FzO3aXZ1he0kbXtSHvQzS9YJAKa2YKcYUnyAcQRKdm0TGqXfYvOVGhSgsQNiFDG5snUvSNAy6XBhd6+HClUA8+duZU8Ml6Q5r4s29mppM8poq6ELOnJmt8EjS8NihEmvmG90/gqC72A7Ptv2rK/aXlbW1Gmc0wQwdA8LXrvXhoHSpTcp8zmlAa2mRufjwxBLj9u1ghieNh1SmrPxOqn6dgFWqn97qGkbCZ3VPoKYFNlTwO2KiUaR0UzkkV8xm4AyokbXYsu+JVEff6IJ1KOykE9ct7rFyzpj7ECZBu9DqIPA8cyw==,iv:SNw9RjwYLSmMO/Nrz2+IA0GVAAcejoz2Af+9+IoDsWw=,tag:0r7OESoIKKvwCTSoJk/GRA==,type:str] + server: ENC[AES256_GCM,data:sjGOJaUEXNJkHxtYJO0f3PpQ5dv8PQ0lehOAnVNvAccRsytt5FFVrRj64v+BLMucAiEOX9oUcsHrikB9C5RbM4FkfvWMe5ULD4fQrYU1Dw==,iv:ONbXi6onE5d8abL5cxr6p3jJd4UW46Ut0CvWKQECLhE=,tag:O6f1X5mvaUdJB4ZnWRGHtA==,type:str] + name: ENC[AES256_GCM,data:T5Q6YzMy/6j42bGlmCYrCqcAUcJYL/c=,iv:TUceXHDNbq561n0944dt/NWf9IRFyJdwhhhlKPZ38Hs=,tag:3fqDqhwDAfhtvQNMJ/q62g==,type:str] +contexts: + - context: + cluster: ENC[AES256_GCM,data:ubveHqV2JHR3apEGcEVVtg4ym0jh+5Q=,iv:DD4OS5+TOwQyN6gevKQjsVrYT6CVK4/xN81n5QCO0Gg=,tag:btI/f20ZuL57Vcu3G8Ncpg==,type:str] + user: ENC[AES256_GCM,data:Kfy0Qnp9eZWH2cK0hEQAMtIFkrlqeq8=,iv:nXtiZUkmKaLLDsMfhl0aApxLsYGC9r+Sh0/yomD3nls=,tag:jVCFSQZHpCHEq/X+dcTT3A==,type:str] + name: ENC[AES256_GCM,data:h8XW7YdCVKOOyinoNY0Pb9UrApNjvW0=,iv:YOwJeUZfE2wyiKgIc1Kjj+b/C8PbNcpkzVK9bjASHKY=,tag:mcW7/QsKATFUEnl8bfni8A==,type:str] +current-context: ENC[AES256_GCM,data:6ldoUtzpD3xpKeIMZ7e85i1Ht9GTwmM=,iv:ci+9F7J+O+XGWw6VMK4PD6u1KEhwu1xLP4GE9F8t6oc=,tag:rUZyda5JRJwC9LzU316zzg==,type:str] +kind: ENC[AES256_GCM,data:RmpDLXcA,iv:2rnkdWK89mTzuM0sYhGO9rbyPNirwRF7czbpdb8V5yA=,tag:VE8KJRElU3sKs3R8qamTig==,type:str] +preferences: {} +users: + - name: ENC[AES256_GCM,data:JBthcd8lbiy1AiRen1ShGDzV6ZVfpyY=,iv:bdqVBy0NWQTEi3YbOqoxrmfmxejGzFXSGcNlsfwDYzc=,tag:Q67OWexlYZxE7L8lzEkVYQ==,type:str] + user: + client-certificate-data: ENC[AES256_GCM,data:Jb9yNhWGZjd6SMqbZq1kxTVnzfWEMFlD94ijBkDnt5LwYtRLlvc7SEaEVhGmnPn5f7m2TJbEGXH572L1QY0AU0LNV21v0+udrrXy8Gct53QLHt+anAK/QPqSk2BAzFtUjw8CHfJKFIaHdB19/0nCNlYx4LECi6jk8Trethj1TvLl0L64/4PkSgOff8NYJmIJfNc7ElvlfXBL3hBDjnKzBJro8+XT15uiY10YhkxvUYnXJmeMxvnmV8IAxKWmVF0rVnVtCAGBskZJbPU5bjPuQGP7aqZZBwnp8SbRX3s8iEdR3pGKe3dE79SYs6t360IwLnI79zutU7SNDzbMQjhQ5KAdqepXBU3G/twU0v2FXlynlBoiwxn2cfw6zVnYJTBDfMh2jshMVCbaTdZNIDqZG1kQOgpjWcCLWVhl5yF/bGcGTd7E0cQbW87IoBKEOgscHpkdLMpCCtRS+vEE1YXAHqRfoc9Q62mCGRDTjzv1puDDAoAvsVrd+CubPVv/2j3kMuj5CYixZIZvcAw9bXCj4uJqr4PN2xCxf4H/QrSjb0kdPmSxDhGTxnt2D7I36M2IgtpKHTEC0afq9mVD0MmBGZmcczZEjM8170anAtw8H4OE8hPMk3nZYFwT1giSCj62ipgEGpLHFq5gSEH62Qdjp7aX47esh3klnx81xeuQilpV24vTpYuhyJFqfEoTDfjdXMAhTr2sq+6zrE8tuplyQcftcw+P3Q5Vq4Vd6KNN7Gz3JlRs3U4uFHx4XOzAy0q752FoF/0EKc1SZi8Rmcc7nAQrRBKX4dfRpDu5bYgu1WCsmFxWcwpenv0VHybIQSEpv3s9rWS72xR9nvzwMWeGv106uOgar16jwoac/AMLYfLZ7XAtu+LkQc6UcFOKCZk+bfA85MQ7t6gErxBi8dg2tivzuaZ4WF9zDGYTrsgmaNeh/TPQEaAp2Wm0qP4xdtehk+YP9YI907ZYyUHD7iKRczuublclpE/zCD7mbjwlWfR/tztTdyGS4Ii/75+E2Nmfo+DdyCNTD/yZUvC826uPufdN5iQtnDVEb10Nc4LfAHE4HyM3k/956afvMcV1IftMf1lK7/MRZ616o1zCyAKxaSXViWfh5HzPHsZBwP5UB/4FXSXLUCrF9n00YLgG1pmc1PS7RL+ZQrEcjJ3us+Jwds/09JkzxU6EFeHFswaERcT1XSRtLv0WYpXq/4x5emcn/ds+kwRCjkfNDHvQmgv+T3yizN3fJlCDnsfPlcpJBcFu4//VO0+lc3U6pWB0cCoe6sOTy1bKon4YISsw+FSLZppzqy4JUf2tj9I6gQHbFdE4YV7sZo1VYR11lfyQUBVMfTjZrzoFFZGfy3q9fv5nuTfWn912RgRh4GiH4HsWGCNPEJQPkMNq+oXbSivQihtuOpizDL0JVMBnd5wHrZoq2Lxfmm/pXFZDPBHjeXYrypJbEHT6JO7ACcZyTkiKXUnOSU5imVV03mpNC62GinT2QGrGGrJnXnDQINdhHo8tBO6/hAbj5rreTzszt1BaSEzNoZWk6RNL7XaEP3+RcfCfEbf4SIDkNCTfS2EZkovhp0Jd7QCD5goxUrIhDNuqe5838oMTY9+CbjG0+/QjkvpRFVQcAdb7s59hOKkpJSwu6jnFoHS0Lv+R9bdR71iJ5OMumk00nhTZ7379llq+Cstv8PHz59iZVLcAc6sHKAwwb2wYyK/YOC6NH1a7cdNwkOS7kUBtqPeoTdtF5omPCUcuvEnMwKH9EfQNye1hZXnalfwqZqGnULjhmFDef2JwjiSQiA60zFKwlRDBPDK+Jzsy1+d94lKq27p29rEP+gNNNJ82blED/I8dAjtH9k1hAJSuc9lp9YQDPV3plzexhjKKeSUQkd7ETD2oOQZNL6cHB7R9KqSztENuh4q+rzvBcOYeHeFt7penrLDWIm7dqhptTEdnVrwxkch9EQm4XU4fgMdwEKzNUDl3e7TvM67zDKtSc9XQwu/+bhxwg5poK2ValVSU9q+m9jZtPuZE1f3HjqiWsW7jNI4CWn5GOlZ/q37084X7uJHY425Eu1s/,iv:oXRJJkfCfp1V9s5VuFer3J0omWc+YqP4agPUnhcv3jQ=,tag:xqBBEZzE6SBjRupTt9iKVg==,type:str] + client-key-data: ENC[AES256_GCM,data:Dn3I/C2YLsXtOrDzsTM53Lay1dChtNwHUpzoPHfEuEvLl8fTm6nNSRmNvl+bkRGYf87njfsyF3D5e7qoRGRpPBNTaePtsk4+pvvpAO+OrUCWZKBc0mXTXdxR3yqLi9CwawfbVZPPcWaMWOD241tGiP3LUL6ojNZdKKCjofN51iSmhszN11mXcmDEvAuF3oEDH7Drd565WkxmOqGtjK+sK3vtUtSB/pZCDKeel8JypczQs/lX+zcJeIGrUhsEDMl4quDZLI1BLE04le+xC0QPkCPL5KNhbwqqox7ccXmNwsCrt2TvWn4Su17lBpWofZwrmcMUihUqp2ckwU25/rNho1VENRxjU/FQ9XNEzQ5XWhqlUa9se4ZKIQjFLsTBmC1MD/1+J5Xpuw75AKbkNLu9xaawTkdzARMgSP6DfZ94mlwB9kdZ/uqkjpoPy957TykMRkYskbGtEz0rZnJqzPyZFwNM2FfOQwaZilwYHd8q1E+A+zsnEIGYIJwG0LllFDy32n849bLhPv+byee9B8CoyB7U+kxLsc/NL2dQmBKlNBxoTxrdAfaHBlACERI/AezA/RTHQzYfAX1Y9CZJEzIXMne0DFf24km3p4rnH1CTGrrDBkRMr71BmDhIRtsZMfoJ4yaz3n2QT1J1qvKt0VIrjkskO2IhJYN8ElzrMDdhuOZ9p9DtpUxSfK3h/3mgQChP5/MXBTAqlnt1jeWNSkNQZpjDrsla324MNulY3OQXAHZwkko8GIP2qxHYQVEzQZSrFKpT70b8g/Ne80atDt1NxoYnW7314jQDKu3AKuEt0BPFfiA7yiWiN3sedEFbF01MDKaHDtzfSdJ3voGBgCkR3d0twVYHjgbwkIEauLfh4X/oUgjKYcaUklJeCMlt8YBZqUv6Ur7d1SeQsce9YNERldWowGkWfuYFh1rv5/B6mE34SJ0hgKbYFt+Ijd4HCF/z9TrrAN6VdPCnwH7BKONVB1x+IOIgAIVUZHIJx/R27YlCqwkbco1qmjozdbze3n3vzcrfklI76oWs+21SWMb5mQ/tlsl3PxdFe+0pygcplOJETWuIG1PEomkmoySmPDCk2hW/Bh+9Ac7tkyfDdhWg9Wav0mmTIYXTUpU8hKCg6Ho1T6dFNjfp2fzFfChFcozoC8T2BX+IYQ7maSG8Ar5TkJDY2r8lafca7HJ2X2UbxWX+Y8OeiVO9cx60Xd1dZosOGQ7Y4Ntdd7irol0FHcWnIII1e9dm4QiScqCOWTD5WYpvZ8oQk8cc6kk7Zm+I+GkHTTaFYhxG4G5X+vHJFi1HqpCvqeS+Gcy2KXBXPamKehMF7FFnGbGllO2Y+dD0G5+woAqSN1fHKnunXWQOYwx2I1yChgqy4YQm8IZV01EIlVroi97kq3LQuWJqdsSGb+uws9hvlTVmprqgAgapeQQBYovuB3kGg/bp+LDwBhtdEoiQ0m+xIbsOxJeYwwgwMoo4wvju63dhUPCgZJutJtNdZwZuRRKsF5EUkfL3nFFqmiu4AFWIRfcM/e64OTDQb8nTTxpkFLAca1i0EAODOK8RBqkYNrIOktgvaS0XgKJ6nG5URsx4QL4dosppz3/MpFxZUKvJxzG7eG7qtrqobq2XMXGBVJ7NyhDCGJv+q8t/A8JK8aen8s5MCCekQnqJityJUS6++BpGWUaRD8ApxajE1OhFfgfIDuB5NX69ytLxWDcwQWlufndSo1yzRYW77mzWdXDHqB0Y5HzhDw1k+JR3kZ8lWge6uh+y3vPAQDNygbx4gXvnqeL3MwcKHGc+fN+DbOgF9Yd9VrP1z7Zk215dPLSWsGF9Qtt3g2/H1lkfERg9UlsPK68hfVqsBJVuoJ13TVt0y6PL1lat88RTCQoVC3iEIW4IxXIX1NMaLFpKZih1Ec0aFeCsrgYDTCm0ojPqS0vTKCisopkknmy7mt3Yzgx37VoCBYSK6ZyG8iXRq28IkOxLukj8L2oWCmj8Nq46+qKRnMghsD3F0gOby3wcbxMU+de2SDUYQHoDdJOuUxoL92J+fHMShT0MuPVQGanIpVBdEqi2uwqotLIk8A4j+wJsTd8P+ICLJSAvyDuZj5iPgkpkfwQBhyUkiYmebVvc5UYNZo6k9H4JlmCFvbPZ6Ahx5Pt90VDPecT9wsFwSXLY/WLn2bZ8ROFpaLVd7btEZ8CBQuC3Am1WSxwb0Wkfy5xkFgOU++gpDPcIrIgYovWfI/CGkMTw3kMFidd0jFBx+16xtleWacejFAZmy6JaHkP2UOpf/DagxH+xPJdRyKZBJ+PmDa7d90Rq2Pql068VouvwvSdkYQ0jXLx6K+nri95SlhY1fNovbDMMRQ/oowIOKsq603lsJ5rRQKsHL/wNJN+GEFk8CcwjZrzdlXY797qS+0OX+h7Vuov2oupYcnJiBUgaaRBhErVBBq/plmtiBPV1/QrF8YXfE50axuDFNgrQ57GxEtrUabFfLIX7GMxQqq82SYtUXZqzPKhrLyBtxypmWdI5/lrgSMbm27ztBpFL2HfWHeL2yD0DTtOuu6xu0X+ZVFhLk/tyjHhYBLv1sUDviKArQ3MNKjU9wBmDsi3RcKe2ENboz1xCqJ8La5t/flMAYfbp/QnYqaO2J2KXZH5+YI3wY+pVh3HQk2Dxa5GGhdj9RioSqcPXfi6IPDptDH1jVmNlZ1YXrP1HjQcG74NxBBJ1GWZGd3hF3X2o6CjERfkurUYpEYIqtaY1nv/YHf/s2LqiiXH1qsLTUgmOW5kWrfC1zO8NvylKg1G0F9FWWGLJvVdGCfzu108tAUMXks9ziTkWveE1uluBhm9KM6c3gUdfcUr3HpftB7rvzsj47JxtmEqLcI3JJItquv7H3fNZTIO1XLOwC2s1Gmd0lhqtvID6pJB65EDhXCaCUOtjTC9dWXHFKQrKgQY/lWGxVveK0QsZYJ/WincSnNekxGhpVUbYCwzaY18vf7HUiEvtOcJWu6XqjzAN0w==,iv:LJG6fPYjjrO6kahxdaLB1vF89f8RA74nwp7THrcHGsk=,tag:q3CPDUMI9MRAptVeNzcMig==,type:str] +sops: + kms: [] + gcp_kms: + - resource_id: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs + created_at: "2021-05-03T18:24:30Z" + enc: CiQA4OM7eBvDdmSosNwoV1H4lGprhSnF6VzDyRaW6U1D9qi6XHQSSQBy9hCY8tILK+/MkhaLO9jCz3fDyhOiF1lxq+zw7GZF8USAZ1ihXFp29CLO69PQa5ejJqZffdWydRTJxuQ1Gd+pEHzC2KmCrGc= + azure_kv: [] + hc_vault: [] + age: [] + lastmodified: "2021-05-03T18:24:31Z" + mac: ENC[AES256_GCM,data:lSuWapMSsXuAeeSCKHGRsVRXtmeUpCQVTTKDFYHfFADyWF+t8TgwNvRvIuicK69esHW8GnRb5saakKzmlZjwFJKZ+lU8ADXRIY38uRo/4OEvzKhi71lNHxSqZHG/RIPbFsuPoF+xNpqBmfoGmKhrJTxP7zs5hKz2zwZ5GuUdgec=,iv:yVgiI7pbLWPXl1++ZVhg31ASIZn7FgjU8NbouldqC5s=,tag:HsQXJX5JbLVuB10c4JzMiA==,type:str] + pgp: [] + unencrypted_suffix: _unencrypted + version: 3.7.1 From 5154f5025f52aaa520d46d63a86ae94caa04e59b Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Tue, 4 May 2021 23:46:24 +0530 Subject: [PATCH 03/12] Fix profile list - Needs to be nested under singleuser - Explicitly set mem_limit too - otherwise, the default 1G limit of memory stays in, and k8s doesn't allow a limit to be smaller than a guarantee --- config/hubs/farallon.cluster.yaml | 41 +++++++++++-------- .../{service-account.yaml => sa.yaml} | 0 2 files changed, 24 insertions(+), 17 deletions(-) rename hub-templates/daskhub/templates/{service-account.yaml => sa.yaml} (100%) diff --git a/config/hubs/farallon.cluster.yaml b/config/hubs/farallon.cluster.yaml index 563869eba2..d51873ff0a 100644 --- a/config/hubs/farallon.cluster.yaml +++ b/config/hubs/farallon.cluster.yaml @@ -37,23 +37,30 @@ hubs: funded_by: name: Farallon Institute urL: http://www.faralloninstitute.org/ - profileList: - # The mem-guarantees are here so k8s doesn't schedule other pods - # on these nodes. - - display_name: "Default: m5.xlarge" - description: "~4CPUs & ~15GB RAM" - kubespawner_override: - mem_guarantee: 14G - cpu_guarantee: 3 - node_selector: - hub.jupyter.org/pool-name: notebook-m5-xlarge - - display_name: "Default: m5.2xlarge" - description: "~8CPUs & ~30GB RAM" - kubespawner_override: - mem_guarantee: 28G - cpu_guarantee: 7 - node_selector: - hub.jupyter.org/pool-name: notebook-m5-2xlarge + singleuser: + profileList: + # The mem-guarantees are here so k8s doesn't schedule other pods + # on these nodes. + - display_name: "Default: m5.xlarge" + description: "~4CPUs & ~15GB RAM" + kubespawner_override: + # Expllicitly unset mem_limit, so it overrides the default memory limit we set in + # base-hub/values.yaml + mem_limit: null + mem_guarantee: 14G + cpu_guarantee: 3 + node_selector: + hub.jupyter.org/pool-name: notebook-m5-xlarge + - display_name: "Default: m5.2xlarge" + description: "~8CPUs & ~30GB RAM" + kubespawner_override: + # Expllicitly unset mem_limit, so it overrides the default memory limit we set in + # base-hub/values.yaml + mem_limit: null + mem_guarantee: 28G + cpu_guarantee: 7 + node_selector: + hub.jupyter.org/pool-name: notebook-m5-2xlarge scheduling: userPlaceholder: enabled: false diff --git a/hub-templates/daskhub/templates/service-account.yaml b/hub-templates/daskhub/templates/sa.yaml similarity index 100% rename from hub-templates/daskhub/templates/service-account.yaml rename to hub-templates/daskhub/templates/sa.yaml From 491a19aea4c0e382ccb898b539499ae5f673481f Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Tue, 4 May 2021 23:50:48 +0530 Subject: [PATCH 04/12] Allow customizing NFS mountoptions EFS doesn't let you set ver=4.2 --- config/hubs/farallon.cluster.yaml | 10 +++++++++- hub-templates/basehub/templates/nfs-pvc.yaml | 5 +---- hub-templates/basehub/values.yaml | 4 ++++ 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/config/hubs/farallon.cluster.yaml b/config/hubs/farallon.cluster.yaml index d51873ff0a..4d16af30f1 100644 --- a/config/hubs/farallon.cluster.yaml +++ b/config/hubs/farallon.cluster.yaml @@ -14,8 +14,16 @@ hubs: base-hub: nfsPVC: nfs: + # from https://docs.aws.amazon.com/efs/latest/ug/mounting-fs-nfs-mount-settings.html + mountOptions: + - rsize=1048576 + - wsize=1048576 + - timeo=600 + - soft # We pick soft over hard, so NFS lockups don't lead to hung processes + - retrans=2 + - noresvport serverIP: fs-7b129903.efs.us-east-2.amazonaws.com - baseShareName: /homes + baseShareName: /homes/ shareCreator: tolerations: - key: node-role.kubernetes.io/master diff --git a/hub-templates/basehub/templates/nfs-pvc.yaml b/hub-templates/basehub/templates/nfs-pvc.yaml index cb6b1ab61d..61c7b02e91 100644 --- a/hub-templates/basehub/templates/nfs-pvc.yaml +++ b/hub-templates/basehub/templates/nfs-pvc.yaml @@ -11,10 +11,7 @@ spec: nfs: server: {{ .Values.nfsPVC.nfs.serverIP | quote}} path: "{{ .Values.nfsPVC.nfs.baseShareName }}{{ .Release.Name }}" - mountOptions: - - soft - - noatime - - vers=4.2 + mountOptions: {{ .Values.nfsPVC.nfs.mountOptions | toJson }} --- apiVersion: v1 kind: PersistentVolumeClaim diff --git a/hub-templates/basehub/values.yaml b/hub-templates/basehub/values.yaml index 62b2822dcd..3645962d65 100644 --- a/hub-templates/basehub/values.yaml +++ b/hub-templates/basehub/values.yaml @@ -19,6 +19,10 @@ nfsPVC: shareCreator: tolerations: [] nfs: + mountOptions: + - soft + - noatime + - vers=4.2 serverIP: nfs-server-01 # MUST HAVE TRAILING SLASH baseShareName: /export/home-01/homes/ From 4157a174a6f6c6f5341fc9db79d38615dbb85324 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Tue, 4 May 2021 23:51:29 +0530 Subject: [PATCH 05/12] Turn on HTTPS for farallon staging again --- config/hubs/farallon.cluster.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/config/hubs/farallon.cluster.yaml b/config/hubs/farallon.cluster.yaml index 4d16af30f1..37cc1608fa 100644 --- a/config/hubs/farallon.cluster.yaml +++ b/config/hubs/farallon.cluster.yaml @@ -78,7 +78,10 @@ hubs: proxy: service: type: LoadBalancer - + https: + enabled: true + hosts: + - staging.farallon.2i2c.cloud chp: nodeSelector: {} tolerations: From 3c46f822ee497dd37096fee768f862254c5e3478 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Tue, 4 May 2021 23:51:57 +0530 Subject: [PATCH 06/12] Create k8s service account unconditionally We mount this into all daskhub user accounts regardless - by default they don't do anything --- hub-templates/daskhub/templates/gcp-iam.yaml | 50 ++++++++++++++++++++ hub-templates/daskhub/templates/sa.yaml | 45 ++---------------- 2 files changed, 53 insertions(+), 42 deletions(-) create mode 100644 hub-templates/daskhub/templates/gcp-iam.yaml diff --git a/hub-templates/daskhub/templates/gcp-iam.yaml b/hub-templates/daskhub/templates/gcp-iam.yaml new file mode 100644 index 0000000000..957337fad0 --- /dev/null +++ b/hub-templates/daskhub/templates/gcp-iam.yaml @@ -0,0 +1,50 @@ +{{- define "daskhub.serviceAccountName" -}} +{{.Release.Name}}-user-sa +{{- end }} +{{ if .Values.scratchBucket.enabled }} +apiVersion: iam.cnrm.cloud.google.com/v1beta1 +kind: IAMServiceAccount +metadata: + name: {{ include "daskhub.serviceAccountName" . }} + annotations: + cnrm.cloud.google.com/project-id : {{ .Values.iam.projectId | quote }} +spec: + displayName: {{ .Release.Name }} hub user service account +--- +apiVersion: iam.cnrm.cloud.google.com/v1beta1 +kind: IAMPolicy +metadata: + name: workload-identity-binding + annotations: + cnrm.cloud.google.com/project-id : {{ .Values.iam.projectId | quote }} +spec: + resourceRef: + apiVersion: iam.cnrm.cloud.google.com/v1beta1 + kind: IAMServiceAccount + name: {{ include "daskhub.serviceAccountName" . }} + bindings: + - role: roles/iam.workloadIdentityUser + members: + - serviceAccount:{{ .Values.iam.projectId }}.svc.id.goog[{{ .Release.Namespace }}/user-sa] +--- +apiVersion: iam.cnrm.cloud.google.com/v1beta1 +kind: IAMPolicyMember +metadata: + name: sa-requester-pays-binding + annotations: + cnrm.cloud.google.com/project-id : {{ .Values.iam.projectId | quote }} +spec: + member: serviceAccount:{{ include "daskhub.serviceAccountName" . }}@{{ .Values.iam.projectId }}.iam.gserviceaccount.com + role: roles/serviceusage.serviceUsageConsumer + resourceRef: + apiVersion: resourcemanager.cnrm.cloud.google.com/v1beta1 + kind: Project + external: projects/{{ .Values.iam.projectId }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + iam.gke.io/gcp-service-account: {{ include "daskhub.serviceAccountName" .}}@{{ .Values.iam.projectId }}.iam.gserviceaccount.com + name: user-sa +{{- end }} \ No newline at end of file diff --git a/hub-templates/daskhub/templates/sa.yaml b/hub-templates/daskhub/templates/sa.yaml index 957337fad0..25385efb54 100644 --- a/hub-templates/daskhub/templates/sa.yaml +++ b/hub-templates/daskhub/templates/sa.yaml @@ -1,50 +1,11 @@ {{- define "daskhub.serviceAccountName" -}} {{.Release.Name}}-user-sa {{- end }} -{{ if .Values.scratchBucket.enabled }} -apiVersion: iam.cnrm.cloud.google.com/v1beta1 -kind: IAMServiceAccount -metadata: - name: {{ include "daskhub.serviceAccountName" . }} - annotations: - cnrm.cloud.google.com/project-id : {{ .Values.iam.projectId | quote }} -spec: - displayName: {{ .Release.Name }} hub user service account ---- -apiVersion: iam.cnrm.cloud.google.com/v1beta1 -kind: IAMPolicy -metadata: - name: workload-identity-binding - annotations: - cnrm.cloud.google.com/project-id : {{ .Values.iam.projectId | quote }} -spec: - resourceRef: - apiVersion: iam.cnrm.cloud.google.com/v1beta1 - kind: IAMServiceAccount - name: {{ include "daskhub.serviceAccountName" . }} - bindings: - - role: roles/iam.workloadIdentityUser - members: - - serviceAccount:{{ .Values.iam.projectId }}.svc.id.goog[{{ .Release.Namespace }}/user-sa] ---- -apiVersion: iam.cnrm.cloud.google.com/v1beta1 -kind: IAMPolicyMember -metadata: - name: sa-requester-pays-binding - annotations: - cnrm.cloud.google.com/project-id : {{ .Values.iam.projectId | quote }} -spec: - member: serviceAccount:{{ include "daskhub.serviceAccountName" . }}@{{ .Values.iam.projectId }}.iam.gserviceaccount.com - role: roles/serviceusage.serviceUsageConsumer - resourceRef: - apiVersion: resourcemanager.cnrm.cloud.google.com/v1beta1 - kind: Project - external: projects/{{ .Values.iam.projectId }} ---- apiVersion: v1 kind: ServiceAccount metadata: + {{ if .Values.scratchBucket.enabled }} annotations: iam.gke.io/gcp-service-account: {{ include "daskhub.serviceAccountName" .}}@{{ .Values.iam.projectId }}.iam.gserviceaccount.com - name: user-sa -{{- end }} \ No newline at end of file + {{- end }} + name: user-sa \ No newline at end of file From 4b938c8415dfe9575ca583b5ab28d14677b4ce39 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Wed, 5 May 2021 03:09:40 +0530 Subject: [PATCH 07/12] Add image tag manually for farallon hub Needs to be automated eventually --- config/hubs/farallon.cluster.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/hubs/farallon.cluster.yaml b/config/hubs/farallon.cluster.yaml index 37cc1608fa..134340fa39 100644 --- a/config/hubs/farallon.cluster.yaml +++ b/config/hubs/farallon.cluster.yaml @@ -46,6 +46,9 @@ hubs: name: Farallon Institute urL: http://www.faralloninstitute.org/ singleuser: + image: + name: 677861182063.dkr.ecr.us-east-2.amazonaws.com/2i2c-hub/user-image + tag: 9cd76f1 profileList: # The mem-guarantees are here so k8s doesn't schedule other pods # on these nodes. From 37fd77c25486ea89b2c7b8fcbeb9044de8d43887 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Wed, 5 May 2021 03:19:40 +0530 Subject: [PATCH 08/12] Set permissions of user home dirs on startup --- config/hubs/farallon.cluster.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/config/hubs/farallon.cluster.yaml b/config/hubs/farallon.cluster.yaml index 134340fa39..7a83f7c4ef 100644 --- a/config/hubs/farallon.cluster.yaml +++ b/config/hubs/farallon.cluster.yaml @@ -46,6 +46,17 @@ hubs: name: Farallon Institute urL: http://www.faralloninstitute.org/ singleuser: + initContainers: + # Need to explicitly fix ownership here, since EFS doesn't do anonuid + - name: volume-mount-ownership-fix + image: busybox + command: ["sh", "-c", "id && chown 1000:1000 /home/jovyan && ls -lhd /home/jovyan"] + securityContext: + runAsUser: 0 + volumeMounts: + - name: home + mountPath: /home/jovyan + subPath: "{username}" image: name: 677861182063.dkr.ecr.us-east-2.amazonaws.com/2i2c-hub/user-image tag: 9cd76f1 From 2c1626bb17af59afceadc6ce121da12affd2f935 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Wed, 5 May 2021 03:28:20 +0530 Subject: [PATCH 09/12] Update kubeconfig to last 30days With KUBECONFIG=secrets/farallon.yaml kops export kubecfg --admin=730h farallon-2i2c.k8s.local --- secrets/farallon.yaml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/secrets/farallon.yaml b/secrets/farallon.yaml index 736d6c299b..0c89d5c39e 100644 --- a/secrets/farallon.yaml +++ b/secrets/farallon.yaml @@ -1,33 +1,33 @@ -apiVersion: ENC[AES256_GCM,data:v78=,iv:jtNO2Px/wbm94DieLvsv88uEI5Smf5p4Us6NvSuzLKE=,tag:FaxNM/Z8AXoQQlrSYjJ0CA==,type:str] +apiVersion: ENC[AES256_GCM,data:6ZE=,iv:uSEgAdks5fDnloUx5WMAewuhNi5T+4MBek+E8OxwxfU=,tag:40Q3Fb5NLXiBlpua5yCGHg==,type:str] clusters: - cluster: - certificate-authority-data: ENC[AES256_GCM,data:GV7rhSD+g8HdZ/04a/sxS/afQj/MlG4ujy6+ejPj/IrMWFYsxrQi22+Uzj6+3Uvrdpb9XtfKGgeRFHCGysMetxjdg5vjDxnRZswkgbIx2b5bn376Dg2jZev2TheOwJa59H3nVSyj3JSvBiBrvXuNV3CWkTigqz4WtYKAd6bjJC9JPkLoZroHfh0U3gVw28JIgd5Mwu2RYnR6ivcd/2o/jaDbcIJHKZtxWLplBX34Oa1OVKj+fVTdG16gUCzgS00s06szSsuwxoT9IE5mBAuAC8gcBoP6r1wLbnu/QEErEMDXcW6ar72vwjDAUJcJQp1DPvhSB5QOR0sv0r/Xza1MskPRbKrKYEXV+xGCoU9MWk+98V+HK2ssqLx4NgMgYZeRW6hcI0FkpqBzItTfAuI2fpzHkEAfb7h91RnJVMAOwBmh88Wrb0IDWaWaqKcmaG5wEL3hL1ppwIVKQHebN1sEIjVrGK2iCNk4Cvt6YolZL2FiP1FzpeDRbSg+0c0enyriQjejLRWlym4e7+UeR9583kvGNHZzs7aotfqdDQfyjsBg4Mal4v8FGGpEqqkdp0T7CpgxWPOegqGkykewo8Q/M63w9vfL/OSLjSqIRaPBsRwpjjh8fMivNxJDfL5JcOTi7e9/iZBItIqBIMD6v3jfjp+ntm5kCYYJgPPDudYWn6/cheKnB0jGvb9Q02++a7ZaDpM/Lj1GQJp9C5j5paDzwYyHuaAJBMRnU5G9vNAsM6/hmW5OktacVA6FirTL2TaBGPia8PvGtHxP9E57DDPc6quVcGgoEdKGeHVszENfuuYqJjFWClddF8vqihdbWLiq/pER1ru7JN+jGOv61WYBTLF97zTc7g2GWLHEv4UQpSkciY0m2WG3et1U4XzXIQgbwPOUSe36KBmYgPpPceXrTuWRM5n58ax0085fGNpKb6QuX5dG2w8cKto5EgDkRnnfZ4ZLFfZkagmHbiByYGltC3RcmU3VrGXcTN6oNAAy6+Nz9O8ac3m6aht6PAMVoWIWBgsFLdvyMg/X28i38vtSST3c9LYQBgD83cGaRgK6XDZWNOTobYniRB0QftjF9eVUY45azC846KlnpKzgGKG173TS0mILhME2MFW9zmQc+7lZR/FSrYk7HD10/ForqNlmUzC0+arEOxVGIpgpH9FuYxHDoidhQp43/3bdXXkIDLwKFj8DmATEbRLmhjlHuivBYh+z21YH8jyI2kQ6RWca4O0JpKDwLK9DKRn2QjqHfqT5jO4D/78vbH/91faMF+wNhV70UGEbshlQkABUC0KHuZcRZQBXMTXJkWVrChttl4yhDmi8lrSMmd9ajZIjslkFujG9LNpyrPJLqSUczASU5hSnW5h8wpDjWW0Rigol6NV8Jok/CZArenJ4HM7FWw065uNPxnJnfDvLgnub+fWSah2ECF17AA/yprghl+R/rooItyGw7WOMaP5t5b3YqidTvhz7+WrYGSIN5J4IBbcTbUvToVu93IFNv90cPM3ZAP2D/DuFAaGA+u2uEocLkdyR63UmOCoF/osHjbdjdkwvrY/bf7BSj4gUUqps47rqyljy6ytidgZln2Qmr0px71+fT7hKxyZsRW2Clll4L/KAV8FzO3aXZ1he0kbXtSHvQzS9YJAKa2YKcYUnyAcQRKdm0TGqXfYvOVGhSgsQNiFDG5snUvSNAy6XBhd6+HClUA8+duZU8Ml6Q5r4s29mppM8poq6ELOnJmt8EjS8NihEmvmG90/gqC72A7Ptv2rK/aXlbW1Gmc0wQwdA8LXrvXhoHSpTcp8zmlAa2mRufjwxBLj9u1ghieNh1SmrPxOqn6dgFWqn97qGkbCZ3VPoKYFNlTwO2KiUaR0UzkkV8xm4AyokbXYsu+JVEff6IJ1KOykE9ct7rFyzpj7ECZBu9DqIPA8cyw==,iv:SNw9RjwYLSmMO/Nrz2+IA0GVAAcejoz2Af+9+IoDsWw=,tag:0r7OESoIKKvwCTSoJk/GRA==,type:str] - server: ENC[AES256_GCM,data:sjGOJaUEXNJkHxtYJO0f3PpQ5dv8PQ0lehOAnVNvAccRsytt5FFVrRj64v+BLMucAiEOX9oUcsHrikB9C5RbM4FkfvWMe5ULD4fQrYU1Dw==,iv:ONbXi6onE5d8abL5cxr6p3jJd4UW46Ut0CvWKQECLhE=,tag:O6f1X5mvaUdJB4ZnWRGHtA==,type:str] - name: ENC[AES256_GCM,data:T5Q6YzMy/6j42bGlmCYrCqcAUcJYL/c=,iv:TUceXHDNbq561n0944dt/NWf9IRFyJdwhhhlKPZ38Hs=,tag:3fqDqhwDAfhtvQNMJ/q62g==,type:str] + certificate-authority-data: ENC[AES256_GCM,data:AylHh/ZF+an80mTUhqeE2vpDS2ALNO2N0CxXlmHPCGpBYLAH7Bq0FY1fM0EXbzip9VLV+AxcBQ8SS3IsmuGQcNpn2CBkmbOL24m9yBqAOACZ4zKUDANtkzTILPyUkwbmOYZgPxptSC/n3IUw6YuyJcTlkTuLit9Vsoz7GLLC7B0zQ/sg0DchGeKINMiD36L2tmdBbIHJZFa6ZgZiLVu7LAGI+zc9QtnkuMYsrrhw+1XQnVhQCpl/a/8vEGoO8qAXx6rPSIk89Y0XpKqn0+iASamOb0F7eKtQZDXNf3fJzQ7pBYmj3h3XfRySkkLEvDC8WH1DjMNsY0shZFxUpszxGkruodDbZard91TU3JS5B700SFd22G1wp4r81hSEg6KwBLM91P/zpgaTE5l2tOdVRkX7Dui15sPPRiutyRIIFSUCF8kXukbyzRFC9T0jjcMCh2+046Vn6EG1bzx3zdXXayeODo4YW6B6wEpIjReieVb+kx74yIn9EJgKz1PiYMh4Va2e0LiX4629IJ6uwlrzFtAeLcjWhsPJv1gT5n1I8Y443PSEAlGqdRsrHuf0LKVPkdT75NzQ0iFQ0EyRium2ikkjHH3YvFqStngy8odgAddwIw3wzrLz0QTCC8yg4sHUpqQVFH+Ls2/oga4bWg/VuuC3JIT/XPXO0qr0kO9TUn18e2tvXbXdo/D+sZZh621kSTvWSMKEZcXRgDQutIavnKeaFjSIAn3hzmXXteZfXsxnOeTSzDb9M6CNe4MVR6PyOrKzXuCCa/IOQ1MJn7ET0Q4zimjEkrkNzm/grjFYavKey++rvOcvRRBTswv7iaPjoR7cqh3oRXhkr0lHgrvu2R3d0LgE/EDsRtV7R0YoD3UGXyWm5y9pVOuzgKQXYEiuSOaUnuUVq5D1xcF9H79O5aeERXnJJqCd+LIr3dcm7iAZMipVjAzgv/Sf8vNusG7kQHAtMt2olug2Jpxq+3KLDyrdFwEU+x2FOpPX1M4TMQZO5wUtP5f72XJ6nc1FQYjPGgP5HpGSKkoh0Rs/kLqrnv/0OgQ2TtggNZGxBhWdmoeh7BA189xb7iTKb8npeGfg2JanOVD6WRLXcT4B4sRIk6DXPSIsVfKyeoSaIt7i9Ponk/K4MK6KeV0WV+XdNpDEUe/frdfcofXfl/vz1qfzKBaV12GECFwlKOqLRKfU6mXqBqM2XBge5qyYiMT+BlKcvXtI4A04n2M56jpRqaaG+55f5G/zI/+ze/jaUMfpoDb5nqnlAgcEIPHRNnR0ns9Ots02GDxOK0Vh+cbgGqhfRefgLBYuQ8OY20NljcdgZlta37qg9jF4qmfk7q2wn3RAYYk5YRq9T9kBhFWx4RiaCOhyuzxYBXI7JxMuGWhopvi75SRlD2QtLylSNEhXsDaJ1aUdsiuG94vkLV6VGogg7pshEBi4FGbEZDMr3eYN1+q7SSAYAp1b5/p4j2l0qvpUaW6wNBG0MWGtptgSgIyNEWqB89agQaKqEY4vvZw7QdXk4HqrIo2VwBmxF6te2FYQxli4+JUFsmN83jVWiQWJM+eMqmc+r+0iMG1BQVc2Nsc4Btz85+yJOS9XGRQCyfrK4UNvvLTo7AMiaTRjj3oDA8vazESXB/jYh+7ZVPusjaysLGvgYNWdsD8BVajl3f+z7ActnJNuyXrpzFvnYd8glHRrzPXtJxXPu3W+cWubtMUQfOtPix3zJYuKsquFwS6vaVysaD4e9/mKjMKVH1dEkfRiawsu07dnnZ5YCZ/Y9xdN6tklLnMHh2iEJMwbjzmLBO+MU9U8ETqvjwaukfeWNny5SnzBTw95mmxhUhaNeKgrtWe91H6kxGt/TlCWOzLragcKGvau8WyPet6rKGrE9r4WGmOibt2GC+yqZ9ZzRArqRYQ/696GFa45xl+6K8pUK+LCEg==,iv:pbWOYLHhJAInwkHqwGC48Im4xdyRWcsG5i+XvO2lcik=,tag:tc1lwgAqD3oPZa7uB1gang==,type:str] + server: ENC[AES256_GCM,data:S2b7TDewzyCMpuEgBO+lwODqaBPZotF0I7gzMOf18cSG9mjkz6ojs9Vk8jK5fwMZME4tSFjwC+yI+UwnQak/m9tZasZfOEK0FuN1lRQ54A==,iv:hyzG7zKNeVdJp13UZaH/sBTQXe9PuSykgCKfI0DkyCY=,tag:fGf/MhbkIk2aT4UKquh+5Q==,type:str] + name: ENC[AES256_GCM,data:6MDBvzbHDKERzFlLWi9AnOkeWy36GEQ=,iv:uWzSspjA0emXdg3+FtkdTGVVLUi9FtJ5PAM8LsukM0I=,tag:qGYdst9pki1n4vZzbfXmrA==,type:str] contexts: - context: - cluster: ENC[AES256_GCM,data:ubveHqV2JHR3apEGcEVVtg4ym0jh+5Q=,iv:DD4OS5+TOwQyN6gevKQjsVrYT6CVK4/xN81n5QCO0Gg=,tag:btI/f20ZuL57Vcu3G8Ncpg==,type:str] - user: ENC[AES256_GCM,data:Kfy0Qnp9eZWH2cK0hEQAMtIFkrlqeq8=,iv:nXtiZUkmKaLLDsMfhl0aApxLsYGC9r+Sh0/yomD3nls=,tag:jVCFSQZHpCHEq/X+dcTT3A==,type:str] - name: ENC[AES256_GCM,data:h8XW7YdCVKOOyinoNY0Pb9UrApNjvW0=,iv:YOwJeUZfE2wyiKgIc1Kjj+b/C8PbNcpkzVK9bjASHKY=,tag:mcW7/QsKATFUEnl8bfni8A==,type:str] -current-context: ENC[AES256_GCM,data:6ldoUtzpD3xpKeIMZ7e85i1Ht9GTwmM=,iv:ci+9F7J+O+XGWw6VMK4PD6u1KEhwu1xLP4GE9F8t6oc=,tag:rUZyda5JRJwC9LzU316zzg==,type:str] -kind: ENC[AES256_GCM,data:RmpDLXcA,iv:2rnkdWK89mTzuM0sYhGO9rbyPNirwRF7czbpdb8V5yA=,tag:VE8KJRElU3sKs3R8qamTig==,type:str] + cluster: ENC[AES256_GCM,data:NXfhLeqU4J0pbL6RV+oKjcISaUddhg0=,iv:bSch9N8ptQ7Io/1G++HH0UxRqaeT5Xnzru0e8wXjpJk=,tag:xy+VV7Ff7fzZxVIT/err/w==,type:str] + user: ENC[AES256_GCM,data:ySwATaTIWt/wbu8A7kBFpdwQzgD1j/s=,iv:FBA4/px2whl7gcotXa82zWsvVc1lN9A9XH9Z3VWDPvo=,tag:I7MqS9KNDGLH2534C4OmMQ==,type:str] + name: ENC[AES256_GCM,data:4dMxibxCOjlyqct9So24bcxuzImrYCI=,iv:P/Fp8/Zofybjcdm3s2hsUP4PqkiBPMIxVGb9jHvIiY4=,tag:/eTZxiy56Pv31J1TldnfLw==,type:str] +current-context: ENC[AES256_GCM,data:BsvfN8X4lWF+w/WUiDzCmJ9zj3fefOk=,iv:35osaNc0egPZda6KImdSSoRAui6oIQHIaHTEktWgkqI=,tag:s0fVURcej7Hr8pz32udMZA==,type:str] +kind: ENC[AES256_GCM,data:dbi0U3/R,iv:HXH+Dlv5wLI7Hfklkm1EHzG6xr/zyDVYzS2uPqGO4DU=,tag:OCIucU9fMKxUKQqfoqI8wg==,type:str] preferences: {} users: - - name: ENC[AES256_GCM,data:JBthcd8lbiy1AiRen1ShGDzV6ZVfpyY=,iv:bdqVBy0NWQTEi3YbOqoxrmfmxejGzFXSGcNlsfwDYzc=,tag:Q67OWexlYZxE7L8lzEkVYQ==,type:str] + - name: ENC[AES256_GCM,data:DlseQ4eDb9x/dOjO4BccHS9BPRA9mlo=,iv:jncX4keun8alirCaRDJ+PAkBf7BCsDBFYATYBPxG5O4=,tag:l42QMOEdaWX3l6CZVkm93Q==,type:str] user: - client-certificate-data: ENC[AES256_GCM,data:Jb9yNhWGZjd6SMqbZq1kxTVnzfWEMFlD94ijBkDnt5LwYtRLlvc7SEaEVhGmnPn5f7m2TJbEGXH572L1QY0AU0LNV21v0+udrrXy8Gct53QLHt+anAK/QPqSk2BAzFtUjw8CHfJKFIaHdB19/0nCNlYx4LECi6jk8Trethj1TvLl0L64/4PkSgOff8NYJmIJfNc7ElvlfXBL3hBDjnKzBJro8+XT15uiY10YhkxvUYnXJmeMxvnmV8IAxKWmVF0rVnVtCAGBskZJbPU5bjPuQGP7aqZZBwnp8SbRX3s8iEdR3pGKe3dE79SYs6t360IwLnI79zutU7SNDzbMQjhQ5KAdqepXBU3G/twU0v2FXlynlBoiwxn2cfw6zVnYJTBDfMh2jshMVCbaTdZNIDqZG1kQOgpjWcCLWVhl5yF/bGcGTd7E0cQbW87IoBKEOgscHpkdLMpCCtRS+vEE1YXAHqRfoc9Q62mCGRDTjzv1puDDAoAvsVrd+CubPVv/2j3kMuj5CYixZIZvcAw9bXCj4uJqr4PN2xCxf4H/QrSjb0kdPmSxDhGTxnt2D7I36M2IgtpKHTEC0afq9mVD0MmBGZmcczZEjM8170anAtw8H4OE8hPMk3nZYFwT1giSCj62ipgEGpLHFq5gSEH62Qdjp7aX47esh3klnx81xeuQilpV24vTpYuhyJFqfEoTDfjdXMAhTr2sq+6zrE8tuplyQcftcw+P3Q5Vq4Vd6KNN7Gz3JlRs3U4uFHx4XOzAy0q752FoF/0EKc1SZi8Rmcc7nAQrRBKX4dfRpDu5bYgu1WCsmFxWcwpenv0VHybIQSEpv3s9rWS72xR9nvzwMWeGv106uOgar16jwoac/AMLYfLZ7XAtu+LkQc6UcFOKCZk+bfA85MQ7t6gErxBi8dg2tivzuaZ4WF9zDGYTrsgmaNeh/TPQEaAp2Wm0qP4xdtehk+YP9YI907ZYyUHD7iKRczuublclpE/zCD7mbjwlWfR/tztTdyGS4Ii/75+E2Nmfo+DdyCNTD/yZUvC826uPufdN5iQtnDVEb10Nc4LfAHE4HyM3k/956afvMcV1IftMf1lK7/MRZ616o1zCyAKxaSXViWfh5HzPHsZBwP5UB/4FXSXLUCrF9n00YLgG1pmc1PS7RL+ZQrEcjJ3us+Jwds/09JkzxU6EFeHFswaERcT1XSRtLv0WYpXq/4x5emcn/ds+kwRCjkfNDHvQmgv+T3yizN3fJlCDnsfPlcpJBcFu4//VO0+lc3U6pWB0cCoe6sOTy1bKon4YISsw+FSLZppzqy4JUf2tj9I6gQHbFdE4YV7sZo1VYR11lfyQUBVMfTjZrzoFFZGfy3q9fv5nuTfWn912RgRh4GiH4HsWGCNPEJQPkMNq+oXbSivQihtuOpizDL0JVMBnd5wHrZoq2Lxfmm/pXFZDPBHjeXYrypJbEHT6JO7ACcZyTkiKXUnOSU5imVV03mpNC62GinT2QGrGGrJnXnDQINdhHo8tBO6/hAbj5rreTzszt1BaSEzNoZWk6RNL7XaEP3+RcfCfEbf4SIDkNCTfS2EZkovhp0Jd7QCD5goxUrIhDNuqe5838oMTY9+CbjG0+/QjkvpRFVQcAdb7s59hOKkpJSwu6jnFoHS0Lv+R9bdR71iJ5OMumk00nhTZ7379llq+Cstv8PHz59iZVLcAc6sHKAwwb2wYyK/YOC6NH1a7cdNwkOS7kUBtqPeoTdtF5omPCUcuvEnMwKH9EfQNye1hZXnalfwqZqGnULjhmFDef2JwjiSQiA60zFKwlRDBPDK+Jzsy1+d94lKq27p29rEP+gNNNJ82blED/I8dAjtH9k1hAJSuc9lp9YQDPV3plzexhjKKeSUQkd7ETD2oOQZNL6cHB7R9KqSztENuh4q+rzvBcOYeHeFt7penrLDWIm7dqhptTEdnVrwxkch9EQm4XU4fgMdwEKzNUDl3e7TvM67zDKtSc9XQwu/+bhxwg5poK2ValVSU9q+m9jZtPuZE1f3HjqiWsW7jNI4CWn5GOlZ/q37084X7uJHY425Eu1s/,iv:oXRJJkfCfp1V9s5VuFer3J0omWc+YqP4agPUnhcv3jQ=,tag:xqBBEZzE6SBjRupTt9iKVg==,type:str] - client-key-data: ENC[AES256_GCM,data:Dn3I/C2YLsXtOrDzsTM53Lay1dChtNwHUpzoPHfEuEvLl8fTm6nNSRmNvl+bkRGYf87njfsyF3D5e7qoRGRpPBNTaePtsk4+pvvpAO+OrUCWZKBc0mXTXdxR3yqLi9CwawfbVZPPcWaMWOD241tGiP3LUL6ojNZdKKCjofN51iSmhszN11mXcmDEvAuF3oEDH7Drd565WkxmOqGtjK+sK3vtUtSB/pZCDKeel8JypczQs/lX+zcJeIGrUhsEDMl4quDZLI1BLE04le+xC0QPkCPL5KNhbwqqox7ccXmNwsCrt2TvWn4Su17lBpWofZwrmcMUihUqp2ckwU25/rNho1VENRxjU/FQ9XNEzQ5XWhqlUa9se4ZKIQjFLsTBmC1MD/1+J5Xpuw75AKbkNLu9xaawTkdzARMgSP6DfZ94mlwB9kdZ/uqkjpoPy957TykMRkYskbGtEz0rZnJqzPyZFwNM2FfOQwaZilwYHd8q1E+A+zsnEIGYIJwG0LllFDy32n849bLhPv+byee9B8CoyB7U+kxLsc/NL2dQmBKlNBxoTxrdAfaHBlACERI/AezA/RTHQzYfAX1Y9CZJEzIXMne0DFf24km3p4rnH1CTGrrDBkRMr71BmDhIRtsZMfoJ4yaz3n2QT1J1qvKt0VIrjkskO2IhJYN8ElzrMDdhuOZ9p9DtpUxSfK3h/3mgQChP5/MXBTAqlnt1jeWNSkNQZpjDrsla324MNulY3OQXAHZwkko8GIP2qxHYQVEzQZSrFKpT70b8g/Ne80atDt1NxoYnW7314jQDKu3AKuEt0BPFfiA7yiWiN3sedEFbF01MDKaHDtzfSdJ3voGBgCkR3d0twVYHjgbwkIEauLfh4X/oUgjKYcaUklJeCMlt8YBZqUv6Ur7d1SeQsce9YNERldWowGkWfuYFh1rv5/B6mE34SJ0hgKbYFt+Ijd4HCF/z9TrrAN6VdPCnwH7BKONVB1x+IOIgAIVUZHIJx/R27YlCqwkbco1qmjozdbze3n3vzcrfklI76oWs+21SWMb5mQ/tlsl3PxdFe+0pygcplOJETWuIG1PEomkmoySmPDCk2hW/Bh+9Ac7tkyfDdhWg9Wav0mmTIYXTUpU8hKCg6Ho1T6dFNjfp2fzFfChFcozoC8T2BX+IYQ7maSG8Ar5TkJDY2r8lafca7HJ2X2UbxWX+Y8OeiVO9cx60Xd1dZosOGQ7Y4Ntdd7irol0FHcWnIII1e9dm4QiScqCOWTD5WYpvZ8oQk8cc6kk7Zm+I+GkHTTaFYhxG4G5X+vHJFi1HqpCvqeS+Gcy2KXBXPamKehMF7FFnGbGllO2Y+dD0G5+woAqSN1fHKnunXWQOYwx2I1yChgqy4YQm8IZV01EIlVroi97kq3LQuWJqdsSGb+uws9hvlTVmprqgAgapeQQBYovuB3kGg/bp+LDwBhtdEoiQ0m+xIbsOxJeYwwgwMoo4wvju63dhUPCgZJutJtNdZwZuRRKsF5EUkfL3nFFqmiu4AFWIRfcM/e64OTDQb8nTTxpkFLAca1i0EAODOK8RBqkYNrIOktgvaS0XgKJ6nG5URsx4QL4dosppz3/MpFxZUKvJxzG7eG7qtrqobq2XMXGBVJ7NyhDCGJv+q8t/A8JK8aen8s5MCCekQnqJityJUS6++BpGWUaRD8ApxajE1OhFfgfIDuB5NX69ytLxWDcwQWlufndSo1yzRYW77mzWdXDHqB0Y5HzhDw1k+JR3kZ8lWge6uh+y3vPAQDNygbx4gXvnqeL3MwcKHGc+fN+DbOgF9Yd9VrP1z7Zk215dPLSWsGF9Qtt3g2/H1lkfERg9UlsPK68hfVqsBJVuoJ13TVt0y6PL1lat88RTCQoVC3iEIW4IxXIX1NMaLFpKZih1Ec0aFeCsrgYDTCm0ojPqS0vTKCisopkknmy7mt3Yzgx37VoCBYSK6ZyG8iXRq28IkOxLukj8L2oWCmj8Nq46+qKRnMghsD3F0gOby3wcbxMU+de2SDUYQHoDdJOuUxoL92J+fHMShT0MuPVQGanIpVBdEqi2uwqotLIk8A4j+wJsTd8P+ICLJSAvyDuZj5iPgkpkfwQBhyUkiYmebVvc5UYNZo6k9H4JlmCFvbPZ6Ahx5Pt90VDPecT9wsFwSXLY/WLn2bZ8ROFpaLVd7btEZ8CBQuC3Am1WSxwb0Wkfy5xkFgOU++gpDPcIrIgYovWfI/CGkMTw3kMFidd0jFBx+16xtleWacejFAZmy6JaHkP2UOpf/DagxH+xPJdRyKZBJ+PmDa7d90Rq2Pql068VouvwvSdkYQ0jXLx6K+nri95SlhY1fNovbDMMRQ/oowIOKsq603lsJ5rRQKsHL/wNJN+GEFk8CcwjZrzdlXY797qS+0OX+h7Vuov2oupYcnJiBUgaaRBhErVBBq/plmtiBPV1/QrF8YXfE50axuDFNgrQ57GxEtrUabFfLIX7GMxQqq82SYtUXZqzPKhrLyBtxypmWdI5/lrgSMbm27ztBpFL2HfWHeL2yD0DTtOuu6xu0X+ZVFhLk/tyjHhYBLv1sUDviKArQ3MNKjU9wBmDsi3RcKe2ENboz1xCqJ8La5t/flMAYfbp/QnYqaO2J2KXZH5+YI3wY+pVh3HQk2Dxa5GGhdj9RioSqcPXfi6IPDptDH1jVmNlZ1YXrP1HjQcG74NxBBJ1GWZGd3hF3X2o6CjERfkurUYpEYIqtaY1nv/YHf/s2LqiiXH1qsLTUgmOW5kWrfC1zO8NvylKg1G0F9FWWGLJvVdGCfzu108tAUMXks9ziTkWveE1uluBhm9KM6c3gUdfcUr3HpftB7rvzsj47JxtmEqLcI3JJItquv7H3fNZTIO1XLOwC2s1Gmd0lhqtvID6pJB65EDhXCaCUOtjTC9dWXHFKQrKgQY/lWGxVveK0QsZYJ/WincSnNekxGhpVUbYCwzaY18vf7HUiEvtOcJWu6XqjzAN0w==,iv:LJG6fPYjjrO6kahxdaLB1vF89f8RA74nwp7THrcHGsk=,tag:q3CPDUMI9MRAptVeNzcMig==,type:str] + client-certificate-data: ENC[AES256_GCM,data:G7HwQFZRokaLxeyEKU7vVUPYliiowvIDZagr2aVKeXNso0kPaaDM4gcHqwN9nAAm14Nj+GuifnQdzNFdenFbQuTCXhXl6wBg3aAokwig5LD+0+QSVGEmgjJdHZlaxgRG68mOsiQ8rno05SnwnwoVvWORhtHw3SNyujlw0e0lFkJOdUhIbloEV/LUX8zqdqqNOzag8KQr0G+Sn4yFOKlI1EQYA3ZxJcunvzun3ET+qe/Dk1jKkViyfUXWKHNyQFi6XbeKvAxPxtJCIy0WFvPmhD9ISwEWY8IWJ89NtfDwbSdP7iJ2tSwGvuL9nkRS1vSKiKuc4xGpd/GjyMdDzI6sxJQJn6POFqwPlBqHZ3RAZu+Aho8N5tbDzV9R9PLkXhcsl8dfVsrTQ0KghjQqCoc7Yhcndk0prOLo+YmseH+B4ayN+zivp6M65B9vQ2EaxPiw99IupNdDs9/wK5asQpvhXK8kKbvImbcW6NdzBioCMdWXA3XzcMrZJMSuvfXykvM0OCZu8U6gJYq2knu0IFTulyugB+M7i7AhHrUwNoJGO7aHd6CvYIRc5sdIYErYJ5Hdb6lNajdmtrPKGvRJMztHbKmhofjEWeX7+SkyjkAeQyoAWbvZ/AZvL9RyR8fVIqfNWdziBeOFM20jk709oORZ4VMJe5GCD+etFburbCM86mUliNyErdmBu455Mc2dWzCucWRurLcUWS+kNoplNdgGGcXhF9s6eoz5jQAQ/vLoO6Wt3jDilzZQpYmw7XW4zZKiuXskxMQERmKidiQ8FPlVXBDbtjHA5npGLbjUsKm/IBA0ZU4Isq7truinWt+j41923EwHfG8Sl6iFmElmrDrhTMlyROXTWyjzmDedHVg0SjAbC44Y/lDjPLcuIZlB3IolmySuzDLSaHgQRz5prOWzm3qM/6dwJlrKfajNF59xy8hqi6jbv/3XQdhbCEiBG6s65+atOx9mJrlZmxnt+1jNFLCKsOsH1orlGodq/z6vX8Zf4jxClSkWX5Y2t/3MTVrcvr9xwonGENTJOcL4lfgOtz6rE7nQuzQRXOnYJQfr2btsFLKyfM8ibMVWpKhO3PWuhxYUMx171J1kofsd1ux+b3MYciNvv7icwhtjGVssFlSMbsrgPcF666XIZytWoJ/Nyvj4KrfLZ+rQNCPnKiDobEXjRn0K9YxJqzDVtoexEMXCKPwysv5pf45zOppVITC85Dg1KnqwFC4Ok6pcD/DU+rmFyR7mJsQV/a+iqv/3CRizZdtqOv9tIQ1p25CaNp2V02WdCO7F1NdHCnJcA2KJA/wAthqEkElONXq0G/X4KbmtVH1h4T/t0+dlWf/j6+9NudigNkK4lpm2CClWa1yAL9r8SWkFHdnUjuUud7XMZPAn6JmaOBKy4P41k6AOJJODW5y4qSJ+ZDdI02R8Tb4828BRv8JMLCngXV0hzXwBMteBTOsnUacHNFDrw6/duMexoEKPq+nqrhrmMiMizQG1YM0e7Wa3+gSFIN6SltsaocDahr/f3i9RENrG2Ml71Tt3OKxSwATjkY60AbpcCXxhavEqkF8VlKm1IXun6jdlq2S96hBh/I6coE5qRqs325sgfW2bmhqzPMIgLmcGHEnkUcKVxCf11oiNMfD2Dgc9anDd9+7S8E0d0SP7EwzCE7RPPRbZM8S3panqtal8nOtW57kwRdMg9ngAEyvRPBmGRLEMkAlzC+i9iwk+8HdVyJk7ZvRSQUOjkuAAU25d7+g7sLqzBGi2vWLQd47HlmycrzoJcMFHclN2GjvCV4JmIN/22CX3G6cED8HHnBAgYAHO+aTPACe1cmssugc3gHsApSBqvyiX9PdXUZ71VoCJZwIaUYcj6Qm3tXDgDM7dzcgpi5SOIGhNV+zcO9LhPJgEZFfk3gY5pWa92yYBBpNKlovCtX5EdpyFW9KRglL18FdNBR+LbDrsbhMt1UTlDEDCKj+hCtIRIspoSUVXzpvCpn1PpLMiFMMuIpttk2976U6jzr6ijvQBw5DkiI6DfI/ypcu6R3sJr6rjIh2YH2ldURb7JqeSY7Z7s0+/lTX3,iv:onvDUevuQrPu9iZE3rhxDlNFczcekvzA9l+FUZXl6Uk=,tag:pUyO6w73kIpRMAWgUnReAQ==,type:str] + client-key-data: ENC[AES256_GCM,data:r1pzrT+nN/7xYX7x2biJEIn+8lf7C1EYUtaZKy/Y2GpkKC/l3whC19j9Th5SDPOHuC9rQpNlGqaw8mp3vwrzEFBR2myvCEZ9Q/h3wAYCzSy6OXloxdr2uyfst1bU0GJNDkmH0B4ANt9u+hn/7lEgfh+5FDgDeaxnEym/48YN9T2KkV1/8c9pAWPzY4k35Ouhe6iFwOISN65xWUHyPfMBGsiSFb1ILX3AIG32Rd9L5D24MHeXrsvV0pgiczJZnE18Cj3KZo6KSuNX7lRZVSNuF0Z32C7vXX0//rXHX1dtRx1xaWk+0vLKBn3mZfTNG7A0wsRqWBVmeYF5fpVMWJ/zHZsuNkOXZk+5LfjFNwRkGUnqPsXwOGpCyVWygwY+C71AO9p1QOUpYnWJZsMdF12/efvm/egHqXBUmyVfe2YHet4ieAeZSkRwfGMLtB3x/1yxMf3v8Jf6YGeJXRPVp6/g0h479LrWq88rJHhEdF4vJaaIQvbMCnmabzhED4si98LLzl2fgHWQ9zh8wKjgZBL3YVZDB3ovWlkW1cIqx82jPhaO0QUpzp2Z0L/oIMN7h35rc34vBVv8mbfPkio9JRZ93cb5X2Mrk1p83C9RuuvMKZhdy/AuB5+Saz623kIsqLK4x806pvwNGjfmJes+d75AdLx25LozKNPXP3wmd4w0aOkDdtLzhkHGGUvu0o5Xr5qYP5h5w4MrCD9nQ6wy7OC/xl4CKzzADKSXx3JOLwKENVqNB/YO1kKnMARnKKW0Bz+1e4shHJqm+8Y3mKQjqMy89oz735ocVoWt/SHrWEJdbc7/lcRf+gabP83h+iYDMBbD/1LR9HdSwvfoFjvlcOBDP6brxGWTCyTpnrnXEbZXN2Dx0eTR1AWaXmxxpaAOQNWaFdGPUzon/iT2qk1LE1kd36P7RPi/WkdXHtGv9+URVAtf2gP7YBREN4KVEjdz3bLUxSsoBvgVUfeuwQxJRlmXISRi05MCk3+VQdynYt8hqQbjvXDGxbk+KGopmxKGYQZoAiEPGD/mDvTkOts8fr/4Tt9MFPZUAzU5N8uJNLf33PkHl3MHmltzaS4T0p8BQ7SeMx0Ap8+BW0QDTDylyQpbmwSMPNjZXb+bi9aPlQCe31lGs0RWkwcAwDTStyhgzWQ9DogzEE2LieKKQYQx45+SJ8fxRjE/Ako6QUhUC8RNT/Ld+u/dPeQ9x0pH9nTa5OERecQtkHSfcJeB+a+kUTXHsi2U5mbyxupuyxjVsiEZhStpqgoDPODNKJn3Q9+SjF9ND2J4gGfb05gkO/3U4XneIdkoybsJoSzJiIa3LvJnPZZ9S5X4BNm/zPIOlAkmgKX8jsIyZf5rmpnvNRdYNDL1el34RDopFLvVOD6fXyTK8R/yrKhzM/zXxtH/eBchJo3W438E5UGFlHKMt5x9D4wn0SpvVYf7QPiAqAA80QQv2vIcS5pjNaQPDDLk9SAje0eb5Y6r4oGlaNm41aNWxEej67HrG7lOnP2ytZIBaIUHM6C73ed6Sn1UO/HTK8PLsbMJ0GtPemZnqa/mxbhTaBZRyoIL7JM9SQ9xu14fjFUEFFVEQeYs2pV2xgszE5hbCqjX4769sidwihe63I8t8wVwt1A27ogb2vqgxhhEHNYr/K9r3CeVr6nNOuIQfZQX/DAwmAZ33F5QPOKP+12GVn+Hn8189Tl53Jt8SC1WcBVWq8+xS+gljLq7eg4RMQSkN+Z3EtOHdprbs74AMCgEo6bSz7EtQEySre9t8ye+CeyBnNYOoTxV7qU43wl/0DLJ+1gqrFXBcetu2jAcKRcK+qPMtfZt1NxAF3bKclB70Hx5p6zbaLmWxgc1hc+t/d6fXvI0/Z9DJZY0eFb2oeii1pncb9JcD6NiaytxUjFKa37+RSDbR8qS1KF7YUgECJpvAvq4w+E1o2kI0hIQ56gjAI1LVDTLmu6MPlXTVNzakrmDG3lJG9uKrDHWn5B08GwNTJQxLz0R7fE78bvkmfJjeeDO5FYmqbTTp79rpWe88iJxwbslOHwPM8txagh6YYl3WLcYPfn+Vnpvfn9QhOFBTPeJ2EI/MEKLqOhQd5oh0ZYk96be5PPeQ2Zkhpl1DwCTeYMDXWiWEaTFig9ys8/B71MNUqJ9nRriYPh9NlR6tpmolGorQyEiuJaBgIj17bhSytviWX4qjdo/wRrgjya01J+/S36xIKc38hX5duYa82oBj1uUfum5WHuIz1XrUCZd6xgTSo+sclklqBk4tT9NCcudRa6Pn17H+85Oi4Ip+aJZtAUrKbrROv28R2OLiqHopT2nTN+9+dOy1pPELL0l5qf+7ACPrk5VuqX46qviKDQhjK2N69PD8j+0AWQBBxyJhDNgzZXCvNvsf29rGe7nf+87tpjHJhHNZgUOfnXg71cCe3HAko6nZuUQzr4r2GxwymSTdA/KOKxp7b+PFFER2UJ2iEWKqt4/thtf6TCEWnJ+jqB6rNduWXzvlBikvTsy7N+O/kgJCh2NJ9aHIDOBHv7w7Dz77lSL5Mfcgnvrk4Gy82bKp8vCvGQMjHASS8xIbIn/OwEV9HKO2PYaAONAfjorbU0DtUHJgSMUNJ2KZkY204Kaw+IX68w5lnP9EeLAaftbSTEVnt7QPWH5PbvAjW9kjcbf89NfdaWN37TNTsw44zOY+WiBTqLbJtEVmxcCIE4osWk4HAiwVK1wHRYdzJecOP0cbN6xnvtswfW01bg1bmXsjPReliCDTc2qzPaR2g1LuW8J74PNJ9ydQry13aZ2ipOV/jc9oeUVI0zBlSLi9FC6Xfehou1JUB/2UzcESQwIxaADUZlKq6n76FlKl/kRJGKTxnsZ3hRAOZ2JTJsR156VlHl/U9XXqtV618c7XGFWchg16UvTtZLW3uJvHpZJa8WTxvMn1upjH9lz1TTWPcpKy8ATIWE8ENy6chTJYUM0mmA6RujaxUU36inSjAQoZMYFB0mDSW/5QwOvQt0/S9c=,iv:6QCYm/Zq7nLQ3WZArqN7o7QjI6hgb5EusJscswhE214=,tag:PR/+r7Wiq5CXxksDyOmTYA==,type:str] sops: kms: [] gcp_kms: - resource_id: projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs - created_at: "2021-05-03T18:24:30Z" - enc: CiQA4OM7eBvDdmSosNwoV1H4lGprhSnF6VzDyRaW6U1D9qi6XHQSSQBy9hCY8tILK+/MkhaLO9jCz3fDyhOiF1lxq+zw7GZF8USAZ1ihXFp29CLO69PQa5ejJqZffdWydRTJxuQ1Gd+pEHzC2KmCrGc= + created_at: "2021-05-04T21:58:02Z" + enc: CiQA4OM7ePq/gEbeVzyyz49K389+FaatQjjRZeMK1ybwrioWnFkSSQBy9hCYjrj24PexUYPKIw031vRcp2S6Uy0jYjCfAPyDHtzDwKbU/7ZzScV4FyHTC19AJkni/jAsQqe/EWaEk122HvrIQzzUxCg= azure_kv: [] hc_vault: [] age: [] - lastmodified: "2021-05-03T18:24:31Z" - mac: ENC[AES256_GCM,data:lSuWapMSsXuAeeSCKHGRsVRXtmeUpCQVTTKDFYHfFADyWF+t8TgwNvRvIuicK69esHW8GnRb5saakKzmlZjwFJKZ+lU8ADXRIY38uRo/4OEvzKhi71lNHxSqZHG/RIPbFsuPoF+xNpqBmfoGmKhrJTxP7zs5hKz2zwZ5GuUdgec=,iv:yVgiI7pbLWPXl1++ZVhg31ASIZn7FgjU8NbouldqC5s=,tag:HsQXJX5JbLVuB10c4JzMiA==,type:str] + lastmodified: "2021-05-04T21:58:04Z" + mac: ENC[AES256_GCM,data:nHbzDaorMMAL/1xBxftf/SUXiZAGZbj8HgfD+KOChoWV1W+njSa4GtOOfhftccf1Z6N0U2tSocEAYR+hNQc2zJrgQPUJrKg7pZHqk+R530kX1TL30AYtLDLdspnJ7JDuPv84DdR1DuFTzzkpdeK7lDtKH3etD6P2bHU9OeKAwOk=,iv:tgLaKzv9VFMeLEF1hE37Re1ZZA22qe+BzXq8jN0F5zA=,tag:97n6CR+YgpnPxdzZjXQgkw==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.7.1 From e69fa328e8954d05f1d6f04c7a4f4b75061a4c07 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Thu, 6 May 2021 12:25:25 +0530 Subject: [PATCH 10/12] Rename base-hub to basehub Referencing base-hub from helm charts templates is basically impossible - go templates can not use '-' in their name! This makes referencing them easier --- config/hubs/farallon.cluster.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/hubs/farallon.cluster.yaml b/config/hubs/farallon.cluster.yaml index 7a83f7c4ef..ab9bf731e4 100644 --- a/config/hubs/farallon.cluster.yaml +++ b/config/hubs/farallon.cluster.yaml @@ -11,7 +11,7 @@ hubs: config: scratchBucket: enabled: false - base-hub: + basehub: nfsPVC: nfs: # from https://docs.aws.amazon.com/efs/latest/ug/mounting-fs-nfs-mount-settings.html @@ -67,7 +67,7 @@ hubs: description: "~4CPUs & ~15GB RAM" kubespawner_override: # Expllicitly unset mem_limit, so it overrides the default memory limit we set in - # base-hub/values.yaml + # basehub/values.yaml mem_limit: null mem_guarantee: 14G cpu_guarantee: 3 @@ -77,7 +77,7 @@ hubs: description: "~8CPUs & ~30GB RAM" kubespawner_override: # Expllicitly unset mem_limit, so it overrides the default memory limit we set in - # base-hub/values.yaml + # basehub/values.yaml mem_limit: null mem_guarantee: 28G cpu_guarantee: 7 From 00ee5f4c0d1b4fdb402b7957afd52c8833dc73f5 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Fri, 7 May 2021 05:15:03 +0530 Subject: [PATCH 11/12] Set scratch bucket env vars only if needed --- .../cloud-resources/gcp/env-vars.yaml | 9 ---- .../cloud-resources/gcp/service-account.yaml | 7 --- hub-templates/basehub/templates/user-sa.yaml | 10 ++++ hub-templates/daskhub/values.yaml | 47 +++++++++---------- 4 files changed, 33 insertions(+), 40 deletions(-) delete mode 100644 hub-templates/basehub/templates/cloud-resources/gcp/env-vars.yaml create mode 100644 hub-templates/basehub/templates/user-sa.yaml diff --git a/hub-templates/basehub/templates/cloud-resources/gcp/env-vars.yaml b/hub-templates/basehub/templates/cloud-resources/gcp/env-vars.yaml deleted file mode 100644 index 442e8c23f1..0000000000 --- a/hub-templates/basehub/templates/cloud-resources/gcp/env-vars.yaml +++ /dev/null @@ -1,9 +0,0 @@ -{{ if .Values.jupyterhub.cloudResources.scratchBucket.enabled}} -kind: ConfigMap -apiVersion: v1 -metadata: - name: cloud-env-vars -data: - scratch-bucket-name: {{ include "cloudResources.scratchBucket.name" . }} - scratch-bucket-protocol: "gcs" -{{- end }} \ No newline at end of file diff --git a/hub-templates/basehub/templates/cloud-resources/gcp/service-account.yaml b/hub-templates/basehub/templates/cloud-resources/gcp/service-account.yaml index 9c25341fb5..1023c192ce 100644 --- a/hub-templates/basehub/templates/cloud-resources/gcp/service-account.yaml +++ b/hub-templates/basehub/templates/cloud-resources/gcp/service-account.yaml @@ -37,11 +37,4 @@ spec: apiVersion: resourcemanager.cnrm.cloud.google.com/v1beta1 kind: Project external: projects/{{ .Values.jupyterhub.cloudResources.gcp.projectId }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - annotations: - iam.gke.io/gcp-service-account: {{ include "cloudResources.gcp.serviceAccountName" .}}@{{ .Values.jupyterhub.cloudResources.gcp.projectId }}.iam.gserviceaccount.com - name: user-sa {{- end }} \ No newline at end of file diff --git a/hub-templates/basehub/templates/user-sa.yaml b/hub-templates/basehub/templates/user-sa.yaml new file mode 100644 index 0000000000..102c256576 --- /dev/null +++ b/hub-templates/basehub/templates/user-sa.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: + {{ if .Values.jupyterhub.cloudResources.scratchBucket.enabled}} + {{ if eq .Values.jupyterhub.cloudResources.provider "gcp" }} + iam.gke.io/gcp-service-account: {{ include "cloudResources.gcp.serviceAccountName" .}}@{{ .Values.jupyterhub.cloudResources.gcp.projectId }}.iam.gserviceaccount.com + {{- end }} + {{- end }} + name: user-sa \ No newline at end of file diff --git a/hub-templates/daskhub/values.yaml b/hub-templates/daskhub/values.yaml index 1ab216ffcc..c04a825409 100644 --- a/hub-templates/daskhub/values.yaml +++ b/hub-templates/daskhub/values.yaml @@ -52,30 +52,6 @@ basehub: # The default worker image matches the singleuser image. DASK_GATEWAY__CLUSTER__OPTIONS__IMAGE: '{JUPYTER_IMAGE_SPEC}' - # FIXME: Only set these if scratchBucket.enabled is true - # Explicitly order environment variables that depend on each - # other, since a environment variable needs to be defined first - # before they can be interpolated. - # See https://github.com/jupyterhub/kubespawner/issues/491 - daskhub-01-scratch-bucket-protocol: - name: SCRATCH_BUCKET_PROTOCOL - valueFrom: - configMapKeyRef: - name: cloud-env-vars - key: scratch-bucket-protocol - daskhub-02-scratch-bucket-name: - name: SCRATCH_BUCKET_NAME - valueFrom: - configMapKeyRef: - name: cloud-env-vars - key: scratch-bucket-name - daskhub-03-scratch-bucket: - name: SCRATCH_BUCKET - value: $(SCRATCH_BUCKET_PROTOCOL)://$(SCRATCH_BUCKET_NAME)/$(JUPYTERHUB_USER) - daskhub-04-pangeo-scratch: - name: PANGEO_SCRATCH - value: $(SCRATCH_BUCKET) - hub: networkPolicy: enabled: false @@ -116,6 +92,29 @@ basehub: break else: print("dask-gateway service not found. Did you set jupyterhub.hub.services.dask-gateway.apiToken?") + daskhub-02-cloud-storage-bucket: | + from z2jh import get_config + cloud_resources = get_config('cloudResources') + scratch_bucket = cloud_resources['scratchBucket'] + import os + + if scratch_bucket['enabled']: + # FIXME: Support other providers too + assert cloud_resources['provider'] == 'gcp' + project_id = cloud_resources['gcp']['projectId'] + + release = os.environ['HELM_RELEASE_NAME'] + bucket_protocol = 'gcs' + bucket_name = f'{project_id}-{release}-scratch-bucket' + env = { + 'SCRATCH_BUCKET_PROTOCOL': bucket_protocol, + # Matches "daskhub.scratchBUcket.name" helm template + 'SCRATCH_BUCKET_NAME': bucket_name, + 'SCRATCH_BUCKET': f'{bucket_protocol}://{bucket_name}', + 'PANGEO_SCRATCH': f'{bucket_protocol}://{bucket_name}', + } + + c.KubeSpawner.environment.update(env) dask-gateway: enabled: true # Enabling dask-gateway will install Dask Gateway as a dependency. From 8464f1b109c6eb07e16dd744dd03ea43a63f90f0 Mon Sep 17 00:00:00 2001 From: YuviPanda Date: Sat, 8 May 2021 02:00:01 +0530 Subject: [PATCH 12/12] Specify username in scratch bucket --- hub-templates/daskhub/values.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hub-templates/daskhub/values.yaml b/hub-templates/daskhub/values.yaml index c04a825409..cef9db1a81 100644 --- a/hub-templates/daskhub/values.yaml +++ b/hub-templates/daskhub/values.yaml @@ -110,8 +110,9 @@ basehub: 'SCRATCH_BUCKET_PROTOCOL': bucket_protocol, # Matches "daskhub.scratchBUcket.name" helm template 'SCRATCH_BUCKET_NAME': bucket_name, - 'SCRATCH_BUCKET': f'{bucket_protocol}://{bucket_name}', - 'PANGEO_SCRATCH': f'{bucket_protocol}://{bucket_name}', + # Use k8s syntax of $(ENV_VAR) to substitute env vars dynamically in other env vars + 'SCRATCH_BUCKET': f'{bucket_protocol}://{bucket_name}/$(JUPYTERHUB_USER)', + 'PANGEO_SCRATCH': f'{bucket_protocol}://{bucket_name}/$(JUPYTERHUB_USER)', } c.KubeSpawner.environment.update(env)