Merge pull request #651 from yuvipanda/google-filestore

Use Google Filestore for home directories

config/hubs/pangeo-hubs.cluster.yaml

@@ -25,9 +25,7 @@ support:
         admissionWebhooks:
           enabled: false
     nfs-server-provisioner:
-      enabled: true
-      persistence:
-        size: 500Gi
+      enabled: false
 hubs:
   - name: staging
     domain: staging.pangeo.2i2c.cloud
@@ -36,11 +34,16 @@ hubs:
       connection: github
     config: &stagingConfig
       basehub:
-        inClusterNFS:
-          enabled: true
-          size: 10Gi
         nfsPVC:
-          enabled: false
+          enabled: true
+          nfs:
+            mountOptions:
+            - soft
+            - noatime
+            # Google FileStore IP
+            serverIP: 10.229.44.234
+            # Name of Google Filestore share
+            baseShareName: /homes/
         jupyterhub:
           proxy:
             https:
@@ -68,11 +71,56 @@ hubs:
                   - sgibson91
                   - yuvipanda
                   - damianavila
+                  - choldgraf
+                  - rabernat
                 admin_users: *staging_users
           singleuser:
+            profileList:
+              # The mem-guarantees are here so k8s doesn't schedule other pods
+              # on these nodes. They need to be just under total allocatable
+              # RAM on a node, not total node capacity
+              - display_name: "Small"
+                description: "~2 CPU, ~8G RAM"
+                kubespawner_override:
+                  mem_limit: 8G
+                  mem_guarantee: 5.5G
+                  node_selector:
+                    node.kubernetes.io/instance-type: n1-standard-2
+              - display_name: "Medium"
+                description: "~8 CPU, ~32G RAM"
+                kubespawner_override:
+                  mem_limit: 32G
+                  mem_guarantee: 25G
+                  node_selector:
+                    node.kubernetes.io/instance-type: n1-standard-8
+              - display_name: "Large"
+                description: "~16 CPU, ~64G RAM"
+                kubespawner_override:
+                  mem_limit: 64G
+                  mem_guarantee: 55G
+                  node_selector:
+                    node.kubernetes.io/instance-type: n1-standard-16
+              - display_name: "Very Large"
+                description: "~32 CPU, ~128G RAM"
+                kubespawner_override:
+                  mem_limit: 128G
+                  mem_guarantee: 115G
+                  node_selector:
+                    node.kubernetes.io/instance-type: n1-standard-32
+            initContainers:
+              # Need to explicitly fix ownership here, since EFS doesn't do anonuid
+            - name: volume-mount-ownership-fix
+              image: busybox
+              command: ["sh", "-c", "id && chown 1000:1000 /home/jovyan && ls -lhd /home/jovyan"]
+              securityContext:
+                runAsUser: 0
+              volumeMounts:
+              - name: home
+                mountPath: /home/jovyan
+                subPath: "{username}"
             image:
               name: pangeo/pangeo-notebook
-              tag: e60dfe1
+              tag: bcfacc5
             cpu:
               limit: 2
               guarantee: 1

terraform/gcp/projects/pangeo-hubs.tfvars

@@ -10,24 +10,36 @@ enable_network_policy    = true
 # Some hubs want a storage bucket, so we need to have config connector enabled
 config_connector_enabled = true
 
+
 notebook_nodes = {
-  "user" : {
+  "small" : {
     min : 0,
-    max : 20,
-    machine_type : "n1-highmem-4"
-    labels : {}
+    max : 100,
+    machine_type : "n1-standard-2",
+    labels: {}
   },
-}
-
-dask_nodes = {
-  "worker" : {
+  "medium" : {
     min : 0,
     max : 100,
-    machine_type : "n1-highmem-4"
-    labels : {}
+    machine_type : "n1-standard-8",
+    labels: {}
+  },
+  "large" : {
+    min : 0,
+    max : 100,
+    machine_type : "n1-standard-16",
+    labels: {}
+  },
+  "very-large" : {
+    min : 0,
+    max : 100,
+    machine_type : "n1-standard-32",
+    labels: {}
   },
 }
 
 user_buckets = [
   "pangeo-scratch"
 ]
+
+enable_filestore = true

terraform/gcp/storage.tf

@@ -0,0 +1,19 @@
+resource "google_filestore_instance" "homedirs" {
+
+  name = "${var.prefix}-homedirs"
+  zone = var.zone
+  tier = var.filestore_tier
+  project = var.project_id
+
+  count = var.enable_filestore ? 1 : 0
+
+  file_shares {
+    capacity_gb = var.filestore_capacity_gb
+    name        = "homes"
+  }
+
+  networks {
+    network    = var.enable_private_cluster ? data.google_compute_network.default_network.name : null
+    modes   = ["MODE_IPV4"]
+  }
+}

terraform/gcp/variables.tf

@@ -183,7 +183,40 @@ variable "enable_private_cluster" {
   and not have public IPs. A cloud NAT will provide outbound internet access from
   these nodes. The kubernetes API will still be exposed publicly, so we can access
   it from our laptops & CD.
-  
+
   This is often required by institutional controls banning VMs from having public IPs.
   EOT
 }
+
+variable "enable_filestore" {
+  type = bool
+  default = false
+  description = <<-EOT
+  Deploy a Google FileStore for home directories
+
+  This provisions a managed NFS solution that can be mounted as
+  home directories for users. If this is not enabled, a manual or
+  in-cluster NFS solution must be set up
+  EOT
+}
+
+variable "filestore_capacity_gb" {
+  type = number
+  default = 1024
+  description = <<-EOT
+  Minimum size (in GB) of Google FileStore.
+
+  Minimum is 1024 for BASIC_HDD tier, and 2560 for BASIC_SSD tier.
+  EOT
+}
+
+variable "filestore_tier" {
+  type = string
+  default = "BASIC_HDD"
+  description = <<-EOT
+  Google FileStore service tier to use.
+
+  Most likely BASIC_HDD (for slower home directories, min $204 / month) or
+  BASIC_SSD (for faster home directories, min $768 / month)
+  EOT
+}