Merge pull request #605 from yuvipanda/spot-cb

Use spot instances for dask workers
2i2c-org · Aug 15, 2021 · 8aa71ba · 8aa71ba
2 parents 808b091 + facddc6
commit 8aa71ba
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 12 deletions.
diff --git a/eksctl/carbonplan.jsonnet b/eksctl/carbonplan.jsonnet
@@ -7,10 +7,11 @@ local clusterRegion = "us-west-2";
 local masterAzs = ["us-west-2a", "us-west-2b", "us-west-2c"];
 local nodeAz = "us-west-2a";
 
-// Node definitions for use with dask and notebook nodes
-// These are merged in with the defaults for either node type,
-// and so can contain any overrides.
-local nodes = [
+// Node definitions for notebook nodes. Config here is merged
+// with our notebook node definition.
+// A `node.kubernetes.io/instance-type label is added, so pods
+// can request a particular kind of node with a nodeSelector
+local notebookNodes = [
     { instanceType: "r5.large" },
     { instanceType: "r5.xlarge" },
     { instanceType: "r5.2xlarge" },
@@ -19,6 +20,19 @@ local nodes = [
     { instanceType: "x1.32xlarge" }
 ];
 
+// Node definitions for dask worker nodes. Config here is merged
+// with our dask worker node definition, which uses spot instances.
+// A `node.kubernetes.io/instance-type label is set to the name of the
+// *first* item in instanceDistribution.instanceTypes, to match
+// what we do with notebook nodes. Pods can request a particular
+// kind of node with a nodeSelector
+local daskNodes = [
+    { instancesDistribution+: { instanceTypes: ["r5.large"] }},
+    { instancesDistribution+: { instanceTypes: ["r5.xlarge"] }},
+    { instancesDistribution+: { instanceTypes: ["r5.2xlarge"] }},
+    { instancesDistribution+: { instanceTypes: ["r5.8xlarge"] }},
+];
+
 cluster {
     metadata+: {
         name: "carbonplanhub",
@@ -44,10 +58,11 @@ cluster {
         ng {
             // NodeGroup names can't have a '.' in them, while
             // instanceTypes always have a .
-            name: "nb-%s" % std.strReplace(self.instanceType, ".", "-"),
+            name: "nb-%s" % std.strReplace(n.instanceType, ".", "-"),
             availabilityZones: [nodeAz],
             minSize: 0,
             maxSize: 500,
+            instanceType: n.instanceType,
             ssh: {
                 publicKeyPath: 'ssh-keys/carbonplan.key.pub'
             },
@@ -60,12 +75,12 @@ cluster {
                 "hub.jupyter.org/dedicated": "user:NoSchedule"
             },
 
-        } + n for n in nodes
+        } + n for n in notebookNodes
     ] + [
         ng {
             // NodeGroup names can't have a '.' in them, while
             // instanceTypes always have a .
-            name: "dask-%s" % std.strReplace(self.instanceType, ".", "-"),
+            name: "dask-%s" % std.strReplace(n.instancesDistribution.instanceTypes[0], ".", "-"),
             availabilityZones: [nodeAz],
             minSize: 0,
             maxSize: 500,
@@ -79,8 +94,12 @@ cluster {
                 "k8s.dask.org_dedicated" : "worker:NoSchedule",
                 "k8s.dask.org/dedicated" : "worker:NoSchedule"
             },
-
-        } + n for n in nodes
+            instancesDistribution+: {
+                onDemandBaseCapacity: 0,
+                onDemandPercentageAboveBaseCapacity: 0,
+                spotAllocationStrategy: "capacity-optimized",
+            },
+        } + n for n in daskNodes
     ]
 
 

diff --git a/eksctl/libsonnet/nodegroup.jsonnet b/eksctl/libsonnet/nodegroup.jsonnet
@@ -15,12 +15,12 @@ local makeCloudTaints(taints) = {
   availabilityZones: [],
   minSize: 0,
   desiredCapacity: self.minSize,
-  instanceType: '',
   volumeSize: 80,
   labels+: {
 	  // Add instance type as label to nodegroups, so they
-	  // can be picked up by the autoscaler
-    'node.kubernetes.io/instance-type': $.instanceType,
+    // can be picked up by the autoscaler. If using spot instances,
+    // pick the first instancetype
+    'node.kubernetes.io/instance-type': if std.objectHas($, 'instanceType') then $.instanceType else $.instancesDistribution.instanceTypes[0],
   },
   taints+: {},
   tags: makeCloudLabels(self.labels) + makeCloudTaints(self.taints),