diff --git a/eksctl/carbonplan.jsonnet b/eksctl/carbonplan.jsonnet index 681a56bab..a7c7c0243 100644 --- a/eksctl/carbonplan.jsonnet +++ b/eksctl/carbonplan.jsonnet @@ -7,10 +7,11 @@ local clusterRegion = "us-west-2"; local masterAzs = ["us-west-2a", "us-west-2b", "us-west-2c"]; local nodeAz = "us-west-2a"; -// Node definitions for use with dask and notebook nodes -// These are merged in with the defaults for either node type, -// and so can contain any overrides. -local nodes = [ +// Node definitions for notebook nodes. Config here is merged +// with our notebook node definition. +// A `node.kubernetes.io/instance-type label is added, so pods +// can request a particular kind of node with a nodeSelector +local notebookNodes = [ { instanceType: "r5.large" }, { instanceType: "r5.xlarge" }, { instanceType: "r5.2xlarge" }, @@ -19,6 +20,19 @@ local nodes = [ { instanceType: "x1.32xlarge" } ]; +// Node definitions for dask worker nodes. Config here is merged +// with our dask worker node definition, which uses spot instances. +// A `node.kubernetes.io/instance-type label is set to the name of the +// *first* item in instanceDistribution.instanceTypes, to match +// what we do with notebook nodes. Pods can request a particular +// kind of node with a nodeSelector +local daskNodes = [ + { instancesDistribution+: { instanceTypes: ["r5.large"] }}, + { instancesDistribution+: { instanceTypes: ["r5.xlarge"] }}, + { instancesDistribution+: { instanceTypes: ["r5.2xlarge"] }}, + { instancesDistribution+: { instanceTypes: ["r5.8xlarge"] }}, +]; + cluster { metadata+: { name: "carbonplanhub", @@ -44,10 +58,11 @@ cluster { ng { // NodeGroup names can't have a '.' in them, while // instanceTypes always have a . - name: "nb-%s" % std.strReplace(self.instanceType, ".", "-"), + name: "nb-%s" % std.strReplace(n.instanceType, ".", "-"), availabilityZones: [nodeAz], minSize: 0, maxSize: 500, + instanceType: n.instanceType, ssh: { publicKeyPath: 'ssh-keys/carbonplan.key.pub' }, @@ -60,12 +75,12 @@ cluster { "hub.jupyter.org/dedicated": "user:NoSchedule" }, - } + n for n in nodes + } + n for n in notebookNodes ] + [ ng { // NodeGroup names can't have a '.' in them, while // instanceTypes always have a . - name: "dask-%s" % std.strReplace(self.instanceType, ".", "-"), + name: "dask-%s" % std.strReplace(n.instancesDistribution.instanceTypes[0], ".", "-"), availabilityZones: [nodeAz], minSize: 0, maxSize: 500, @@ -79,8 +94,12 @@ cluster { "k8s.dask.org_dedicated" : "worker:NoSchedule", "k8s.dask.org/dedicated" : "worker:NoSchedule" }, - - } + n for n in nodes + instancesDistribution+: { + onDemandBaseCapacity: 0, + onDemandPercentageAboveBaseCapacity: 0, + spotAllocationStrategy: "capacity-optimized", + }, + } + n for n in daskNodes ] diff --git a/eksctl/libsonnet/nodegroup.jsonnet b/eksctl/libsonnet/nodegroup.jsonnet index 1318e7613..e54ff39e4 100644 --- a/eksctl/libsonnet/nodegroup.jsonnet +++ b/eksctl/libsonnet/nodegroup.jsonnet @@ -15,12 +15,12 @@ local makeCloudTaints(taints) = { availabilityZones: [], minSize: 0, desiredCapacity: self.minSize, - instanceType: '', volumeSize: 80, labels+: { // Add instance type as label to nodegroups, so they - // can be picked up by the autoscaler - 'node.kubernetes.io/instance-type': $.instanceType, + // can be picked up by the autoscaler. If using spot instances, + // pick the first instancetype + 'node.kubernetes.io/instance-type': if std.objectHas($, 'instanceType') then $.instanceType else $.instancesDistribution.instanceTypes[0], }, taints+: {}, tags: makeCloudLabels(self.labels) + makeCloudTaints(self.taints),