Skip to content

adds volume configuration options #982

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 20 commits into from
Apr 28, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 21 additions & 2 deletions cli/cmd/lib_cluster_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,16 @@ func getClusterUpdateConfig(cachedClusterConfig clusterconfig.Config, awsCreds A
}
userClusterConfig.InstanceVolumeSize = cachedClusterConfig.InstanceVolumeSize

if userClusterConfig.InstanceVolumeType != cachedClusterConfig.InstanceVolumeType {
return nil, clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.InstanceVolumeTypeKey, cachedClusterConfig.InstanceVolumeType)
}
userClusterConfig.InstanceVolumeType = cachedClusterConfig.InstanceVolumeType

if userClusterConfig.InstanceVolumeIOPS != cachedClusterConfig.InstanceVolumeIOPS {
return nil, clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.InstanceVolumeIOPSKey, cachedClusterConfig.InstanceVolumeIOPS)
}
userClusterConfig.InstanceVolumeIOPS = cachedClusterConfig.InstanceVolumeIOPS

if userClusterConfig.Spot != nil && *userClusterConfig.Spot != *cachedClusterConfig.Spot {
return nil, clusterconfig.ErrorConfigCannotBeChangedOnUpdate(clusterconfig.SpotKey, *cachedClusterConfig.Spot)
}
Expand Down Expand Up @@ -310,10 +320,13 @@ func getClusterUpdateConfig(cachedClusterConfig clusterconfig.Config, awsCreds A
func confirmInstallClusterConfig(clusterConfig *clusterconfig.Config, awsCreds AWSCredentials, awsClient *aws.Client, envName string, disallowPrompt bool) {
eksPrice := aws.EKSPrices[*clusterConfig.Region]
operatorInstancePrice := aws.InstanceMetadatas[*clusterConfig.Region]["t3.medium"].Price
operatorEBSPrice := aws.EBSMetadatas[*clusterConfig.Region].Price * 20 / 30 / 24
operatorEBSPrice := aws.EBSMetadatas[*clusterConfig.Region]["gp2"].PriceGB * 20 / 30 / 24
elbPrice := aws.ELBMetadatas[*clusterConfig.Region].Price
apiInstancePrice := aws.InstanceMetadatas[*clusterConfig.Region][*clusterConfig.InstanceType].Price
apiEBSPrice := aws.EBSMetadatas[*clusterConfig.Region].Price * float64(clusterConfig.InstanceVolumeSize) / 30 / 24
apiEBSPrice := aws.EBSMetadatas[*clusterConfig.Region][clusterConfig.InstanceVolumeType.String()].PriceGB * float64(clusterConfig.InstanceVolumeSize) / 30 / 24
if clusterConfig.InstanceVolumeType.String() == "io1" && clusterConfig.InstanceVolumeIOPS != nil {
apiEBSPrice += aws.EBSMetadatas[*clusterConfig.Region][clusterConfig.InstanceVolumeType.String()].PriceIOPS * float64(*clusterConfig.InstanceVolumeIOPS) / 30 / 24
}
fixedPrice := eksPrice + operatorInstancePrice + operatorEBSPrice + 2*elbPrice
totalMinPrice := fixedPrice + float64(*clusterConfig.MinInstances)*(apiInstancePrice+apiEBSPrice)
totalMaxPrice := fixedPrice + float64(*clusterConfig.MaxInstances)*(apiInstancePrice+apiEBSPrice)
Expand Down Expand Up @@ -431,6 +444,12 @@ func clusterConfigConfirmaionStr(clusterConfig clusterconfig.Config, awsCreds AW
if clusterConfig.InstanceVolumeSize != defaultConfig.InstanceVolumeSize {
items.Add(clusterconfig.InstanceVolumeSizeUserKey, clusterConfig.InstanceVolumeSize)
}
if clusterConfig.InstanceVolumeType != defaultConfig.InstanceVolumeType {
items.Add(clusterconfig.InstanceVolumeTypeUserKey, clusterConfig.InstanceVolumeType)
}
if clusterConfig.InstanceVolumeIOPS != nil {
items.Add(clusterconfig.InstanceVolumeIOPSUserKey, *clusterConfig.InstanceVolumeIOPS)
}

if clusterConfig.Spot != nil && *clusterConfig.Spot != *defaultConfig.Spot {
items.Add(clusterconfig.SpotUserKey, s.YesNo(clusterConfig.Spot != nil && *clusterConfig.Spot))
Expand Down
6 changes: 6 additions & 0 deletions docs/cluster-management/config.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,12 @@ max_instances: 5
# instance volume size (GB) (default: 50)
instance_volume_size: 50

# instance volume type [gp2, io1, st1, sc1] (default: gp2)
instance_volume_type: gp2

# instance volume iops (only applicable to io1 storage type) (default: 3000)
# instance_volume_iops: 3000

# CloudWatch log group for cortex (default: <cluster_name>)
log_group: cortex

Expand Down
10 changes: 5 additions & 5 deletions manager/generate_eks.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,19 +60,19 @@ def apply_worker_settings(nodegroup):


def apply_clusterconfig(nodegroup, config):
if config["min_instances"] == 0:
desired_capacity = 1
else:
desired_capacity = config["min_instances"]

clusterconfig_settings = {
"instanceType": config["instance_type"],
"availabilityZones": config["availability_zones"],
"volumeSize": config["instance_volume_size"],
"minSize": config["min_instances"],
"maxSize": config["max_instances"],
"desiredCapacity": desired_capacity,
"volumeType": config["instance_volume_type"],
"desiredCapacity": 1 if config["min_instances"] == 0 else config["min_instances"],
}
# add iops to settings if volume_type is io1
if config["instance_volume_type"] == "io1":
clusterconfig_settings["volumeIOPS"] = config["instance_volume_iops"]

return merge_override(nodegroup, clusterconfig_settings)

Expand Down
86 changes: 75 additions & 11 deletions pkg/lib/aws/gen_resource_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@
def get_instance_metadatas(pricing):
instance_mapping = {}

for product_id, product in pricing["products"].items():
for _, product in pricing["products"].items():
if product.get("attributes") is None:
continue
if product["attributes"].get("servicecode") != "AmazonEC2":
Expand Down Expand Up @@ -88,7 +88,7 @@ def get_instance_metadatas(pricing):


def get_elb_metadata(pricing):
for product_id, product in pricing["products"].items():
for _, product in pricing["products"].items():
if product.get("attributes") is None:
continue
if product.get("productFamily") != "Load Balancer":
Expand All @@ -106,7 +106,7 @@ def get_elb_metadata(pricing):


def get_nat_metadata(pricing):
for product_id, product in pricing["products"].items():
for _, product in pricing["products"].items():
if product.get("attributes") is None:
continue
if product.get("productFamily") != "NAT Gateway":
Expand All @@ -126,26 +126,66 @@ def get_nat_metadata(pricing):


def get_ebs_metadata(pricing):
for product_id, product in pricing["products"].items():
storage_mapping = {}

for _, product in pricing["products"].items():
if product.get("attributes") is None:
continue
if product.get("productFamily") != "Storage":
continue
if product["attributes"].get("volumeApiName") != "gp2":
# ignore legacy standard storage
if product["attributes"].get("volumeApiName") == "standard":
continue

price_dimensions = list(pricing["terms"]["OnDemand"][product["sku"]].values())[0][
"priceDimensions"
]
price = list(price_dimensions.values())[0]["pricePerUnit"]["USD"]
return {"price": float(price)}

metadata = {
"type": product["attributes"].get("volumeApiName"),
"price_gb": float(price),
}

# io1 has per IOPS pricing --> add pricing to metadata
# if storagedevice does not price per IOPS will set value to 0
if product["attributes"].get("volumeApiName") == "io1":
# go through pricing data until found data about IOPS pricing
for _, product_iops in pricing["products"].items():
if product_iops.get("attributes") is None:
continue
if product_iops.get("productFamily") != "System Operation":
continue
if product_iops["attributes"].get("volumeApiName") != "io1":
continue
if product_iops["attributes"].get("group") != "EBS IOPS":
continue
if product_iops["attributes"].get("provisioned") != "Yes":
continue

price_dimensions = list(pricing["terms"]["OnDemand"][product_iops["sku"]].values())[
0
]["priceDimensions"]
price = list(price_dimensions.values())[0]["pricePerUnit"]["USD"]

metadata["price_iops"] = price
metadata["iops_configurable"] = "true"

# set default values for all other storage types
else:
metadata["price_iops"] = 0
metadata["iops_configurable"] = "false"

storage_mapping[product["attributes"]["volumeApiName"]] = metadata

return storage_mapping


def get_eks_price(region):
response = requests.get(EKS_PRICING_ENDPOINT_TEMPLATE.format(region))
pricing = response.json()

for product_id, product in pricing["products"].items():
for _, product in pricing["products"].items():
if product.get("attributes") is None:
continue
if product.get("productFamily") != "Compute":
Expand Down Expand Up @@ -210,7 +250,10 @@ def get_eks_price(region):

type EBSMetadata struct {
Region string `json:"region"`
Price float64 `json:"price"`
PriceGB float64 `json:"price_gb"`
PriceIOPS float64 `json:"price_iops"`
IOPSConfigurable bool `json:"iops_configurable"`
Type string `json:"type"`
}

// region -> instance type -> instance metadata
Expand All @@ -229,7 +272,7 @@ def get_eks_price(region):
}

// region -> EBS metadata
var EBSMetadatas = map[string]EBSMetadata{
var EBSMetadatas = map[string]map[string]EBSMetadata{
${ebs_region_map}
}

Expand Down Expand Up @@ -263,7 +306,14 @@ def get_eks_price(region):
)

ebs_region_map_template = Template(
""""${region}": {Region: "${region}", Price: ${price}},
""""${region}": map[string]EBSMetadata{
${ebs_metadata}
},
"""
)

ebs_type_map_template = Template(
""""${type}": {Region: "${region}",Type: "${type}", PriceGB: ${price_gb}, PriceIOPS: ${price_iops}, IOPSConfigurable: ${iops_configurable}},
"""
)

Expand Down Expand Up @@ -307,6 +357,20 @@ def main():
}
)

ebs_metadatas_str = ""

for ebs_type in sorted(ebs_metadata.keys()):
metadata = ebs_metadata[ebs_type]
ebs_metadatas_str += ebs_type_map_template.substitute(
{
"region": region,
"type": ebs_type,
"price_gb": metadata["price_gb"],
"price_iops": metadata["price_iops"],
"iops_configurable": metadata["iops_configurable"],
}
)

instance_region_map_str += instance_region_map_template.substitute(
{"region": region, "instance_metadatas": instance_metadatas_str}
)
Expand All @@ -317,7 +381,7 @@ def main():
{"region": region, "price": nat_metadata["price"]}
)
ebs_region_map_str += ebs_region_map_template.substitute(
{"region": region, "price": ebs_metadata["price"]}
{"region": region, "ebs_metadata": ebs_metadatas_str}
)
eks_region_map_str += eks_region_map_template.substitute(
{"region": region, "price": eks_price}
Expand Down
128 changes: 108 additions & 20 deletions pkg/lib/aws/resource_metadata.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,11 @@ type NATMetadata struct {
}

type EBSMetadata struct {
Region string `json:"region"`
Price float64 `json:"price"`
Region string `json:"region"`
PriceGB float64 `json:"price_gb"`
PriceIOPS float64 `json:"price_iops"`
IOPSConfigurable bool `json:"iops_configurable"`
Type string `json:"type"`
}

// region -> instance type -> instance metadata
Expand Down Expand Up @@ -3355,24 +3358,109 @@ var NATMetadatas = map[string]NATMetadata{
}

// region -> EBS metadata
var EBSMetadatas = map[string]EBSMetadata{
"ap-east-1": {Region: "ap-east-1", Price: 0.132},
"ap-northeast-1": {Region: "ap-northeast-1", Price: 0.12},
"ap-northeast-2": {Region: "ap-northeast-2", Price: 0.114},
"ap-south-1": {Region: "ap-south-1", Price: 0.114},
"ap-southeast-1": {Region: "ap-southeast-1", Price: 0.12},
"ap-southeast-2": {Region: "ap-southeast-2", Price: 0.12},
"ca-central-1": {Region: "ca-central-1", Price: 0.11},
"eu-central-1": {Region: "eu-central-1", Price: 0.119},
"eu-north-1": {Region: "eu-north-1", Price: 0.1045},
"eu-west-1": {Region: "eu-west-1", Price: 0.11},
"eu-west-2": {Region: "eu-west-2", Price: 0.116},
"eu-west-3": {Region: "eu-west-3", Price: 0.116},
"me-south-1": {Region: "me-south-1", Price: 0.121},
"sa-east-1": {Region: "sa-east-1", Price: 0.19},
"us-east-1": {Region: "us-east-1", Price: 0.1},
"us-east-2": {Region: "us-east-2", Price: 0.1},
"us-west-2": {Region: "us-west-2", Price: 0.1},
var EBSMetadatas = map[string]map[string]EBSMetadata{
"ap-east-1": {
"gp2": {Region: "ap-east-1", Type: "gp2", PriceGB: 0.132, PriceIOPS: 0, IOPSConfigurable: false},
"io1": {Region: "ap-east-1", Type: "io1", PriceGB: 0.1518, PriceIOPS: 0.0792000000, IOPSConfigurable: true},
"sc1": {Region: "ap-east-1", Type: "sc1", PriceGB: 0.033, PriceIOPS: 0, IOPSConfigurable: false},
"st1": {Region: "ap-east-1", Type: "st1", PriceGB: 0.0594, PriceIOPS: 0, IOPSConfigurable: false},
},
"ap-northeast-1": {
"gp2": {Region: "ap-northeast-1", Type: "gp2", PriceGB: 0.12, PriceIOPS: 0, IOPSConfigurable: false},
"io1": {Region: "ap-northeast-1", Type: "io1", PriceGB: 0.142, PriceIOPS: 0.0740000000, IOPSConfigurable: true},
"sc1": {Region: "ap-northeast-1", Type: "sc1", PriceGB: 0.03, PriceIOPS: 0, IOPSConfigurable: false},
"st1": {Region: "ap-northeast-1", Type: "st1", PriceGB: 0.054, PriceIOPS: 0, IOPSConfigurable: false},
},
"ap-northeast-2": {
"gp2": {Region: "ap-northeast-2", Type: "gp2", PriceGB: 0.114, PriceIOPS: 0, IOPSConfigurable: false},
"io1": {Region: "ap-northeast-2", Type: "io1", PriceGB: 0.1278, PriceIOPS: 0.0666000000, IOPSConfigurable: true},
"sc1": {Region: "ap-northeast-2", Type: "sc1", PriceGB: 0.029, PriceIOPS: 0, IOPSConfigurable: false},
"st1": {Region: "ap-northeast-2", Type: "st1", PriceGB: 0.051, PriceIOPS: 0, IOPSConfigurable: false},
},
"ap-south-1": {
"gp2": {Region: "ap-south-1", Type: "gp2", PriceGB: 0.114, PriceIOPS: 0, IOPSConfigurable: false},
"io1": {Region: "ap-south-1", Type: "io1", PriceGB: 0.131, PriceIOPS: 0.0680000000, IOPSConfigurable: true},
"sc1": {Region: "ap-south-1", Type: "sc1", PriceGB: 0.029, PriceIOPS: 0, IOPSConfigurable: false},
"st1": {Region: "ap-south-1", Type: "st1", PriceGB: 0.051, PriceIOPS: 0, IOPSConfigurable: false},
},
"ap-southeast-1": {
"gp2": {Region: "ap-southeast-1", Type: "gp2", PriceGB: 0.12, PriceIOPS: 0, IOPSConfigurable: false},
"io1": {Region: "ap-southeast-1", Type: "io1", PriceGB: 0.138, PriceIOPS: 0.0720000000, IOPSConfigurable: true},
"sc1": {Region: "ap-southeast-1", Type: "sc1", PriceGB: 0.03, PriceIOPS: 0, IOPSConfigurable: false},
"st1": {Region: "ap-southeast-1", Type: "st1", PriceGB: 0.054, PriceIOPS: 0, IOPSConfigurable: false},
},
"ap-southeast-2": {
"gp2": {Region: "ap-southeast-2", Type: "gp2", PriceGB: 0.12, PriceIOPS: 0, IOPSConfigurable: false},
"io1": {Region: "ap-southeast-2", Type: "io1", PriceGB: 0.138, PriceIOPS: 0.0720000000, IOPSConfigurable: true},
"sc1": {Region: "ap-southeast-2", Type: "sc1", PriceGB: 0.03, PriceIOPS: 0, IOPSConfigurable: false},
"st1": {Region: "ap-southeast-2", Type: "st1", PriceGB: 0.054, PriceIOPS: 0, IOPSConfigurable: false},
},
"ca-central-1": {
"gp2": {Region: "ca-central-1", Type: "gp2", PriceGB: 0.11, PriceIOPS: 0, IOPSConfigurable: false},
"io1": {Region: "ca-central-1", Type: "io1", PriceGB: 0.138, PriceIOPS: 0.0720000000, IOPSConfigurable: true},
"sc1": {Region: "ca-central-1", Type: "sc1", PriceGB: 0.028, PriceIOPS: 0, IOPSConfigurable: false},
"st1": {Region: "ca-central-1", Type: "st1", PriceGB: 0.05, PriceIOPS: 0, IOPSConfigurable: false},
},
"eu-central-1": {
"gp2": {Region: "eu-central-1", Type: "gp2", PriceGB: 0.119, PriceIOPS: 0, IOPSConfigurable: false},
"io1": {Region: "eu-central-1", Type: "io1", PriceGB: 0.149, PriceIOPS: 0.0780000000, IOPSConfigurable: true},
"sc1": {Region: "eu-central-1", Type: "sc1", PriceGB: 0.03, PriceIOPS: 0, IOPSConfigurable: false},
"st1": {Region: "eu-central-1", Type: "st1", PriceGB: 0.054, PriceIOPS: 0, IOPSConfigurable: false},
},
"eu-north-1": {
"gp2": {Region: "eu-north-1", Type: "gp2", PriceGB: 0.1045, PriceIOPS: 0, IOPSConfigurable: false},
"io1": {Region: "eu-north-1", Type: "io1", PriceGB: 0.1311, PriceIOPS: 0.0684000000, IOPSConfigurable: true},
"sc1": {Region: "eu-north-1", Type: "sc1", PriceGB: 0.0266, PriceIOPS: 0, IOPSConfigurable: false},
"st1": {Region: "eu-north-1", Type: "st1", PriceGB: 0.0475, PriceIOPS: 0, IOPSConfigurable: false},
},
"eu-west-1": {
"gp2": {Region: "eu-west-1", Type: "gp2", PriceGB: 0.11, PriceIOPS: 0, IOPSConfigurable: false},
"io1": {Region: "eu-west-1", Type: "io1", PriceGB: 0.138, PriceIOPS: 0.0720000000, IOPSConfigurable: true},
"sc1": {Region: "eu-west-1", Type: "sc1", PriceGB: 0.028, PriceIOPS: 0, IOPSConfigurable: false},
"st1": {Region: "eu-west-1", Type: "st1", PriceGB: 0.05, PriceIOPS: 0, IOPSConfigurable: false},
},
"eu-west-2": {
"gp2": {Region: "eu-west-2", Type: "gp2", PriceGB: 0.116, PriceIOPS: 0, IOPSConfigurable: false},
"io1": {Region: "eu-west-2", Type: "io1", PriceGB: 0.145, PriceIOPS: 0.0760000000, IOPSConfigurable: true},
"sc1": {Region: "eu-west-2", Type: "sc1", PriceGB: 0.029, PriceIOPS: 0, IOPSConfigurable: false},
"st1": {Region: "eu-west-2", Type: "st1", PriceGB: 0.053, PriceIOPS: 0, IOPSConfigurable: false},
},
"eu-west-3": {
"gp2": {Region: "eu-west-3", Type: "gp2", PriceGB: 0.116, PriceIOPS: 0, IOPSConfigurable: false},
"io1": {Region: "eu-west-3", Type: "io1", PriceGB: 0.145, PriceIOPS: 0.0760000000, IOPSConfigurable: true},
"sc1": {Region: "eu-west-3", Type: "sc1", PriceGB: 0.029, PriceIOPS: 0, IOPSConfigurable: false},
"st1": {Region: "eu-west-3", Type: "st1", PriceGB: 0.053, PriceIOPS: 0, IOPSConfigurable: false},
},
"me-south-1": {
"gp2": {Region: "me-south-1", Type: "gp2", PriceGB: 0.121, PriceIOPS: 0, IOPSConfigurable: false},
"io1": {Region: "me-south-1", Type: "io1", PriceGB: 0.1518, PriceIOPS: 0.0792000000, IOPSConfigurable: true},
"sc1": {Region: "me-south-1", Type: "sc1", PriceGB: 0.0308, PriceIOPS: 0, IOPSConfigurable: false},
"st1": {Region: "me-south-1", Type: "st1", PriceGB: 0.055, PriceIOPS: 0, IOPSConfigurable: false},
},
"sa-east-1": {
"gp2": {Region: "sa-east-1", Type: "gp2", PriceGB: 0.19, PriceIOPS: 0, IOPSConfigurable: false},
"io1": {Region: "sa-east-1", Type: "io1", PriceGB: 0.238, PriceIOPS: 0.0910000000, IOPSConfigurable: true},
"sc1": {Region: "sa-east-1", Type: "sc1", PriceGB: 0.048, PriceIOPS: 0, IOPSConfigurable: false},
"st1": {Region: "sa-east-1", Type: "st1", PriceGB: 0.086, PriceIOPS: 0, IOPSConfigurable: false},
},
"us-east-1": {
"gp2": {Region: "us-east-1", Type: "gp2", PriceGB: 0.1, PriceIOPS: 0, IOPSConfigurable: false},
"io1": {Region: "us-east-1", Type: "io1", PriceGB: 0.125, PriceIOPS: 0.0650000000, IOPSConfigurable: true},
"sc1": {Region: "us-east-1", Type: "sc1", PriceGB: 0.025, PriceIOPS: 0, IOPSConfigurable: false},
"st1": {Region: "us-east-1", Type: "st1", PriceGB: 0.045, PriceIOPS: 0, IOPSConfigurable: false},
},
"us-east-2": {
"gp2": {Region: "us-east-2", Type: "gp2", PriceGB: 0.1, PriceIOPS: 0, IOPSConfigurable: false},
"io1": {Region: "us-east-2", Type: "io1", PriceGB: 0.125, PriceIOPS: 0.0650000000, IOPSConfigurable: true},
"sc1": {Region: "us-east-2", Type: "sc1", PriceGB: 0.025, PriceIOPS: 0, IOPSConfigurable: false},
"st1": {Region: "us-east-2", Type: "st1", PriceGB: 0.045, PriceIOPS: 0, IOPSConfigurable: false},
},
"us-west-2": {
"gp2": {Region: "us-west-2", Type: "gp2", PriceGB: 0.1, PriceIOPS: 0, IOPSConfigurable: false},
"io1": {Region: "us-west-2", Type: "io1", PriceGB: 0.125, PriceIOPS: 0.0650000000, IOPSConfigurable: true},
"sc1": {Region: "us-west-2", Type: "sc1", PriceGB: 0.025, PriceIOPS: 0, IOPSConfigurable: false},
"st1": {Region: "us-west-2", Type: "st1", PriceGB: 0.045, PriceIOPS: 0, IOPSConfigurable: false},
},
}

// region -> EKS price
Expand Down
Loading