diff --git a/cli/cmd/lib_cluster_config.go b/cli/cmd/lib_cluster_config.go
index 97a9b86058..edd6cd7732 100644
--- a/cli/cmd/lib_cluster_config.go
+++ b/cli/cmd/lib_cluster_config.go
@@ -281,6 +281,10 @@ func confirmInstallClusterConfig(clusterConfig *clusterconfig.Config, awsClient
func confirmConfigureClusterConfig(configureChanges clusterconfig.ConfigureChanges, oldCc, newCc clusterconfig.Config, disallowPrompt bool) {
fmt.Printf("your %s cluster in region %s will be updated as follows:\n\n", newCc.ClusterName, newCc.Region)
+ for _, fieldToUpdate := range configureChanges.FieldsToUpdate {
+ fmt.Printf("￮ %s will be updated\n", fieldToUpdate)
+ }
+
for _, ngName := range configureChanges.NodeGroupsToScale {
ngOld := oldCc.GetNodeGroupByName(ngName)
ngScaled := newCc.GetNodeGroupByName(ngName)
diff --git a/manager/install.sh b/manager/install.sh
index 204e1c63b0..8d42997637 100755
--- a/manager/install.sh
+++ b/manager/install.sh
@@ -90,6 +90,8 @@ function cluster_configure() {
add_nodegroups
remove_nodegroups
+ update_networking
+
echo -n "￮ updating cluster configuration "
setup_configmap
echo "✓"
@@ -365,6 +367,51 @@ function setup_istio() {
output_if_error istio-${ISTIO_VERSION}/bin/istioctl install --skip-confirmation --filename /workspace/istio.yaml
}
+function update_networking() {
+ prev_ssl_certificate_arn=$(kubectl get svc ingressgateway-apis -n=istio-system -o json | jq -r '.metadata.annotations."service.beta.kubernetes.io/aws-load-balancer-ssl-cert"')
+
+ if [ "$prev_ssl_certificate_arn" = "null" ]; then
+ prev_ssl_certificate_arn=""
+ fi
+
+ new_ssl_certificate_arn=$(cat $CORTEX_CLUSTER_CONFIG_FILE | yq -r .ssl_certificate_arn)
+
+ if [ "$new_ssl_certificate_arn" = "null" ]; then
+ new_ssl_certificate_arn=""
+ fi
+
+ prev_api_whitelist_ip_address=$(kubectl get svc ingressgateway-apis -n=istio-system -o yaml | yq -r -c ".spec.loadBalancerSourceRanges")
+ prev_operator_whitelist_ip_address=$(kubectl get svc ingressgateway-operator -n=istio-system -o yaml | yq -r -c ".spec.loadBalancerSourceRanges")
+
+ new_api_whitelist_ip_address=$(cat $CORTEX_CLUSTER_CONFIG_FILE | yq -r -c ".api_load_balancer_cidr_white_list")
+ new_operator_whitelist_ip_address=$(cat $CORTEX_CLUSTER_CONFIG_FILE | yq -r -c ".operator_load_balancer_cidr_white_list")
+
+ if [ "$prev_ssl_certificate_arn" = "$new_ssl_certificate_arn" ] && [ "$prev_api_whitelist_ip_address" = "$new_api_whitelist_ip_address" ] && [ "$prev_operator_whitelist_ip_address" = "$new_operator_whitelist_ip_address" ] ; then
+ return
+ fi
+
+ echo -n "￮ updating networking configuration "
+
+ if [ "$new_ssl_certificate_arn" != "$prev_ssl_certificate_arn" ] ; then
+ # there is a bug where changing the certificate annotation will not cause the HTTPS listener in the NLB to update
+ # the current workaround is to delete the HTTPS listener and have it recreated with istioctl
+ if [ "$prev_ssl_certificate_arn" != "" ] ; then
+ kubectl patch svc ingressgateway-apis -n=istio-system --type=json -p="[{'op': 'remove', 'path': '/metadata/annotations/service.beta.kubernetes.io~1aws-load-balancer-ssl-cert'}]" >/dev/null
+ fi
+ https_index=$(kubectl get svc ingressgateway-apis -n=istio-system -o json | jq '.spec.ports | map(.name == "https") | index(true)')
+ if [ "$https_index" != "null" ] ; then
+ kubectl patch svc ingressgateway-apis -n=istio-system --type=json -p="[{'op': 'remove', 'path': '/spec/ports/$https_index'}]" >/dev/null
+ fi
+ fi
+
+ python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/istio.yaml.j2 > /workspace/istio.yaml
+ output_if_error istio-${ISTIO_VERSION}/bin/istioctl install --skip-confirmation --filename /workspace/istio.yaml
+ python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/apis.yaml.j2 > /workspace/apis.yaml
+ kubectl apply -f /workspace/apis.yaml >/dev/null
+
+ echo "✓"
+}
+
function validate_cortex() {
set +e
diff --git a/manager/manifests/istio.yaml.j2 b/manager/manifests/istio.yaml.j2
index bf0a60af41..9df2d198a5 100644
--- a/manager/manifests/istio.yaml.j2
+++ b/manager/manifests/istio.yaml.j2
@@ -50,9 +50,7 @@ spec:
service:
type: LoadBalancer
externalTrafficPolicy: Cluster # https://medium.com/pablo-perez/k8s-externaltrafficpolicy-local-or-cluster-40b259a19404, https://www.asykim.com/blog/deep-dive-into-kubernetes-external-traffic-policies
- {% if config.get('operator_load_balancer_cidr_white_list', [])|length > 0 %}
- loadBalancerSourceRanges: {{ config['operator_load_balancer_cidr_white_list'] }}
- {% endif %}
+ loadBalancerSourceRanges: {{ config.get('operator_load_balancer_cidr_white_list', ['0.0.0.0/0']) }}
selector:
app: operator-istio-gateway
istio: ingressgateway-operator
@@ -110,9 +108,7 @@ spec:
{% endif %}
service:
type: LoadBalancer
- {% if config.get('api_load_balancer_cidr_white_list', [])|length > 0 %}
- loadBalancerSourceRanges: {{ config['api_load_balancer_cidr_white_list'] }}
- {% endif %}
+ loadBalancerSourceRanges: {{ config.get('api_load_balancer_cidr_white_list', ['0.0.0.0/0']) }}
externalTrafficPolicy: Cluster # https://medium.com/pablo-perez/k8s-externaltrafficpolicy-local-or-cluster-40b259a19404, https://www.asykim.com/blog/deep-dive-into-kubernetes-external-traffic-policies
selector:
app: apis-istio-gateway
diff --git a/pkg/types/clusterconfig/cluster_config.go b/pkg/types/clusterconfig/cluster_config.go
index b48f9e403a..0029acfe3b 100644
--- a/pkg/types/clusterconfig/cluster_config.go
+++ b/pkg/types/clusterconfig/cluster_config.go
@@ -39,6 +39,7 @@ import (
"github.com/cortexlabs/cortex/pkg/lib/pointer"
"github.com/cortexlabs/cortex/pkg/lib/sets/strset"
"github.com/cortexlabs/cortex/pkg/lib/slices"
+ libstr "github.com/cortexlabs/cortex/pkg/lib/strings"
"github.com/cortexlabs/cortex/pkg/lib/structs"
"github.com/cortexlabs/yaml"
)
@@ -191,10 +192,11 @@ type ConfigureChanges struct {
NodeGroupsToRemove []string
NodeGroupsToScale []string
EKSNodeGroupsToRemove []string // EKS node group names of (NodeGroupsToRemove ∩ Cortex-converted EKS node groups) ∪ (Cortex-converted EKS node groups - the new cluster config's nodegroups)
+ FieldsToUpdate []string
}
func (c *ConfigureChanges) HasChanges() bool {
- return len(c.NodeGroupsToAdd) != 0 || len(c.NodeGroupsToRemove) != 0 || len(c.NodeGroupsToScale) != 0 || len(c.EKSNodeGroupsToRemove) != 0
+ return len(c.NodeGroupsToAdd)+len(c.NodeGroupsToRemove)+len(c.NodeGroupsToScale)+len(c.EKSNodeGroupsToRemove)+len(c.FieldsToUpdate) != 0
}
// GetGhostEKSNodeGroups returns the set difference between EKSNodeGroupsToRemove and the EKS-converted NodeGroupsToRemove
@@ -1029,43 +1031,54 @@ func (cc *Config) validate(awsClient *aws.Client) error {
return nil
}
-func (cc *Config) validateConfigDiff(oldConfig Config) error {
- err := cc.validateTopLevelSectionDiff(oldConfig)
- if err != nil {
- return err
- }
-
- return cc.validateSharedNodeGroupsDiff(oldConfig)
-}
-
-func (cc *Config) validateTopLevelSectionDiff(oldConfig Config) error {
+func (cc *Config) validateTopLevelSectionDiff(oldConfig Config) ([]string, error) {
+ var fieldsToUpdate []string
// validate actionable changes
newClusterConfigCopy, err := cc.DeepCopy()
if err != nil {
- return err
+ return nil, err
}
oldClusterConfigCopy, err := oldConfig.DeepCopy()
if err != nil {
- return err
+ return nil, err
+ }
+
+ if libstr.Obj(newClusterConfigCopy.SSLCertificateARN) != libstr.Obj(oldClusterConfigCopy.SSLCertificateARN) {
+ fieldsToUpdate = append(fieldsToUpdate, SSLCertificateARNKey)
}
- newClusterConfigCopy.NodeGroups = []*NodeGroup{}
- oldClusterConfigCopy.NodeGroups = []*NodeGroup{}
+ if libstr.Obj(newClusterConfigCopy.APILoadBalancerCIDRWhiteList) != libstr.Obj(oldClusterConfigCopy.APILoadBalancerCIDRWhiteList) {
+ fieldsToUpdate = append(fieldsToUpdate, APILoadBalancerCIDRWhiteListKey)
+ }
+
+ if libstr.Obj(newClusterConfigCopy.OperatorLoadBalancerCIDRWhiteList) != libstr.Obj(oldClusterConfigCopy.OperatorLoadBalancerCIDRWhiteList) {
+ fieldsToUpdate = append(fieldsToUpdate, OperatorLoadBalancerCIDRWhiteListKey)
+ }
+
+ clearUpdatableFields(&newClusterConfigCopy)
+ clearUpdatableFields(&oldClusterConfigCopy)
h1, err := newClusterConfigCopy.Hash()
if err != nil {
- return err
+ return nil, err
}
h2, err := oldClusterConfigCopy.Hash()
if err != nil {
- return err
+ return nil, err
}
if h1 != h2 {
- return ErrorConfigCannotBeChangedOnConfigure()
+ return nil, ErrorConfigCannotBeChangedOnConfigure()
}
- return nil
+ return fieldsToUpdate, nil
+}
+
+func clearUpdatableFields(clusterConfig *Config) {
+ clusterConfig.SSLCertificateARN = nil
+ clusterConfig.APILoadBalancerCIDRWhiteList = nil
+ clusterConfig.OperatorLoadBalancerCIDRWhiteList = nil
+ clusterConfig.NodeGroups = []*NodeGroup{}
}
func (cc *Config) validateSharedNodeGroupsDiff(oldConfig Config) error {
@@ -1139,7 +1152,12 @@ func (cc *Config) ValidateOnConfigure(awsClient *aws.Client, oldConfig Config, e
return ConfigureChanges{}, err
}
- err = cc.validateConfigDiff(oldConfig)
+ fieldsToUpdate, err := cc.validateTopLevelSectionDiff(oldConfig)
+ if err != nil {
+ return ConfigureChanges{}, err
+ }
+
+ err = cc.validateSharedNodeGroupsDiff(oldConfig)
if err != nil {
return ConfigureChanges{}, err
}
@@ -1170,6 +1188,7 @@ func (cc *Config) ValidateOnConfigure(awsClient *aws.Client, oldConfig Config, e
NodeGroupsToRemove: GetNodeGroupNames(ngsToBeRemoved),
NodeGroupsToScale: GetNodeGroupNames(ngNamesToBeScaled),
EKSNodeGroupsToRemove: getStaleEksNodeGroups(cc.ClusterName, eksNodeGroupStacks, cc.NodeGroups, ngsToBeRemoved),
+ FieldsToUpdate: fieldsToUpdate,
}, nil
}
diff --git a/pkg/types/clusterconfig/config_key.go b/pkg/types/clusterconfig/config_key.go
index 19df34d3f2..bbb7e6dc15 100644
--- a/pkg/types/clusterconfig/config_key.go
+++ b/pkg/types/clusterconfig/config_key.go
@@ -53,6 +53,8 @@ const (
NATGatewayKey = "nat_gateway"
APILoadBalancerSchemeKey = "api_load_balancer_scheme"
OperatorLoadBalancerSchemeKey = "operator_load_balancer_scheme"
+ APILoadBalancerCIDRWhiteListKey = "api_load_balancer_cidr_white_list"
+ OperatorLoadBalancerCIDRWhiteListKey = "operator_load_balancer_cidr_white_list"
VPCCIDRKey = "vpc_cidr"
AccountIDKey = "account_id"
TelemetryKey = "telemetry"
diff --git a/pkg/types/clusterconfig/errors.go b/pkg/types/clusterconfig/errors.go
index b5126d94bc..5c00328654 100644
--- a/pkg/types/clusterconfig/errors.go
+++ b/pkg/types/clusterconfig/errors.go
@@ -280,7 +280,7 @@ func ErrorNoNATGatewayWithSubnets() error {
func ErrorConfigCannotBeChangedOnConfigure() error {
return errors.WithStack(&errors.Error{
Kind: ErrConfigCannotBeChangedOnConfigure,
- Message: fmt.Sprintf("in a running cluster, only the %s field can be modified", NodeGroupsKey),
+ Message: fmt.Sprintf("in a running cluster, only %s can be modified", s.StrsAnd([]string{NodeGroupsKey, SSLCertificateARNKey, OperatorLoadBalancerCIDRWhiteListKey, APILoadBalancerCIDRWhiteListKey})),
})
}