From 59113ee4c72b05d1d88d559ab00ce8e499315aab Mon Sep 17 00:00:00 2001 From: Mark Beacom <7315957+mbeacom@users.noreply.github.com> Date: Tue, 31 Jan 2023 10:52:56 -0500 Subject: [PATCH] Fix #8 - Autoformat/style codebase, add pyproject configs, and remove unused imports --- eksupdate/cli.py | 65 +- eksupdate/src/boto_aws.py | 473 +++--- eksupdate/src/eks_get_image_type.py | 108 +- eksupdate/src/eksctlfinal.py | 554 +++--- eksupdate/src/ekslogs.py | 70 +- eksupdate/src/k8s_client.py | 513 +++--- eksupdate/src/latest_ami.py | 45 +- eksupdate/src/preflight_module.py | 2450 ++++++++++++++++----------- eksupdate/src/self_managed.py | 274 +-- eksupdate/starter.py | 486 ++++-- pyproject.toml | 19 + 11 files changed, 2933 insertions(+), 2124 deletions(-) diff --git a/eksupdate/cli.py b/eksupdate/cli.py index 6fbd025..31022dd 100644 --- a/eksupdate/cli.py +++ b/eksupdate/cli.py @@ -1,9 +1,10 @@ -from eksupdate.starter import main import argparse +from eksupdate.starter import main + def entry(): - example_text = ''' + example_text = """ example: python3 eks_updater.py Name_Of_Cluster new_Version Region @@ -21,26 +22,54 @@ def entry(): -> python3 eks_updater.py Cluster_Name new_Version aws_Region --preflight - ''' - regions_list=['af-south-1', 'eu-north-1', 'ap-south-1', 'eu-west-3', 'eu-west-2', 'eu-south-1', 'eu-west-1', 'ap-northeast-3', 'ap-northeast-2', 'me-south-1', 'ap-northeast-1', 'sa-east-1', 'ca-central-1', 'ap-east-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-central-1', 'us-east-1', 'us-east-2', 'us-west-1', 'us-west-2'] + """ + regions_list = [ + "af-south-1", + "eu-north-1", + "ap-south-1", + "eu-west-3", + "eu-west-2", + "eu-south-1", + "eu-west-1", + "ap-northeast-3", + "ap-northeast-2", + "me-south-1", + "ap-northeast-1", + "sa-east-1", + "ca-central-1", + "ap-east-1", + "ap-southeast-1", + "ap-southeast-2", + "eu-central-1", + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + ] - parser = argparse.ArgumentParser(description='Eks Cluster OneClick Upgrade',epilog=example_text, - formatter_class=argparse.RawDescriptionHelpFormatter - ) + parser = argparse.ArgumentParser( + description="Eks Cluster OneClick Upgrade", + epilog=example_text, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) parser.add_argument("name", help="Cluster Name") - parser.add_argument("version",help="new version which you want to update") - parser.add_argument("region",help="Give the region name "+', '.join(regions_list)) - parser.add_argument('--pass_vpc', action="store_true", default=False,help='this --pass-vpc will skip the vpc cni upgrade') - parser.add_argument("--max_retry",default=2,type=int,help="you can specify max retry or else by default it is 2") - parser.add_argument("--force",action="store_true", default=False,help="force pod eviction when you have pdb") - #Eksctl will be added in future version - parser.add_argument("--eksctl",action="store_true", default=False,help="eksctl upgrade process") - parser.add_argument("--preflight",action="store_true", default=False,help="Run preflight check without upgrade") - parser.add_argument("--email",default = False , help='Email for sharing the preflight report') - parser.add_argument("--parallel",action="store_true",default = False , help='Parllel Upgrade all node groups together ') + parser.add_argument("version", help="new version which you want to update") + parser.add_argument("region", help="Give the region name " + ", ".join(regions_list)) + parser.add_argument( + "--pass_vpc", action="store_true", default=False, help="this --pass-vpc will skip the vpc cni upgrade" + ) + parser.add_argument("--max_retry", default=2, type=int, help="you can specify max retry or else by default it is 2") + parser.add_argument("--force", action="store_true", default=False, help="force pod eviction when you have pdb") + # Eksctl will be added in future version + parser.add_argument("--eksctl", action="store_true", default=False, help="eksctl upgrade process") + parser.add_argument("--preflight", action="store_true", default=False, help="Run preflight check without upgrade") + parser.add_argument("--email", default=False, help="Email for sharing the preflight report") + parser.add_argument( + "--parallel", action="store_true", default=False, help="Parllel Upgrade all node groups together " + ) args = parser.parse_args() main(args) if __name__ == "__main__": - entry() + entry() diff --git a/eksupdate/src/boto_aws.py b/eksupdate/src/boto_aws.py index b09d13b..ad9a09c 100644 --- a/eksupdate/src/boto_aws.py +++ b/eksupdate/src/boto_aws.py @@ -1,15 +1,17 @@ -from platform import version -import boto3 import datetime import time import uuid + +import boto3 + from .ekslogs import logs_pusher -def status_of_cluster(Clustname,regionName): - eks_client = boto3.client('eks',region_name=regionName) - '''checking the satus of the cluster and version of the cluster ''' + + +def status_of_cluster(Clustname, regionName): + eks_client = boto3.client("eks", region_name=regionName) + """checking the satus of the cluster and version of the cluster """ try: - response = eks_client.describe_cluster( - name=Clustname) + response = eks_client.describe_cluster(name=Clustname) return [response["cluster"]["status"], response["cluster"]["version"]] except Exception as e: @@ -17,114 +19,120 @@ def status_of_cluster(Clustname,regionName): print(e) raise Exception(str(e)) -def is_cluster_exists(Clustname,regionName): - '''checking wether the cluster exists or not ''' + +def is_cluster_exists(Clustname, regionName): + """checking wether the cluster exists or not""" try: - response = status_of_cluster(Clustname,regionName) + response = status_of_cluster(Clustname, regionName) return response[0] except Exception as e: # logs_pusher(e,"error") raise Exception(e) -def get_latest_instance(asg_name, add_time,regionName): +def get_latest_instance(asg_name, add_time, regionName): """Retrieve the most recently launched/launching instance. Note that this is not necessarily the same one that was launched by `add_node()`, but it's the best I could think of""" - asg_client = boto3.client("autoscaling",region_name=regionName) - ec2_client = boto3.client("ec2",region_name=regionName) + asg_client = boto3.client("autoscaling", region_name=regionName) + ec2_client = boto3.client("ec2", region_name=regionName) instances = [] - response = asg_client.describe_auto_scaling_groups( - AutoScalingGroupNames=[ - asg_name - ] - ) + response = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name]) time.sleep(20) - instance_ids = [instance["InstanceId"] - for instance in response["AutoScalingGroups"][0]["Instances"]] + instance_ids = [instance["InstanceId"] for instance in response["AutoScalingGroups"][0]["Instances"]] response = ec2_client.describe_instances(InstanceIds=instance_ids) for reservation in response["Reservations"]: for instance in reservation["Instances"]: instances.append(instance) # print(instances) - instance_launch_times = [ - {x["PrivateDnsName"]: x["LaunchTime"]} for x in instances] + instance_launch_times = [{x["PrivateDnsName"]: x["LaunchTime"]} for x in instances] instances_valid = [] - instances_valid = [instance for instance in instances if instance["State"]["Name"] in [ - "pending", "running"] and instance["LaunchTime"] > add_time] + instances_valid = [ + instance + for instance in instances + if instance["State"]["Name"] in ["pending", "running"] and instance["LaunchTime"] > add_time + ] latest_instance = "" try: time.sleep(10) - latest_instance = sorted( - instances_valid, key=lambda instance: instance["LaunchTime"])[-1] + latest_instance = sorted(instances_valid, key=lambda instance: instance["LaunchTime"])[-1] except Exception as e: # logs_pusher(e,"error") raise Exception(e) - return latest_instance.get('InstanceId') + return latest_instance.get("InstanceId") -def wait_for_ready(instanceid,regionName): - - ec2_client = boto3.client("ec2",region_name=regionName) - st=time.time() +def wait_for_ready(instanceid, regionName): - ''' waiting for the cluster to pass the status checks ''' - print(instanceid+" waiting for the instance to pass the Health Checks ") + ec2_client = boto3.client("ec2", region_name=regionName) + st = time.time() + + """ waiting for the cluster to pass the status checks """ + print(instanceid + " waiting for the instance to pass the Health Checks ") try: - while ec2_client.describe_instance_status(InstanceIds=[instanceid])['InstanceStatuses'][0]['InstanceStatus']['Details'][0]['Status']!='passed': + while ( + ec2_client.describe_instance_status(InstanceIds=[instanceid])["InstanceStatuses"][0]["InstanceStatus"][ + "Details" + ][0]["Status"] + != "passed" + ): end = time.time() - hours, rem = divmod(end-st, 3600) + hours, rem = divmod(end - st, 3600) minutes, seconds = divmod(rem, 60) - print(instanceid," waiting for the instance to pass the Health Checks ","{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) + print( + instanceid, + " waiting for the instance to pass the Health Checks ", + "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds), + ) time.sleep(20) except Exception as e: print(e) - raise Exception(str(e)+" Please rerun the Script the instance will be created") + raise Exception(str(e) + " Please rerun the Script the instance will be created") return True -def check_asg_autoscaler(asg_name,regionName): + +def check_asg_autoscaler(asg_name, regionName): """Checking wether the autoscaling is present or not""" - asg_client = boto3.client("autoscaling",region_name=regionName) - response = asg_client.describe_auto_scaling_groups( - AutoScalingGroupNames=[ - asg_name - ] - ) - pat="k8s.io/cluster-autoscaler/enabled" - asg_list=[] + asg_client = boto3.client("autoscaling", region_name=regionName) + response = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name]) + pat = "k8s.io/cluster-autoscaler/enabled" + asg_list = [] for asg in response["AutoScalingGroups"][0]["Tags"]: - if asg["Key"]==pat: + if asg["Key"] == pat: asg_list.append(asg) - ''' checking wether asg is present ''' - return len(asg_list)>0 -def enable_disable_autoscaler(asg_name,action,regionName): - asg_client=boto3.client('autoscaling',region_name=regionName) + """ checking wether asg is present """ + return len(asg_list) > 0 + + +def enable_disable_autoscaler(asg_name, action, regionName): + asg_client = boto3.client("autoscaling", region_name=regionName) try: - if action=="pause": + if action == "pause": asg_client.delete_tags( - Tags=[ - { - 'ResourceId': asg_name, - 'ResourceType': "auto-scaling-group", - 'Key': "k8s.io/cluster-autoscaler/enabled" - }, - ] ) + Tags=[ + { + "ResourceId": asg_name, + "ResourceType": "auto-scaling-group", + "Key": "k8s.io/cluster-autoscaler/enabled", + }, + ] + ) return "done" - elif action=="start": + elif action == "start": asg_client.create_or_update_tags( - Tags=[ - { - 'ResourceId': asg_name, - 'ResourceType': "auto-scaling-group", - 'Key': "k8s.io/cluster-autoscaler/enabled", - 'Value': "true", - 'PropagateAtLaunch': False - }, - ] + Tags=[ + { + "ResourceId": asg_name, + "ResourceType": "auto-scaling-group", + "Key": "k8s.io/cluster-autoscaler/enabled", + "Value": "true", + "PropagateAtLaunch": False, + }, + ] ) return "done" else: @@ -134,139 +142,155 @@ def enable_disable_autoscaler(asg_name,action,regionName): raise Exception(e) finally: return "Something went Wrong auto scaling operation failed" - - - -def update_cluster(Clustname, Version,regionName): - eks_client = boto3.client('eks',region_name=regionName) - ''' checking for cluster update ''' - logs_pusher(regionName=regionName,cluster_name=Clustname,msg="The Cluster status = {Status} and version = {Version}".format(Status=status_of_cluster(Clustname,regionName)[0],Version=status_of_cluster(Clustname,regionName)[1])) +def update_cluster(Clustname, Version, regionName): + eks_client = boto3.client("eks", region_name=regionName) + """ checking for cluster update """ + logs_pusher( + regionName=regionName, + cluster_name=Clustname, + msg="The Cluster status = {Status} and version = {Version}".format( + Status=status_of_cluster(Clustname, regionName)[0], Version=status_of_cluster(Clustname, regionName)[1] + ), + ) try: - if status_of_cluster(Clustname,regionName)[1]==Version: - print("The {clustname} cluster is already Updated to {version}".format(clustname=Clustname,version=Version)) - logs_pusher(regionName=regionName,cluster_name=Clustname,msg="The {clustname} cluster is already Updated to {version}".format(clustname=Clustname,version=Version)) + if status_of_cluster(Clustname, regionName)[1] == Version: + print( + "The {clustname} cluster is already Updated to {version}".format(clustname=Clustname, version=Version) + ) + logs_pusher( + regionName=regionName, + cluster_name=Clustname, + msg="The {clustname} cluster is already Updated to {version}".format( + clustname=Clustname, version=Version + ), + ) return True - start=time.time() + start = time.time() while True: - if is_cluster_exists(Clustname,regionName) == "ACTIVE" and status_of_cluster(Clustname,regionName)[1] != Version: - resp=eks_client.update_cluster_version( - name=Clustname, - version=Version) - print("The {clustname} Cluster upgrade is initiated and getting updated to {version} ".format(clustname=Clustname,version=Version)) + if ( + is_cluster_exists(Clustname, regionName) == "ACTIVE" + and status_of_cluster(Clustname, regionName)[1] != Version + ): + resp = eks_client.update_cluster_version(name=Clustname, version=Version) + print( + "The {clustname} Cluster upgrade is initiated and getting updated to {version} ".format( + clustname=Clustname, version=Version + ) + ) time.sleep(40) - print("The {clustname} is still in the upgrade process this usually takes longer time..... ".format(clustname=Clustname)) + print( + "The {clustname} is still in the upgrade process this usually takes longer time..... ".format( + clustname=Clustname + ) + ) time.sleep(20) - - - if is_cluster_exists(Clustname,regionName) == "UPDATING": + + if is_cluster_exists(Clustname, regionName) == "UPDATING": # logs_pusher("Still Updating","info") end = time.time() - hours, rem = divmod(end-start, 3600) + hours, rem = divmod(end - start, 3600) minutes, seconds = divmod(rem, 60) - print("The Cluster {Clustname} is Still Updating to {version} .....".format(Clustname=Clustname,version=Version),"{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) + print( + "The Cluster {Clustname} is Still Updating to {version} .....".format( + Clustname=Clustname, version=Version + ), + "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds), + ) time.sleep(20) - - if is_cluster_exists(Clustname,regionName) == "ACTIVE" and status_of_cluster(Clustname,regionName)[1] == Version: - print("The {clustname} Updated to {version}".format(clustname=Clustname,version=Version)) - logs_pusher(regionName=regionName,cluster_name=Clustname,msg="The {clustname} cluster is already Updated to {version}".format(clustname=Clustname,version=Version)) + if ( + is_cluster_exists(Clustname, regionName) == "ACTIVE" + and status_of_cluster(Clustname, regionName)[1] == Version + ): + print("The {clustname} Updated to {version}".format(clustname=Clustname, version=Version)) + logs_pusher( + regionName=regionName, + cluster_name=Clustname, + msg="The {clustname} cluster is already Updated to {version}".format( + clustname=Clustname, version=Version + ), + ) - break - return True except Exception as e: # logs_pusher(e,"error") raise Exception(e) -def worker_terminate(instance_id,regionName): - ''' terminating instance and decreasing the desired capacity whit asg terminate instance''' - asg_client = boto3.client("autoscaling",region_name=regionName) +def worker_terminate(instance_id, regionName): + """terminating instance and decreasing the desired capacity whit asg terminate instance""" + asg_client = boto3.client("autoscaling", region_name=regionName) try: - asg_client.terminate_instance_in_auto_scaling_group( - InstanceId=instance_id, - ShouldDecrementDesiredCapacity=True - ) + asg_client.terminate_instance_in_auto_scaling_group(InstanceId=instance_id, ShouldDecrementDesiredCapacity=True) except Exception as e: # logs_pusher(e,"error") raise Exception(e) -def add_node(asg_name,regionName): - ''' add node to particular asg ''' - asg_client = boto3.client("autoscaling",region_name=regionName) +def add_node(asg_name, regionName): + """add node to particular asg""" + asg_client = boto3.client("autoscaling", region_name=regionName) - response = asg_client.describe_auto_scaling_groups( - AutoScalingGroupNames=[ - asg_name - ] - ) + response = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name]) try: - old_capacity_mx = response["AutoScalingGroups"][0]['MaxSize'] + old_capacity_mx = response["AutoScalingGroups"][0]["MaxSize"] old_capacity_des = response["AutoScalingGroups"][0]["DesiredCapacity"] except Exception as e: raise Exception("Error Index out of bound due to no max capacity field") if int(old_capacity_des) >= int(old_capacity_mx): asg_client.update_auto_scaling_group( - AutoScalingGroupName=asg_name, - MaxSize=(int(old_capacity_mx)+int(old_capacity_des)) + AutoScalingGroupName=asg_name, MaxSize=(int(old_capacity_mx) + int(old_capacity_des)) ) old_capacity = response["AutoScalingGroups"][0]["DesiredCapacity"] new_capacity = old_capacity + 1 try: - asg_client.set_desired_capacity( - AutoScalingGroupName=asg_name, - DesiredCapacity=new_capacity - ) + asg_client.set_desired_capacity(AutoScalingGroupName=asg_name, DesiredCapacity=new_capacity) # logs_pusher("New Node Added to"+asg_name+"with capacity"+str(new_capacity),"info") - print("New Node has been Added to "+asg_name) + print("New Node has been Added to " + asg_name) except Exception as e: # logs_pusher(e,"error") raise Exception(e) -def get_num_of_instances(asg_name, exclude_ids,regionName): - asg_client = boto3.client("autoscaling",region_name=regionName) +def get_num_of_instances(asg_name, exclude_ids, regionName): + asg_client = boto3.client("autoscaling", region_name=regionName) - ec2_client = boto3.client("ec2",region_name=regionName) - + ec2_client = boto3.client("ec2", region_name=regionName) - '''counting the number of instances ''' + """counting the number of instances """ instances = [] - response = asg_client.describe_auto_scaling_groups( - AutoScalingGroupNames=[ - asg_name - ] - ) - instance_ids = [instance["InstanceId"] for instance in response["AutoScalingGroups"] - [0]["Instances"] if instance["InstanceId"] not in exclude_ids] + response = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name]) + instance_ids = [ + instance["InstanceId"] + for instance in response["AutoScalingGroups"][0]["Instances"] + if instance["InstanceId"] not in exclude_ids + ] response = ec2_client.describe_instances(InstanceIds=instance_ids) for reservation in response["Reservations"]: for instance in reservation["Instances"]: instances.append(instance) - ''' getting the instance in running or pending state''' - instances = [instance for instance in instances if instance["State"] - ["Name"] in ["running", "pending"]] + """ getting the instance in running or pending state""" + instances = [instance for instance in instances if instance["State"]["Name"] in ["running", "pending"]] return len(instances) -def get_Asgs(cluster_name,regionName): - asg_client = boto3.client("autoscaling",region_name=regionName) +def get_Asgs(cluster_name, regionName): + asg_client = boto3.client("autoscaling", region_name=regionName) - ''' + """ We get a list of Asg's (auto scaling groups) Which will mach our format "kubernetes.io/cluster/{cluster_name}" and returns an empty list if none are found - ''' + """ pat = "kubernetes.io/cluster/{clusterName}" response = asg_client.describe_auto_scaling_groups() matching = [] @@ -280,28 +304,24 @@ def get_Asgs(cluster_name,regionName): if tg["Key"] == pat.format(clusterName=cluster_name): matching.append(asg) matching_names = [x["AutoScalingGroupName"] for x in matching] - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg="ASG Matched = "+" ,".join(matching_names)) + logs_pusher(regionName=regionName, cluster_name=cluster_name, msg="ASG Matched = " + " ,".join(matching_names)) return matching_names -def get_latest_lt_version(lt_id,regionName): - ec2_client = boto3.client("ec2",region_name=regionName) - - ''' getting the latest launch template version ''' - response = ec2_client.describe_launch_templates( - LaunchTemplateIds=[ - lt_id - ] - ) - latest_version = response['LaunchTemplates'][0]['LatestVersionNumber'] +def get_latest_lt_version(lt_id, regionName): + ec2_client = boto3.client("ec2", region_name=regionName) + + """ getting the latest launch template version """ + response = ec2_client.describe_launch_templates(LaunchTemplateIds=[lt_id]) + latest_version = response["LaunchTemplates"][0]["LatestVersionNumber"] # logs_pusher(latest_version,"info") return latest_version def old_lt_secanarios(inst, asg_lt_name, asg_lt_version): - ''' Getting old launch template based on launch template name and version 1!=2 ''' - lt_name = inst['LaunchTemplate']['LaunchTemplateName'] - lt_version = int(inst['LaunchTemplate']['Version']) + """Getting old launch template based on launch template name and version 1!=2""" + lt_name = inst["LaunchTemplate"]["LaunchTemplateName"] + lt_version = int(inst["LaunchTemplate"]["Version"]) if lt_name != asg_lt_name: return True elif lt_version != int(asg_lt_version): @@ -310,41 +330,40 @@ def old_lt_secanarios(inst, asg_lt_name, asg_lt_version): return False -def get_old_lt(asg_name,regionName): +def get_old_lt(asg_name, regionName): """Get old launc template""" - asg_client = boto3.client("autoscaling",region_name=regionName) - ec2_client = boto3.client("ec2",region_name=regionName) - + asg_client = boto3.client("autoscaling", region_name=regionName) + ec2_client = boto3.client("ec2", region_name=regionName) old_lt_instance_ids = [] instances = [] - response = asg_client.describe_auto_scaling_groups( - AutoScalingGroupNames=[ - asg_name - ] - - ) + response = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name]) asg_lt_name = "" # print(response) - ''' finding the launch type''' - if 'LaunchTemplate' in response['AutoScalingGroups'][0]: + """ finding the launch type""" + if "LaunchTemplate" in response["AutoScalingGroups"][0]: lt_id = response["AutoScalingGroups"][0]["LaunchTemplate"]["LaunchTemplateId"] - asg_lt_name = response["AutoScalingGroups"][0]['LaunchTemplate']['LaunchTemplateName'] - elif 'MixedInstancesPolicy' in response['AutoScalingGroups'][0]: - lt_id = response["AutoScalingGroups"][0]['MixedInstancesPolicy'][ - "LaunchTemplate"]['LaunchTemplateSpecification']["LaunchTemplateId"] - asg_lt_name = response['AutoScalingGroups'][0]['MixedInstancesPolicy']['LaunchTemplate']['LaunchTemplateSpecification'][ - 'LaunchTemplateName'] + asg_lt_name = response["AutoScalingGroups"][0]["LaunchTemplate"]["LaunchTemplateName"] + elif "MixedInstancesPolicy" in response["AutoScalingGroups"][0]: + lt_id = response["AutoScalingGroups"][0]["MixedInstancesPolicy"]["LaunchTemplate"][ + "LaunchTemplateSpecification" + ]["LaunchTemplateId"] + asg_lt_name = response["AutoScalingGroups"][0]["MixedInstancesPolicy"]["LaunchTemplate"][ + "LaunchTemplateSpecification" + ]["LaunchTemplateName"] else: # logs_pusher("None found","error") return "error" # print(latest_lt) - ''' checking wethether there are instances with 1!=2 mismatch template version ''' - old_lt_instance_ids = [instance["InstanceId"] for instance in response["AutoScalingGroups"][0] - ["Instances"] if old_lt_secanarios(instance, asg_lt_name, int(instance["LaunchTemplate"]["Version"]))] + """ checking wethether there are instances with 1!=2 mismatch template version """ + old_lt_instance_ids = [ + instance["InstanceId"] + for instance in response["AutoScalingGroups"][0]["Instances"] + if old_lt_secanarios(instance, asg_lt_name, int(instance["LaunchTemplate"]["Version"])) + ] if len(old_lt_instance_ids) == 0: return [] response = ec2_client.describe_instances(InstanceIds=old_lt_instance_ids) @@ -356,62 +375,55 @@ def get_old_lt(asg_name,regionName): return instances -def old_launchConfiguation_instances(asg_name,regionName): - asg_client = boto3.client("autoscaling",region_name=regionName) +def old_launchConfiguation_instances(asg_name, regionName): + asg_client = boto3.client("autoscaling", region_name=regionName) old_lc_ids = [] - ''' describing the asg group ''' - response = asg_client.describe_auto_scaling_groups( - AutoScalingGroupNames=[ - asg_name - ] - ) + """ describing the asg group """ + response = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name]) # print(response) - instances = response['AutoScalingGroups'][0]["Instances"] + instances = response["AutoScalingGroups"][0]["Instances"] # print(instances) for inst in instances: # print(inst) # logs_pusher(inst) - ''' checking the LaunchConfiguration is matching or not''' - if inst.get('LaunchConfigurationName') != response['AutoScalingGroups'][0]['LaunchConfigurationName']: - old_lc_ids.append(inst['InstanceId']) + """checking the LaunchConfiguration is matching or not""" + if inst.get("LaunchConfigurationName") != response["AutoScalingGroups"][0]["LaunchConfigurationName"]: + old_lc_ids.append(inst["InstanceId"]) return old_lc_ids -def outdated_lt(asgs,regionName): - ''' Getting outdated launch template''' - asg_client = boto3.client("autoscaling",region_name=regionName) - asg = asg_client.describe_auto_scaling_groups( - AutoScalingGroupNames=[ - asgs - ]) +def outdated_lt(asgs, regionName): + """Getting outdated launch template""" + asg_client = boto3.client("autoscaling", region_name=regionName) + asg = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asgs]) # print(asg) # # print(asg) - asg_name = asg['AutoScalingGroups'][0]['AutoScalingGroupName'] + asg_name = asg["AutoScalingGroups"][0]["AutoScalingGroupName"] # print(asg_name) # logs_pusher(asg_name) launch_type = "" - if 'LaunchConfigurationName' in asg['AutoScalingGroups'][0]: + if "LaunchConfigurationName" in asg["AutoScalingGroups"][0]: launch_type = "LaunchConfiguration" - elif 'LaunchTemplate' in asg['AutoScalingGroups'][0]: + elif "LaunchTemplate" in asg["AutoScalingGroups"][0]: launch_type = "LaunchTemplate" - elif 'MixedInstancesPolicy' in asg['AutoScalingGroups'][0]: + elif "MixedInstancesPolicy" in asg["AutoScalingGroups"][0]: launch_type = "LaunchTemplate" else: return [] old_instances = [] - + # logs_pusher(launch_type) if launch_type == "LaunchConfiguration": - temp = old_launchConfiguation_instances(asg_name,regionName) + temp = old_launchConfiguation_instances(asg_name, regionName) if len(temp) > 0: old_instances = temp return old_instances else: return [] - ''' checking with launch Template''' + """ checking with launch Template""" if launch_type == "LaunchTemplate": - temp = get_old_lt(asg_name,regionName) + temp = get_old_lt(asg_name, regionName) if len(temp) > 0: old_instances = temp return old_instances @@ -421,65 +433,54 @@ def outdated_lt(asgs,regionName): return [] -def addAutoScaling(asg_name, img_id,regionName): - asg_client = boto3.client("autoscaling",region_name=regionName) +def addAutoScaling(asg_name, img_id, regionName): + asg_client = boto3.client("autoscaling", region_name=regionName) - ''' Adding new Launch Configuration to the Asg''' + """ Adding new Launch Configuration to the Asg""" timeStamp = time.time() - timeStampString = datetime.datetime.fromtimestamp( - timeStamp).strftime('%Y-%m-%d %H-%M-%S') - response = asg_client.describe_auto_scaling_groups( - AutoScalingGroupNames=[asg_name]) + timeStampString = datetime.datetime.fromtimestamp(timeStamp).strftime("%Y-%m-%d %H-%M-%S") + response = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name]) # print(response) - sourceInstanceId = response.get('AutoScalingGroups')[ - 0]['Instances'][0]['InstanceId'] + sourceInstanceId = response.get("AutoScalingGroups")[0]["Instances"][0]["InstanceId"] # print(sourceInstanceId) - k=str(uuid.uuid4()) - newLaunchConfigName = 'LC ' + img_id+" "+timeStampString+" "+k + k = str(uuid.uuid4()) + newLaunchConfigName = "LC " + img_id + " " + timeStampString + " " + k try: asg_client.create_launch_configuration( - InstanceId=sourceInstanceId, - LaunchConfigurationName=newLaunchConfigName, - ImageId=img_id) + InstanceId=sourceInstanceId, LaunchConfigurationName=newLaunchConfigName, ImageId=img_id + ) response = asg_client.update_auto_scaling_group( - AutoScalingGroupName=asg_name, LaunchConfigurationName=newLaunchConfigName) + AutoScalingGroupName=asg_name, LaunchConfigurationName=newLaunchConfigName + ) print("updated to latest launch configuration") except Exception as e: print(e) # logs_pusher(e) return response -def get_outdated_Asg(asg_name,latest_img,regionName): - asg_client = boto3.client("autoscaling",region_name=regionName) - ec2_client = boto3.client("ec2",region_name=regionName) - - '''Getting outdate asuto scaling group ''' - response=asg_client.describe_auto_scaling_groups( - AutoScalingGroupNames=[ - asg_name - ] +def get_outdated_Asg(asg_name, latest_img, regionName): + asg_client = boto3.client("autoscaling", region_name=regionName) + ec2_client = boto3.client("ec2", region_name=regionName) - ) - instance_ids = [instance["InstanceId"] - for instance in response["AutoScalingGroups"][0]["Instances"]] - old_ami_inst=[] - ''' filetering old instance where the logic is used to check wether we should add new launch configuration or not ''' + """Getting outdate asuto scaling group """ + response = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name]) + instance_ids = [instance["InstanceId"] for instance in response["AutoScalingGroups"][0]["Instances"]] + old_ami_inst = [] + """ filetering old instance where the logic is used to check wether we should add new launch configuration or not """ inst_response = ec2_client.describe_instances(InstanceIds=instance_ids) for reservation in inst_response["Reservations"]: for instance in reservation["Instances"]: - if instance['ImageId'] != latest_img: - old_ami_inst.append(instance['InstanceId']) + if instance["ImageId"] != latest_img: + old_ami_inst.append(instance["InstanceId"]) instance_ids.sort() old_ami_inst.sort() - if len(old_ami_inst)!=len(instance_ids): + if len(old_ami_inst) != len(instance_ids): return False else: for i in range(len(old_ami_inst)): - if old_ami_inst[i]!=instance_ids[i]: + if old_ami_inst[i] != instance_ids[i]: return False return True - - diff --git a/eksupdate/src/eks_get_image_type.py b/eksupdate/src/eks_get_image_type.py index 9407271..a4664eb 100644 --- a/eksupdate/src/eks_get_image_type.py +++ b/eksupdate/src/eks_get_image_type.py @@ -1,63 +1,81 @@ -from platform import node import boto3 + from .k8s_client import find_node -def image_type(node_type,Presentversion,inst,regionName): - ''' returning image location ''' - ec2_client = boto3.client('ec2',region_name=regionName) - if node_type=="Amazon Linux 2": - filters = [{'Name':'owner-id','Values':['602401143452']},{'Name': 'name', 'Values': ["amazon-eks-node-*"]},{'Name':'is-public','Values':['true']}] + + +def image_type(node_type, Presentversion, inst, regionName): + """returning image location""" + ec2_client = boto3.client("ec2", region_name=regionName) + if node_type == "Amazon Linux 2": + filters = [ + {"Name": "owner-id", "Values": ["602401143452"]}, + {"Name": "name", "Values": ["amazon-eks-node-*"]}, + {"Name": "is-public", "Values": ["true"]}, + ] elif "ubuntu" in node_type.lower(): - filters = [{'Name':'owner-id','Values':['099720109477']},{'Name': 'name', 'Values': ["ubuntu-eks/k8s_*"]},{'Name':'is-public','Values':['true']}] + filters = [ + {"Name": "owner-id", "Values": ["099720109477"]}, + {"Name": "name", "Values": ["ubuntu-eks/k8s_*"]}, + {"Name": "is-public", "Values": ["true"]}, + ] elif "bottlerocket" in node_type.lower(): - filters = [{'Name':'owner-id','Values':['092701018921']},{'Name': 'name', 'Values': ["bottlerocket-aws-k8s-*"]},{'Name':'is-public','Values':['true']}] - elif "Windows" in node_type: - filters = [{'Name':'owner-id','Values':['801119661308']},{'Name': 'name', 'Values': ["Windows_Server-*-English-*-EKS_Optimized-*"]},{'Name':'is-public','Values':['true']}] + filters = [ + {"Name": "owner-id", "Values": ["092701018921"]}, + {"Name": "name", "Values": ["bottlerocket-aws-k8s-*"]}, + {"Name": "is-public", "Values": ["true"]}, + ] + elif "Windows" in node_type: + filters = [ + {"Name": "owner-id", "Values": ["801119661308"]}, + {"Name": "name", "Values": ["Windows_Server-*-English-*-EKS_Optimized-*"]}, + {"Name": "is-public", "Values": ["true"]}, + ] else: return True - ''' decribing image types''' + """ decribing image types""" images = ec2_client.describe_images(Filters=filters) - instances_list=[] - for i in images.get('Images'): - instances_list.append([i.get('ImageId'),i.get('Name')]) + instances_list = [] + for i in images.get("Images"): + instances_list.append([i.get("ImageId"), i.get("Name")]) for i in instances_list: if inst in i[0]: return i[1] return inst in instances_list -def get_ami_name(cluster_name,asg_name,PresentVersion,regionName): - asg_client = boto3.client('autoscaling',region_name=regionName) - ec2_client = boto3.client('ec2',region_name=regionName) - response=asg_client.describe_auto_scaling_groups( - AutoScalingGroupNames=[ - asg_name - ] - ) - instance_ids = [instance["InstanceId"] - for instance in response["AutoScalingGroups"][0]["Instances"]] - if len(instance_ids)==0: +def get_ami_name(cluster_name, asg_name, PresentVersion, regionName): + asg_client = boto3.client("autoscaling", region_name=regionName) + ec2_client = boto3.client("ec2", region_name=regionName) + response = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name]) + instance_ids = [instance["InstanceId"] for instance in response["AutoScalingGroups"][0]["Instances"]] + if len(instance_ids) == 0: raise Exception("No Instances") response = ec2_client.describe_instances(InstanceIds=instance_ids) - ans=[] + ans = [] for reservation in response["Reservations"]: for instance in reservation["Instances"]: - instance_id=instance['ImageId'] - ''' getting the instance type as amz2 or windows or ubuntu ''' - node_type=find_node(cluster_name,instance['InstanceId'],"os_type",regionName) - ans.append([node_type,image_type(node_type=node_type,Presentversion=PresentVersion,inst=instance_id,regionName=regionName)]) - ''' custom logic to check wether the os_type is same if same returning and if not returing the least repeated name''' - result=False - if len(ans) > 0 : - result = all(elem[0] == ans[0][0] for i,elem in enumerate(ans)) - if result: - return ans[0] - else: - dd = {} - ac = {} - for (d,ak) in ans: - dd[d] = dd.get(d, 0) + 1 - ac[d] = ac.get(d, ak) - return min((ac.get(d,""), d) for d in dd) - - + instance_id = instance["ImageId"] + """ getting the instance type as amz2 or windows or ubuntu """ + node_type = find_node(cluster_name, instance["InstanceId"], "os_type", regionName) + ans.append( + [ + node_type, + image_type( + node_type=node_type, Presentversion=PresentVersion, inst=instance_id, regionName=regionName + ), + ] + ) + """ custom logic to check wether the os_type is same if same returning and if not returing the least repeated name""" + result = False + if len(ans) > 0: + result = all(elem[0] == ans[0][0] for i, elem in enumerate(ans)) + if result: + return ans[0] + else: + dd = {} + ac = {} + for (d, ak) in ans: + dd[d] = dd.get(d, 0) + 1 + ac[d] = ac.get(d, ak) + return min((ac.get(d, ""), d) for d in dd) diff --git a/eksupdate/src/eksctlfinal.py b/eksupdate/src/eksctlfinal.py index 12cf97a..dbde7ff 100644 --- a/eksupdate/src/eksctlfinal.py +++ b/eksupdate/src/eksctlfinal.py @@ -1,391 +1,405 @@ - -import subprocess -import re -import boto3 +import base64 import json -import os -from .ekslogs import logs_pusher +import re +import subprocess import time -import datetime -import kubernetes.client -from kubernetes import client, config, watch -from kubernetes.client.rest import ApiException -import argparse -from botocore.signers import RequestSigner -import base64 +import boto3 +from botocore.signers import RequestSigner +from kubernetes import client +from .ekslogs import logs_pusher # will be updated soon in future releases right now still in alpha -vpc_version='v1.7.5-eksbuild.1' - - - - +vpc_version = "v1.7.5-eksbuild.1" def botoclient(region): - bclient = boto3.client('eks',region) - return bclient - - - + bclient = boto3.client("eks", region) + return bclient def get_bearer_token(cluster_id, region): - '''' AUthenticating the session with sts token''' + """' AUthenticating the session with sts token""" STS_TOKEN_EXPIRES_IN = 60 session = boto3.session.Session() - client = session.client('sts', region_name=region) + client = session.client("sts", region_name=region) service_id = client.meta.service_model.service_id - signer = RequestSigner( - service_id, - region, - 'sts', - 'v4', - session.get_credentials(), - session.events - ) + signer = RequestSigner(service_id, region, "sts", "v4", session.get_credentials(), session.events) params = { - 'method': 'GET', - 'url': 'https://sts.{}.amazonaws.com/?Action=GetCallerIdentity&Version=2011-06-15'.format(region), - 'body': {}, - 'headers': { - 'x-k8s-aws-id': cluster_id - }, - 'context': {} + "method": "GET", + "url": "https://sts.{}.amazonaws.com/?Action=GetCallerIdentity&Version=2011-06-15".format(region), + "body": {}, + "headers": {"x-k8s-aws-id": cluster_id}, + "context": {}, } - '''Getting a presigned Url''' + """Getting a presigned Url""" signed_url = signer.generate_presigned_url( - params, - region_name=region, - expires_in=STS_TOKEN_EXPIRES_IN, - operation_name='' + params, region_name=region, expires_in=STS_TOKEN_EXPIRES_IN, operation_name="" ) - base64_url = base64.urlsafe_b64encode( - signed_url.encode('utf-8')).decode('utf-8') + base64_url = base64.urlsafe_b64encode(signed_url.encode("utf-8")).decode("utf-8") # remove any base64 encoding padding and returing the kubernets token - return 'k8s-aws-v1.' + re.sub(r'=*', '', base64_url) + return "k8s-aws-v1." + re.sub(r"=*", "", base64_url) -def loading_config(cluster_name,regionName): - ''' loading kubeconfig with sts''' - eks = boto3.client('eks',region_name=regionName) - resp = eks.describe_cluster( - - name=cluster_name - ) +def loading_config(cluster_name, regionName): + """loading kubeconfig with sts""" + eks = boto3.client("eks", region_name=regionName) + resp = eks.describe_cluster(name=cluster_name) endPoint = resp["cluster"]["endpoint"] configs = client.Configuration() configs.host = endPoint configs.verify_ssl = False - configs.debug = False - configs.api_key = {"authorization": "Bearer " + - get_bearer_token(cluster_name, regionName)} + configs.debug = False + configs.api_key = {"authorization": "Bearer " + get_bearer_token(cluster_name, regionName)} client.Configuration.set_default(configs) return "Initialiazed" - - - -#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -def upgrade_cluster(region,clust_name,d,bclient,version): - start_time=time.ctime() - start=time.time() - logs_pusher(regionName=region,cluster_name=clust_name,msg="The cluster Upgrade Started At "+str(start_time)) - - - response=bclient.describe_cluster(name=clust_name) #to describe the cluster using boto3 - print("cluster verion before upgrade "+response['cluster']['version']) #to print present version - d['cluster_prev_version']=response['cluster']['version'] - args='~/eksctl upgrade cluster --name='+clust_name+' --version '+version+' --approve' #upgrades cluster to one version above - output=subprocess.call(args,shell=True) - response=bclient.describe_cluster(name=clust_name) - print("cluster verion after upgrade "+response['cluster']['version']) #to print updated/new version - d['cluster_updated_version:']=response['cluster']['version'] - - end=time.time() - hours, rem = divmod(end-start, 3600) +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +def upgrade_cluster(region, clust_name, d, bclient, version): + start_time = time.ctime() + start = time.time() + logs_pusher(regionName=region, cluster_name=clust_name, msg="The cluster Upgrade Started At " + str(start_time)) + + response = bclient.describe_cluster(name=clust_name) # to describe the cluster using boto3 + print("cluster verion before upgrade " + response["cluster"]["version"]) # to print present version + d["cluster_prev_version"] = response["cluster"]["version"] + args = ( + "~/eksctl upgrade cluster --name=" + clust_name + " --version " + version + " --approve" + ) # upgrades cluster to one version above + output = subprocess.call(args, shell=True) + response = bclient.describe_cluster(name=clust_name) + print("cluster verion after upgrade " + response["cluster"]["version"]) # to print updated/new version + d["cluster_updated_version:"] = response["cluster"]["version"] + + end = time.time() + hours, rem = divmod(end - start, 3600) minutes, seconds = divmod(rem, 60) - print("The time Taken For the cluster Upgrade ","{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) - logs_pusher(regionName=region,cluster_name=clust_name,msg="The time Taken For the cluster Upgrade "+"{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) + print("The time Taken For the cluster Upgrade ", "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds)) + logs_pusher( + regionName=region, + cluster_name=clust_name, + msg="The time Taken For the cluster Upgrade " + + "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds), + ) return d -#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -#upgrades addon availble in given cluster -def add_on_upgrade(region,clust_name,d,v1): - print('addon upgrade started') - start_time=time.ctime() - start=time.time() - logs_pusher(regionName=region,cluster_name=clust_name,msg="The Addons Upgrade Started At "+str(start_time)) - loading_config(clust_name,region) +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# upgrades addon availble in given cluster +def add_on_upgrade(region, clust_name, d, v1): + print("addon upgrade started") + start_time = time.ctime() + start = time.time() + logs_pusher(regionName=region, cluster_name=clust_name, msg="The Addons Upgrade Started At " + str(start_time)) + + loading_config(clust_name, region) v1 = client.CoreV1Api() - + rep = v1.list_namespaced_pod("kube-system") - - d['addonsbeforeupdate']={} + + d["addonsbeforeupdate"] = {} for pod in rep.items: - print(pod.metadata.name ,"Current Version = ", pod.spec.containers[0].image.split(":")[-1]) - d['addonsbeforeupdate'][pod.metadata.name]=pod.spec.containers[0].image.split(":")[-1] + print(pod.metadata.name, "Current Version = ", pod.spec.containers[0].image.split(":")[-1]) + d["addonsbeforeupdate"][pod.metadata.name] = pod.spec.containers[0].image.split(":")[-1] + args = "~/eksctl utils update-kube-proxy --cluster=" + clust_name + " --approve" # to update kube-proxy + output = subprocess.call(args, shell=True) + args = "~/eksctl utils update-coredns --cluster=" + clust_name + " --approve" # to update coredns + output = subprocess.call(args, shell=True) - args='~/eksctl utils update-kube-proxy --cluster='+clust_name+' --approve'#to update kube-proxy - output=subprocess.call(args,shell=True) - args='~/eksctl utils update-coredns --cluster='+clust_name+' --approve' #to update coredns - output=subprocess.call(args,shell=True) + output = botoclient("us-west-2").list_addons(clusterName=clust_name) - output=botoclient('us-west-2').list_addons( clusterName=clust_name) - - if 'vpc-cni' in output['addons']: + if "vpc-cni" in output["addons"]: try: - response = botoclient('us-west-2').describe_cluster( - name=clust_name - ) - if response['ResponseMetadata']['HTTPStatusCode'] == 200: - print("Success response recieved for describing cluster "+ clust_name) - oidc = (response['cluster']['identity']['oidc']['issuer']) - print('OIDC output recieved '+ oidc + ' for Cluster Name ' + clust_name) - response = botoclient('us-west-2').describe_addon(clusterName=clust_name,addonName='vpc-cni') - # print(response['addon']['addonVersion']) - if(response['addon']['addonVersion']!=vpc_version): - args='~/eksctl update addon --name vpc-cni --version '+vpc_version+' --cluster '+clust_name #to update aws-node - output=subprocess.call(args,shell=True) + response = botoclient("us-west-2").describe_cluster(name=clust_name) + if response["ResponseMetadata"]["HTTPStatusCode"] == 200: + print("Success response recieved for describing cluster " + clust_name) + oidc = response["cluster"]["identity"]["oidc"]["issuer"] + print("OIDC output recieved " + oidc + " for Cluster Name " + clust_name) + response = botoclient("us-west-2").describe_addon(clusterName=clust_name, addonName="vpc-cni") + # print(response['addon']['addonVersion']) + if response["addon"]["addonVersion"] != vpc_version: + args = ( + "~/eksctl update addon --name vpc-cni --version " + vpc_version + " --cluster " + clust_name + ) # to update aws-node + output = subprocess.call(args, shell=True) except Exception as e: - print('Failed to fetch Cluster OIDC value for cluster name ' + clust_name, e) + print("Failed to fetch Cluster OIDC value for cluster name " + clust_name, e) else: - args='~/eksctl utils update-aws-node --cluster='+clust_name+' --approve' - output=subprocess.call(args,shell=True) - - print('addons update completed') - end=time.time() - hours, rem = divmod(end-start, 3600) + args = "~/eksctl utils update-aws-node --cluster=" + clust_name + " --approve" + output = subprocess.call(args, shell=True) + + print("addons update completed") + end = time.time() + hours, rem = divmod(end - start, 3600) minutes, seconds = divmod(rem, 60) - print("The time Taken For the Addons Upgrade ","{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) - - - logs_pusher(regionName=region,cluster_name=clust_name,msg="The time Taken For the addons Upgrade "+"{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) - + print("The time Taken For the Addons Upgrade ", "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds)) - return d + logs_pusher( + regionName=region, + cluster_name=clust_name, + msg="The time Taken For the addons Upgrade " + + "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds), + ) + return d -#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -#retrives the self managed nodegroup name -def get_old_smg_node_groups(region,clust_name,bclient): +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# retrives the self managed nodegroup name +def get_old_smg_node_groups(region, clust_name, bclient): - args='~/eksctl get nodegroup --cluster='+clust_name+' -o json' - output=subprocess.check_output(args,shell=True) - output=json.loads((output)) #to extract all nodegroups present in the cluster - old_smg=[] - response = bclient.list_nodegroups( - clusterName=clust_name, + args = "~/eksctl get nodegroup --cluster=" + clust_name + " -o json" + output = subprocess.check_output(args, shell=True) + output = json.loads((output)) # to extract all nodegroups present in the cluster + old_smg = [] + response = bclient.list_nodegroups( + clusterName=clust_name, ) - ls=response['nodegroups'] #to extract MANAGED NODE GROUPS + ls = response["nodegroups"] # to extract MANAGED NODE GROUPS for i in output: - if(i["Name"] not in ls): - old_smg.append(i["Name"]) #to extract SELF MANAGED NODEGROUPS + if i["Name"] not in ls: + old_smg.append(i["Name"]) # to extract SELF MANAGED NODEGROUPS return old_smg -#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -#updates the unmanaged nodegroup by migrating to new nodegroup -def update_unmanaged_nodegroup(region,clust_name,d,bclient): - start_time=time.ctime() - start=time.time() - logs_pusher(regionName=region,cluster_name=clust_name,msg=" update managed nodegroups Started At "+str(start_time)) +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# updates the unmanaged nodegroup by migrating to new nodegroup +def update_unmanaged_nodegroup(region, clust_name, d, bclient): + start_time = time.ctime() + start = time.time() + logs_pusher( + regionName=region, cluster_name=clust_name, msg=" update managed nodegroups Started At " + str(start_time) + ) - old_smg=get_old_smg_node_groups(region,clust_name,bclient) #extract SELF MANAGED NODEGROUPS - args='~/eksctl get nodegroup --cluster='+clust_name+' -o json' - output=subprocess.check_output(args,shell=True) - output=json.loads((output)) + old_smg = get_old_smg_node_groups(region, clust_name, bclient) # extract SELF MANAGED NODEGROUPS + args = "~/eksctl get nodegroup --cluster=" + clust_name + " -o json" + output = subprocess.check_output(args, shell=True) + output = json.loads((output)) response = bclient.list_nodegroups( - clusterName=clust_name, + clusterName=clust_name, ) - d['un_managed_ndgrp_before_update']={} - ls=response['nodegroups'] + d["un_managed_ndgrp_before_update"] = {} + ls = response["nodegroups"] for i in output: - if(i['Name'] not in ls): - print(i['Name'] +" image-id befrore update "+i['ImageID']) #to print PRESENT - SELF MANAGED NODEGROUP IMAGE and ID - d['un_managed_ndgrp_before_update'][i['Name']]=i['ImageID'] - - if(old_smg!=[ ]): - try: #to verify "if SELF MANAGED GROUP exists" + if i["Name"] not in ls: + print( + i["Name"] + " image-id befrore update " + i["ImageID"] + ) # to print PRESENT - SELF MANAGED NODEGROUP IMAGE and ID + d["un_managed_ndgrp_before_update"][i["Name"]] = i["ImageID"] + + if old_smg != []: + try: # to verify "if SELF MANAGED GROUP exists" for i in old_smg: - args="~/eksctl create nodegroup --cluster="+clust_name #creates a node group with CONTROL PLANE version - output=subprocess.call(args,shell=True) - ls=get_old_smg_node_groups(region,clust_name,bclient) + args = ( + "~/eksctl create nodegroup --cluster=" + clust_name + ) # creates a node group with CONTROL PLANE version + output = subprocess.call(args, shell=True) + ls = get_old_smg_node_groups(region, clust_name, bclient) time.sleep(60) - args="~/eksctl drain nodegroup --cluster="+clust_name+" --name="+i #DRAINS the old node groups - output=subprocess.call(args,shell=True) + args = "~/eksctl drain nodegroup --cluster=" + clust_name + " --name=" + i # DRAINS the old node groups + output = subprocess.call(args, shell=True) time.sleep(60) - args="~/eksctl delete nodegroup --cluster="+clust_name+" --name="+i #DELETES the old node groups - output=subprocess.call(args,shell=True) + args = ( + "~/eksctl delete nodegroup --cluster=" + clust_name + " --name=" + i + ) # DELETES the old node groups + output = subprocess.call(args, shell=True) except Exception as e: - print('pdb set cant delete pods'+e) + print("pdb set cant delete pods" + e) else: print("no unmanaged nodegroups") return d - - print('**printing unmanaged nodegroups....waiting for nodegroup to be active') + + print("**printing unmanaged nodegroups....waiting for nodegroup to be active") time.sleep(240) - args='~/eksctl get nodegroup --cluster='+clust_name+' -o json' - output=subprocess.check_output(args,shell=True) - output=json.loads((output)) + args = "~/eksctl get nodegroup --cluster=" + clust_name + " -o json" + output = subprocess.check_output(args, shell=True) + output = json.loads((output)) response = bclient.list_nodegroups( - clusterName=clust_name, + clusterName=clust_name, ) - ls=response['nodegroups'] - - d['un_managed_ndgrp_after_update']={} + ls = response["nodegroups"] + + d["un_managed_ndgrp_after_update"] = {} for i in output: - if(i['Name'] not in ls): - print(i['Name'] +" image-id after update "+i['ImageID']) #to print UPDATED - SELF MANAGED NODEGROUP IMAGE and ID - d['un_managed_ndgrp_after_update'][i['Name']]=i['ImageID'] - - end=time.time() - hours, rem = divmod(end-start, 3600) + if i["Name"] not in ls: + print( + i["Name"] + " image-id after update " + i["ImageID"] + ) # to print UPDATED - SELF MANAGED NODEGROUP IMAGE and ID + d["un_managed_ndgrp_after_update"][i["Name"]] = i["ImageID"] + + end = time.time() + hours, rem = divmod(end - start, 3600) minutes, seconds = divmod(rem, 60) - print("The time Taken For update managed nodegroups ","{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) - logs_pusher(regionName=region,cluster_name=clust_name,msg="The time Taken For update managed nodegroups "+"{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) + print( + "The time Taken For update managed nodegroups ", + "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds), + ) + logs_pusher( + regionName=region, + cluster_name=clust_name, + msg="The time Taken For update managed nodegroups " + + "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds), + ) return d - +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# creates a managed nodegroup +def create_managed_nodegroup(region, clust_name, client): + start_time = time.ctime() + start = time.time() + logs_pusher( + regionName=region, cluster_name=clust_name, msg="creation of managed nodegroups Started At " + str(start_time) + ) -#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -#creates a managed nodegroup -def create_managed_nodegroup(region,clust_name,client): - start_time=time.ctime() - start=time.time() - logs_pusher(regionName=region,cluster_name=clust_name,msg="creation of managed nodegroups Started At "+str(start_time)) - - start_time=time.ctime() - start=time.time() - logs_pusher(regionName=region,cluster_name=clust_name,msg="The Addons Upgrade Started At "+str(start_time)) + start_time = time.ctime() + start = time.time() + logs_pusher(regionName=region, cluster_name=clust_name, msg="The Addons Upgrade Started At " + str(start_time)) - args='~/eksctl create nodegroup --managed --cluster='+clust_name - output=subprocess.call(args,shell=True) + args = "~/eksctl create nodegroup --managed --cluster=" + clust_name + output = subprocess.call(args, shell=True) - end=time.time() - hours, rem = divmod(end-start, 3600) + end = time.time() + hours, rem = divmod(end - start, 3600) minutes, seconds = divmod(rem, 60) - print("The time Taken For the creation of managed nodegroups ","{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) - logs_pusher(regionName=region,cluster_name=clust_name,msg="The time Taken For the creation of managed nodegroups "+"{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) + print( + "The time Taken For the creation of managed nodegroups ", + "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds), + ) + logs_pusher( + regionName=region, + cluster_name=clust_name, + msg="The time Taken For the creation of managed nodegroups " + + "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds), + ) print(output) -#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -#creates unmanaged nodegroup -def create_unmanaged_nodegroup(region,clust_name,client): - start_time=time.ctime() - start=time.time() - logs_pusher(regionName=region,cluster_name=clust_name,msg="creation of Unmanaged nodegroups Started At "+str(start_time)) - args='~/eksctl create nodegroup --cluster='+clust_name - output=subprocess.call(args,shell=True) +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# creates unmanaged nodegroup +def create_unmanaged_nodegroup(region, clust_name, client): + start_time = time.ctime() + start = time.time() + logs_pusher( + regionName=region, cluster_name=clust_name, msg="creation of Unmanaged nodegroups Started At " + str(start_time) + ) - end=time.time() - hours, rem = divmod(end-start, 3600) - minutes, seconds = divmod(rem, 60) - print("The time Taken For the creation of Unmanaged nodegroups ","{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) - logs_pusher(regionName=region,cluster_name=clust_name,msg="The time Taken For the creation of Unmanaged nodegroups "+"{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) + args = "~/eksctl create nodegroup --cluster=" + clust_name + output = subprocess.call(args, shell=True) + end = time.time() + hours, rem = divmod(end - start, 3600) + minutes, seconds = divmod(rem, 60) + print( + "The time Taken For the creation of Unmanaged nodegroups ", + "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds), + ) + logs_pusher( + regionName=region, + cluster_name=clust_name, + msg="The time Taken For the creation of Unmanaged nodegroups " + + "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds), + ) print(output) -#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -#updates managed existing managed node group -def update_managed_nodegroup(region,clust_name,version,d,bclient): - print('managed node group upgrade started') - start_time=time.ctime() - start=time.time() - logs_pusher(regionName=region,cluster_name=clust_name,msg="Updation of managed nodegroups Started At "+str(start_time)) + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# updates managed existing managed node group +def update_managed_nodegroup(region, clust_name, version, d, bclient): + print("managed node group upgrade started") + start_time = time.ctime() + start = time.time() + logs_pusher( + regionName=region, cluster_name=clust_name, msg="Updation of managed nodegroups Started At " + str(start_time) + ) response = bclient.list_nodegroups( - clusterName=clust_name, + clusterName=clust_name, ) - d['managed_nodegroup_after_update']={} - if(response['nodegroups']!=[ ]): #to verify if MANAGED NODEGROUP exists - for i in response['nodegroups']: - response=bclient.describe_nodegroup(clusterName=clust_name,nodegroupName=i) - print("nodegroup "+i+"version before upgrade is "+response['nodegroup']['version']) - args="~/eksctl upgrade nodegroup --name="+i+" --cluster="+clust_name+' --kubernetes-version='+version #UPDATES MANAGED NODEGROUP - output=subprocess.call(args,shell=True) - response=bclient.describe_nodegroup(clusterName=clust_name,nodegroupName=i) - print("nodegroup "+i+"version after upgrade is "+response['nodegroup']['version']) - d['managed_nodegroup_after_update'][i]=response['nodegroup']['version'] + d["managed_nodegroup_after_update"] = {} + if response["nodegroups"] != []: # to verify if MANAGED NODEGROUP exists + for i in response["nodegroups"]: + response = bclient.describe_nodegroup(clusterName=clust_name, nodegroupName=i) + print("nodegroup " + i + "version before upgrade is " + response["nodegroup"]["version"]) + args = ( + "~/eksctl upgrade nodegroup --name=" + + i + + " --cluster=" + + clust_name + + " --kubernetes-version=" + + version + ) # UPDATES MANAGED NODEGROUP + output = subprocess.call(args, shell=True) + response = bclient.describe_nodegroup(clusterName=clust_name, nodegroupName=i) + print("nodegroup " + i + "version after upgrade is " + response["nodegroup"]["version"]) + d["managed_nodegroup_after_update"][i] = response["nodegroup"]["version"] else: print("no managed nodegroups found") return d - - end=time.time() - hours, rem = divmod(end-start, 3600) + + end = time.time() + hours, rem = divmod(end - start, 3600) minutes, seconds = divmod(rem, 60) - print("The time Taken For the Updation of managed nodegroups ","{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) - logs_pusher(regionName=region,cluster_name=clust_name,msg="The time Taken For the Updation of managed nodegroups "+"{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) + print( + "The time Taken For the Updation of managed nodegroups ", + "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds), + ) + logs_pusher( + regionName=region, + cluster_name=clust_name, + msg="The time Taken For the Updation of managed nodegroups " + + "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds), + ) return d -#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ def eksctl_execute(args): # ~~~~~~~~~~~~~~~~~~~~~~~~~INPUT clust_name = args.name version = args.version - region=args.region - - bclient=botoclient(region) - loading_config(clust_name,region) + region = args.region + + bclient = botoclient(region) + loading_config(clust_name, region) v1 = client.CoreV1Api() - + rep = v1.list_namespaced_pod("kube-system") - d={ } + d = {} # ~~~~~~~~~~~~~~~~~~~~~~~~~UPGRADE - #call to upgrade cluster to latest version - d=upgrade_cluster(region,clust_name,d,bclient,version) + # call to upgrade cluster to latest version + d = upgrade_cluster(region, clust_name, d, bclient, version) # #call to upgrade addons to latest version - d=add_on_upgrade(region,clust_name,d,v1) - #call to update managed nodegroup - d=update_managed_nodegroup(region,clust_name,str(version),d,bclient) - #call to update unmanaged nodegroup b - d=update_unmanaged_nodegroup(region,clust_name,d,bclient) - loading_config(clust_name,region) + d = add_on_upgrade(region, clust_name, d, v1) + # call to update managed nodegroup + d = update_managed_nodegroup(region, clust_name, str(version), d, bclient) + # call to update unmanaged nodegroup b + d = update_unmanaged_nodegroup(region, clust_name, d, bclient) + loading_config(clust_name, region) v1 = client.CoreV1Api() - + rep = v1.list_namespaced_pod("kube-system") - d['addonsafterupdate']={} + d["addonsafterupdate"] = {} for pod in rep.items: - print(pod.metadata.name ,"Current Version = ", pod.spec.containers[0].image.split(":")[-1]) - d['addonsafterupdate'][pod.metadata.name]=pod.spec.containers[0].image.split(":")[-1] - - print(json.dumps(d,indent=4)) - - - - - - - - - - - - - - + print(pod.metadata.name, "Current Version = ", pod.spec.containers[0].image.split(":")[-1]) + d["addonsafterupdate"][pod.metadata.name] = pod.spec.containers[0].image.split(":")[-1] + print(json.dumps(d, indent=4)) diff --git a/eksupdate/src/ekslogs.py b/eksupdate/src/ekslogs.py index aab450e..f853791 100644 --- a/eksupdate/src/ekslogs.py +++ b/eksupdate/src/ekslogs.py @@ -1,7 +1,8 @@ -import boto3 import time -#commented for enhancements +import boto3 + +# commented for enhancements # def log_creator(regionName,cluster_name): # logs = boto3.client('logs',region_name=regionName) # LOG_GROUP = 'cluster-' + cluster_name + '-' + regionName @@ -28,58 +29,49 @@ # ) -def logs_pusher(regionName,cluster_name,msg): +def logs_pusher(regionName, cluster_name, msg): - logs = boto3.client('logs',region_name=regionName) - LOG_GROUP = 'cluster-' + cluster_name + '-' + regionName - LOG_STREAM = cluster_name + '-' + regionName + '-'+'eks-update-logs-streams' - is_exist_group = len(logs.describe_log_groups( - logGroupNamePrefix=LOG_GROUP, - ).get('logGroups')) > 0 + logs = boto3.client("logs", region_name=regionName) + LOG_GROUP = "cluster-" + cluster_name + "-" + regionName + LOG_STREAM = cluster_name + "-" + regionName + "-" + "eks-update-logs-streams" + is_exist_group = ( + len( + logs.describe_log_groups( + logGroupNamePrefix=LOG_GROUP, + ).get("logGroups") + ) + > 0 + ) if not is_exist_group: logs.create_log_group(logGroupName=LOG_GROUP) - is_stream_existing = len(logs.describe_log_streams( - logGroupName=LOG_GROUP, - logStreamNamePrefix=LOG_STREAM - )['logStreams'] - ) > 0 + is_stream_existing = ( + len(logs.describe_log_streams(logGroupName=LOG_GROUP, logStreamNamePrefix=LOG_STREAM)["logStreams"]) > 0 + ) if not is_stream_existing: - logs.create_log_stream(logGroupName=LOG_GROUP, - logStreamName=LOG_STREAM) + logs.create_log_stream(logGroupName=LOG_GROUP, logStreamName=LOG_STREAM) - response =response = logs.describe_log_streams( - logGroupName=LOG_GROUP, - logStreamNamePrefix=LOG_STREAM -) + response = response = logs.describe_log_streams(logGroupName=LOG_GROUP, logStreamNamePrefix=LOG_STREAM) - try: timestamp = int(round(time.time() * 1000)) event_log = { - 'logGroupName': LOG_GROUP, - 'logStreamName': LOG_STREAM, - 'logEvents': [ - { - 'timestamp': timestamp, - 'message': str(msg) - }], } - if 'uploadSequenceToken' in response['logStreams'][0]: - event_log.update({'sequenceToken': response['logStreams'][0] ['uploadSequenceToken']}) + "logGroupName": LOG_GROUP, + "logStreamName": LOG_STREAM, + "logEvents": [{"timestamp": timestamp, "message": str(msg)}], + } + if "uploadSequenceToken" in response["logStreams"][0]: + event_log.update({"sequenceToken": response["logStreams"][0]["uploadSequenceToken"]}) response = logs.put_log_events(**event_log) except Exception as e: timestamp = int(round(time.time() * 1000)) event_log = { - 'logGroupName': LOG_GROUP, - 'logStreamName': LOG_STREAM, - 'logEvents': [ - { - 'timestamp': timestamp, - 'message': str(msg) - }], } - event_log.update({'sequenceToken': str(e).split(" ")[-1]}) + "logGroupName": LOG_GROUP, + "logStreamName": LOG_STREAM, + "logEvents": [{"timestamp": timestamp, "message": str(msg)}], + } + event_log.update({"sequenceToken": str(e).split(" ")[-1]}) response = logs.put_log_events(**event_log) - return diff --git a/eksupdate/src/k8s_client.py b/eksupdate/src/k8s_client.py index def5a52..d247484 100644 --- a/eksupdate/src/k8s_client.py +++ b/eksupdate/src/k8s_client.py @@ -1,121 +1,111 @@ import base64 -import boto3 -import re -from botocore.signers import RequestSigner -from kubernetes import client, config, watch -from kubernetes.client.rest import ApiException -import kubernetes.client import json -import yaml -from .ekslogs import logs_pusher -import time import queue +import re import threading +import time + +import boto3 +import kubernetes.client +import yaml +from botocore.signers import RequestSigner +from kubernetes import client, watch +from kubernetes.client.rest import ApiException + +from .ekslogs import logs_pusher + queue = queue.Queue() -class StatsWorker(threading.Thread): - def __init__(self, queue,id): + +class StatsWorker(threading.Thread): + def __init__(self, queue, id): threading.Thread.__init__(self) self.queue = queue - self.id=id + self.id = id def run(self): while self.queue.not_empty: - cluster_name,nameSpace,new_pod_name, podName,regionName= self.queue.get() - x=addon_status(cluster_name=cluster_name,new_pod_name=new_pod_name,podName=podName,regionName=regionName,nameSpace=nameSpace) - # signals to queue job is done + cluster_name, nameSpace, new_pod_name, podName, regionName = self.queue.get() + x = addon_status( + cluster_name=cluster_name, + new_pod_name=new_pod_name, + podName=podName, + regionName=regionName, + nameSpace=nameSpace, + ) + # signals to queue job is done if not x: - raise Exception("Pod Not Started",new_pod_name) + raise Exception("Pod Not Started", new_pod_name) self.queue.task_done() + + def get_bearer_token(cluster_id, region): - '''' AUthenticating the session with sts token''' + """' AUthenticating the session with sts token""" STS_TOKEN_EXPIRES_IN = 60 session = boto3.session.Session() - client = session.client('sts', region_name=region) + client = session.client("sts", region_name=region) service_id = client.meta.service_model.service_id - signer = RequestSigner( - service_id, - region, - 'sts', - 'v4', - session.get_credentials(), - session.events - ) + signer = RequestSigner(service_id, region, "sts", "v4", session.get_credentials(), session.events) params = { - 'method': 'GET', - 'url': 'https://sts.{}.amazonaws.com/?Action=GetCallerIdentity&Version=2011-06-15'.format(region), - 'body': {}, - 'headers': { - 'x-k8s-aws-id': cluster_id - }, - 'context': {} + "method": "GET", + "url": "https://sts.{}.amazonaws.com/?Action=GetCallerIdentity&Version=2011-06-15".format(region), + "body": {}, + "headers": {"x-k8s-aws-id": cluster_id}, + "context": {}, } - '''Getting a presigned Url''' + """Getting a presigned Url""" signed_url = signer.generate_presigned_url( - params, - region_name=region, - expires_in=STS_TOKEN_EXPIRES_IN, - operation_name='' + params, region_name=region, expires_in=STS_TOKEN_EXPIRES_IN, operation_name="" ) - base64_url = base64.urlsafe_b64encode( - signed_url.encode('utf-8')).decode('utf-8') + base64_url = base64.urlsafe_b64encode(signed_url.encode("utf-8")).decode("utf-8") # remove any base64 encoding padding and returing the kubernets token - return 'k8s-aws-v1.' + re.sub(r'=*', '', base64_url) + return "k8s-aws-v1." + re.sub(r"=*", "", base64_url) -def loading_config(cluster_name,regionName): - ''' loading kubeconfig with sts''' - eks = boto3.client('eks',region_name=regionName) - resp = eks.describe_cluster( - - name=cluster_name - ) +def loading_config(cluster_name, regionName): + """loading kubeconfig with sts""" + eks = boto3.client("eks", region_name=regionName) + resp = eks.describe_cluster(name=cluster_name) endPoint = resp["cluster"]["endpoint"] configs = client.Configuration() configs.host = endPoint configs.verify_ssl = False - configs.debug = False - configs.api_key = {"authorization": "Bearer " + - get_bearer_token(cluster_name, regionName)} + configs.debug = False + configs.api_key = {"authorization": "Bearer " + get_bearer_token(cluster_name, regionName)} client.Configuration.set_default(configs) return "Initialiazed" -def unschedule_old_nodes(ClusterName, Nodename,regionName): - loading_config(ClusterName,regionName) - ''' unsheduling the nodes to avaoid new nodes to be launched ''' +def unschedule_old_nodes(ClusterName, Nodename, regionName): + loading_config(ClusterName, regionName) + """ unsheduling the nodes to avaoid new nodes to be launched """ try: v1 = client.CoreV1Api() - ''' unscheduling the nodes''' - body = { - "spec": { - "unschedulable": True - } - } + """ unscheduling the nodes""" + body = {"spec": {"unschedulable": True}} v1.patch_node(Nodename, body) return except Exception as e: raise Exception(str(e)) - -def watcher(cluster_name, name,regionName): - ''' watcher to check wether the pod is deleted or not''' - loading_config(cluster_name=cluster_name,regionName=regionName) +def watcher(cluster_name, name, regionName): + """watcher to check wether the pod is deleted or not""" + loading_config(cluster_name=cluster_name, regionName=regionName) v1 = client.CoreV1Api() w = watch.Watch() try: for event in w.stream(v1.list_pod_for_all_namespaces, timeout_seconds=30): - print(event['type'], event['object'].metadata.name) - if event['type'] == "DELETED" and event['object'].metadata.name == name: + print(event["type"], event["object"].metadata.name) + if event["type"] == "DELETED" and event["object"].metadata.name == name: w.stop() return True else: @@ -125,48 +115,48 @@ def watcher(cluster_name, name,regionName): raise Exception(e) -def drain_nodes(cluster_name, Nodename,forced,regionName): - ''' pod eviction using eviction api ''' - loading_config(cluster_name,regionName) +def drain_nodes(cluster_name, Nodename, forced, regionName): + """pod eviction using eviction api""" + loading_config(cluster_name, regionName) v1 = client.CoreV1Api() - api_response = v1.list_pod_for_all_namespaces(watch=False, field_selector='spec.nodeName=' + Nodename) + api_response = v1.list_pod_for_all_namespaces(watch=False, field_selector="spec.nodeName=" + Nodename) retry = 0 # print(api_response.items) if len(api_response.items) == 0: - return "Empty Nothing to Drain"+Nodename + return "Empty Nothing to Drain" + Nodename for i in api_response.items: if i.spec.node_name == Nodename: try: if forced: - v1.delete_namespaced_pod(i.metadata.name,i.metadata.namespace,grace_period_seconds=0,body=client.V1DeleteOptions()) + v1.delete_namespaced_pod( + i.metadata.name, i.metadata.namespace, grace_period_seconds=0, body=client.V1DeleteOptions() + ) else: eviction_body = kubernetes.client.models.v1beta1_eviction.V1beta1Eviction( - metadata=kubernetes.client.V1ObjectMeta(name=i.metadata.name, namespace=i.metadata.namespace)) - v1.create_namespaced_pod_eviction(name=i.metadata.name, - - namespace=i.metadata.namespace, body=eviction_body) - '''retry to if pod is not deleted with eviction api''' + metadata=kubernetes.client.V1ObjectMeta(name=i.metadata.name, namespace=i.metadata.namespace) + ) + v1.create_namespaced_pod_eviction( + name=i.metadata.name, namespace=i.metadata.namespace, body=eviction_body + ) + """retry to if pod is not deleted with eviction api""" if watcher(cluster_name, i.metadata.name) == False and retry < 2: drain_nodes(cluster_name, i.metadata.name) retry += 1 if retry == 2: - raise Exception( - "Error Not able to delete the Node"+i.metadata.name) + raise Exception("Error Not able to delete the Node" + i.metadata.name) return - except Exception as e: print(e) # logs_pusher(e, "error") raise Exception("Unable to Delete the Node") - -def delete_node(cluster_name, NodeName,regionName): +def delete_node(cluster_name, NodeName, regionName): try: - loading_config(cluster_name,regionName) + loading_config(cluster_name, regionName) v1 = client.CoreV1Api() - ''' delete the node from compute list this doesnt terminate the instance''' + """ delete the node from compute list this doesnt terminate the instance""" v1.delete_node(NodeName) return except ApiException as e: @@ -175,28 +165,35 @@ def delete_node(cluster_name, NodeName,regionName): raise Exception(e) -def find_node(cluster_name, instance_id,op,regionName): - ''' finding the node with instance id''' - loading_config(cluster_name,regionName) +def find_node(cluster_name, instance_id, op, regionName): + """finding the node with instance id""" + loading_config(cluster_name, regionName) v1 = client.CoreV1Api() nodes = [] response = v1.list_node() - if len(response.items)==0: + if len(response.items) == 0: return "NAN" for node in response.items: # print(node.spec.provider_id, node.metadata.name, # node.status.node_info.kube_proxy_version, node.status.node_info.kubelet_version) # print(node.status.node_info.os_image) - nodes.append([node.spec.provider_id.split("/")[-1], node.metadata.name, - node.status.node_info.kube_proxy_version.split("-")[0], node.status.node_info.kubelet_version.split("-")[0],node.status.node_info.os_image]) - if op=="find": + nodes.append( + [ + node.spec.provider_id.split("/")[-1], + node.metadata.name, + node.status.node_info.kube_proxy_version.split("-")[0], + node.status.node_info.kubelet_version.split("-")[0], + node.status.node_info.os_image, + ] + ) + if op == "find": for i in nodes: if i[0] == instance_id: return i[1] return "NAN" - if op=="os_type": + if op == "os_type": for i in nodes: - if i[0]==instance_id: + if i[0] == instance_id: print(i[0]) return i[-1] return "NAN" @@ -207,161 +204,259 @@ def find_node(cluster_name, instance_id,op,regionName): # return x -def addon_status(cluster_name, new_pod_name,podName,regionName,nameSpace): - loading_config(cluster_name,regionName) +def addon_status(cluster_name, new_pod_name, podName, regionName, nameSpace): + loading_config(cluster_name, regionName) v1 = client.CoreV1Api() - tts= 100 + tts = 100 now = time.time() v1 = client.CoreV1Api() - while (time.time() < now+tts): - response=v1.read_namespaced_pod_status(name=new_pod_name,namespace=nameSpace) + while time.time() < now + tts: + response = v1.read_namespaced_pod_status(name=new_pod_name, namespace=nameSpace) if response.status.container_statuses[0].ready and response.status.container_statuses[0].started: return True - - return False -def sort_pods(cluster_name,regionName,original_name,pod_name,old_pods_name,nameSpace,c=90): - if c ==0: + return False + + +def sort_pods(cluster_name, regionName, original_name, pod_name, old_pods_name, nameSpace, c=90): + if c == 0: raise Exception("Pod has No assosicated New Launch") - pods_nodes=[] - loading_config(cluster_name,regionName) + pods_nodes = [] + loading_config(cluster_name, regionName) v1 = client.CoreV1Api() try: - if pod_name=="cluster-autoscaler": - pod_list=v1.list_namespaced_pod(namespace=nameSpace, label_selector='app={name}'.format(name=pod_name)) + if pod_name == "cluster-autoscaler": + pod_list = v1.list_namespaced_pod(namespace=nameSpace, label_selector="app={name}".format(name=pod_name)) else: - pod_list=v1.list_namespaced_pod(namespace=nameSpace, label_selector='k8s-app={name}'.format(name=pod_name)) - + pod_list = v1.list_namespaced_pod( + namespace=nameSpace, label_selector="k8s-app={name}".format(name=pod_name) + ) + except Exception as e: - logs_pusher(regionName,cluster_name,e) + logs_pusher(regionName, cluster_name, e) return "Not Found" - print("Total Pods With {p} = {c}".format(p=pod_name,c=len(pod_list.items))) + print("Total Pods With {p} = {c}".format(p=pod_name, c=len(pod_list.items))) for i in pod_list.items: - pods_nodes.append([i.metadata.name,i.metadata.creation_timestamp]) - if len(pods_nodes)>0: - new_pod_name=sorted(pods_nodes,key=lambda x: x[1])[-1][0] + pods_nodes.append([i.metadata.name, i.metadata.creation_timestamp]) + if len(pods_nodes) > 0: + new_pod_name = sorted(pods_nodes, key=lambda x: x[1])[-1][0] else: - c-=1 - sort_pods(cluster_name,regionName,original_name,pod_name,old_pods_name,nameSpace,c) - ''' aws-node not in aws-node-hshsh ''' + c -= 1 + sort_pods(cluster_name, regionName, original_name, pod_name, old_pods_name, nameSpace, c) + """ aws-node not in aws-node-hshsh """ if original_name != new_pod_name and new_pod_name in old_pods_name: - c-=1 - sort_pods(cluster_name,regionName,original_name,pod_name,old_pods_name,nameSpace,c) + c -= 1 + sort_pods(cluster_name, regionName, original_name, pod_name, old_pods_name, nameSpace, c) return new_pod_name -def update_addons(cluster_name, version,vpcPass,regionName): - loading_config(cluster_name,regionName) + +def update_addons(cluster_name, version, vpcPass, regionName): + loading_config(cluster_name, regionName) for x in range(20): - worker = StatsWorker(queue,x) + worker = StatsWorker(queue, x) worker.setDaemon(True) worker.start() v1 = client.CoreV1Api() api_instance = client.AppsV1Api() rep = v1.list_namespaced_pod("kube-system") - add_on_dict = open('eksupdate/src/S3Files/version_dict.json','r') - add_on_dict=json.load(add_on_dict) - old_pods_name=[] + add_on_dict = open("eksupdate/src/S3Files/version_dict.json", "r") + add_on_dict = json.load(add_on_dict) + old_pods_name = [] for pod in rep.items: old_pods_name.append(pod.metadata.name) - print("The Addons Found = ",*old_pods_name) - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg="The Addons Found = {instan}".format(instan=old_pods_name)) + print("The Addons Found = ", *old_pods_name) + logs_pusher( + regionName=regionName, cluster_name=cluster_name, msg="The Addons Found = {instan}".format(instan=old_pods_name) + ) - flag_vpc,flag_core,flag_proxy,flag_scaler=True,True,True,True + flag_vpc, flag_core, flag_proxy, flag_scaler = True, True, True, True try: for pod in rep.items: images = [c.image for c in pod.spec.containers] # logs_pusher(images, "info") image = "".join(images) - coredns_new=add_on_dict[version].get('coredns') - kubeproxy_new=add_on_dict[version].get('kube-proxy') - autosclaer_new=add_on_dict[version].get('cluster-autoscaler') - cni_new=add_on_dict[version].get('vpc-cni') - vv=int("".join(image.split(':')[-1].replace("v","").replace("-",".").split(".")[:3])) - newv=version.replace(".","") - newv=int(newv) - - if "coredns" in pod.metadata.name and image.split(":")[-1] != "v"+coredns_new+"-eksbuild.1": - print(pod.metadata.name ,"Current Version = ", image.split(":")[-1],"Updating To = " ,"v"+coredns_new+"-eksbuild.1") - body = {"spec": {"template": {"spec": {"containers": [ - {"name": "coredns", "image": image.split(":")[0]+":v"+coredns_new+"-eksbuild.1"}]}}}} + coredns_new = add_on_dict[version].get("coredns") + kubeproxy_new = add_on_dict[version].get("kube-proxy") + autosclaer_new = add_on_dict[version].get("cluster-autoscaler") + cni_new = add_on_dict[version].get("vpc-cni") + vv = int("".join(image.split(":")[-1].replace("v", "").replace("-", ".").split(".")[:3])) + newv = version.replace(".", "") + newv = int(newv) + + if "coredns" in pod.metadata.name and image.split(":")[-1] != "v" + coredns_new + "-eksbuild.1": + print( + pod.metadata.name, + "Current Version = ", + image.split(":")[-1], + "Updating To = ", + "v" + coredns_new + "-eksbuild.1", + ) + body = { + "spec": { + "template": { + "spec": { + "containers": [ + { + "name": "coredns", + "image": image.split(":")[0] + ":v" + coredns_new + "-eksbuild.1", + } + ] + } + } + } + } if flag_core: api_response = api_instance.patch_namespaced_deployment( - name="coredns", namespace='kube-system', body=body, pretty=True) - if vv <=170: - coredns_yaml = open("eksupdate/src/S3Files/core-dns.yaml", 'r') - body= yaml.safe_load(coredns_yaml) - v1.patch_namespaced_config_map(name="coredns",namespace="kube-system",body=body) - flag_core=False + name="coredns", namespace="kube-system", body=body, pretty=True + ) + if vv <= 170: + coredns_yaml = open("eksupdate/src/S3Files/core-dns.yaml", "r") + body = yaml.safe_load(coredns_yaml) + v1.patch_namespaced_config_map(name="coredns", namespace="kube-system", body=body) + flag_core = False time.sleep(20) - new_pod_name=sort_pods(cluster_name=cluster_name,regionName=regionName,original_name=pod.metadata.name,old_pods_name=old_pods_name,pod_name="kube-dns",nameSpace="kube-system") - print("old CoreDNs Pod {oldp} \t new CoreDnsPod {newp}".format(oldp=pod.metadata.name,newp=new_pod_name)) - queue.put([cluster_name,'kube-system',new_pod_name, "coredns",regionName]) + new_pod_name = sort_pods( + cluster_name=cluster_name, + regionName=regionName, + original_name=pod.metadata.name, + old_pods_name=old_pods_name, + pod_name="kube-dns", + nameSpace="kube-system", + ) + print( + "old CoreDNs Pod {oldp} \t new CoreDnsPod {newp}".format(oldp=pod.metadata.name, newp=new_pod_name) + ) + queue.put([cluster_name, "kube-system", new_pod_name, "coredns", regionName]) # if addon_status(cluster_name=cluster_name,nameSpace="kube-system",new_pod_name=new_pod_name,podName=pod.metadata.name,regionName=regionName) != True: # raise Exception("Pod Is not Started"+pod.metadata.name) elif "kube-proxy" in pod.metadata.name: - if newv<=118: - final_ender="-eksbuild.1" + if newv <= 118: + final_ender = "-eksbuild.1" else: - final_ender="-eksbuild.2" - - if image.split(":")[-1] != "v"+kubeproxy_new+final_ender: - print(pod.metadata.name ,"Current Version = ", image.split(":")[-1],"Updating To = " ,"v"+kubeproxy_new+final_ender) - body = {"spec": {"template": {"spec": {"containers": [ - {"name": "kube-proxy", "image": image.split(":")[0]+":v"+kubeproxy_new+final_ender}]}}}} + final_ender = "-eksbuild.2" + + if image.split(":")[-1] != "v" + kubeproxy_new + final_ender: + print( + pod.metadata.name, + "Current Version = ", + image.split(":")[-1], + "Updating To = ", + "v" + kubeproxy_new + final_ender, + ) + body = { + "spec": { + "template": { + "spec": { + "containers": [ + { + "name": "kube-proxy", + "image": image.split(":")[0] + ":v" + kubeproxy_new + final_ender, + } + ] + } + } + } + } if flag_proxy: api_response = api_instance.patch_namespaced_daemon_set( - name="kube-proxy", namespace='kube-system', body=body, pretty=True) - flag_proxy=False + name="kube-proxy", namespace="kube-system", body=body, pretty=True + ) + flag_proxy = False time.sleep(20) - new_pod_name=sort_pods(cluster_name=cluster_name,regionName=regionName,original_name=pod.metadata.name,old_pods_name=old_pods_name,pod_name="kube-proxy",nameSpace="kube-system") - - print("old KubProxy Pod {oldp} \t new KubeProxyPod {newp}".format(oldp=pod.metadata.name,newp=new_pod_name)) - queue.put([cluster_name,'kube-system',new_pod_name, "kube-proxy",regionName]) + new_pod_name = sort_pods( + cluster_name=cluster_name, + regionName=regionName, + original_name=pod.metadata.name, + old_pods_name=old_pods_name, + pod_name="kube-proxy", + nameSpace="kube-system", + ) + + print( + "old KubProxy Pod {oldp} \t new KubeProxyPod {newp}".format( + oldp=pod.metadata.name, newp=new_pod_name + ) + ) + queue.put([cluster_name, "kube-system", new_pod_name, "kube-proxy", regionName]) # if addon_status(cluster_name=cluster_name,nameSpace='kube-system',new_pod_name=new_pod_name, podName=pod.metadata.name,regionName=regionName) != True: # raise Exception("Pod is Not Started"+pod.metadata.name) - elif "cluster-autoscaler" in pod.metadata.name and image.split(":")[-1] != "v"+autosclaer_new: - print(pod.metadata.name ,"Current Version = ", image.split(":")[-1],"Updating To = " ,"v"+autosclaer_new+"-eksbuild.1") - body = {"spec": {"template": {"spec": {"containers": [ - {"name": "cluster-autoscaler", "image":image.split(":")[0]+":v"+autosclaer_new}]}}}} + elif "cluster-autoscaler" in pod.metadata.name and image.split(":")[-1] != "v" + autosclaer_new: + print( + pod.metadata.name, + "Current Version = ", + image.split(":")[-1], + "Updating To = ", + "v" + autosclaer_new + "-eksbuild.1", + ) + body = { + "spec": { + "template": { + "spec": { + "containers": [ + {"name": "cluster-autoscaler", "image": image.split(":")[0] + ":v" + autosclaer_new} + ] + } + } + } + } if flag_scaler: api_response = api_instance.patch_namespaced_deployment( - name="cluster-autoscaler", namespace='kube-system', body=body, pretty=True) - flag_scaler=False + name="cluster-autoscaler", namespace="kube-system", body=body, pretty=True + ) + flag_scaler = False time.sleep(20) - new_pod_name=sort_pods(cluster_name=cluster_name,regionName=regionName,original_name=pod.metadata.name,old_pods_name=old_pods_name,pod_name="cluster-autoscaler",nameSpace="kube-system") - print("old Cluster AutoScaler Pod {oldp} \t new AutoScaler pod {newp}".format(oldp=pod.metadata.name,newp=new_pod_name)) - queue.put([cluster_name,'kube-system',new_pod_name, "cluster-autoscaler",regionName]) + new_pod_name = sort_pods( + cluster_name=cluster_name, + regionName=regionName, + original_name=pod.metadata.name, + old_pods_name=old_pods_name, + pod_name="cluster-autoscaler", + nameSpace="kube-system", + ) + print( + "old Cluster AutoScaler Pod {oldp} \t new AutoScaler pod {newp}".format( + oldp=pod.metadata.name, newp=new_pod_name + ) + ) + queue.put([cluster_name, "kube-system", new_pod_name, "cluster-autoscaler", regionName]) # if addon_status(cluster_name=cluster_name,nameSpace='kube-system',new_pod_name=new_pod_name, podName=pod.metadata.name,regionName=regionName) != True: # raise Exception("Pod is Not Started"+pod.metadata.name) - elif "aws-node" in pod.metadata.name and image.split(":")[-1] !="v"+cni_new and not vpcPass: - print(pod.metadata.name ,"Current Version = ", image.split(":")[-1],"Updating To = " ,"v"+cni_new) + elif "aws-node" in pod.metadata.name and image.split(":")[-1] != "v" + cni_new and not vpcPass: + print(pod.metadata.name, "Current Version = ", image.split(":")[-1], "Updating To = ", "v" + cni_new) if flag_vpc: - vpc_cni_yaml = open("eksupdate/src/S3Files/vpc-cni.yaml", 'r') - body= yaml.safe_load(vpc_cni_yaml) - body['spec']['template']['spec']['containers'][0]['image']=image.split(":")[0]+":v"+cni_new - old=body['spec']['template']['spec']['initContainers'][0]['image'] - body['spec']['template']['spec']['initContainers'][0]['image']=old.split(":")[0]+":v"+cni_new - api_response = api_instance.patch_namespaced_daemon_set(namespace='kube-system', - name="aws-node", body=body, pretty=True) - flag_vpc=False + vpc_cni_yaml = open("eksupdate/src/S3Files/vpc-cni.yaml", "r") + body = yaml.safe_load(vpc_cni_yaml) + body["spec"]["template"]["spec"]["containers"][0]["image"] = image.split(":")[0] + ":v" + cni_new + old = body["spec"]["template"]["spec"]["initContainers"][0]["image"] + body["spec"]["template"]["spec"]["initContainers"][0]["image"] = old.split(":")[0] + ":v" + cni_new + api_response = api_instance.patch_namespaced_daemon_set( + namespace="kube-system", name="aws-node", body=body, pretty=True + ) + flag_vpc = False time.sleep(20) - new_pod_name=sort_pods(cluster_name=cluster_name,regionName=regionName,original_name=pod.metadata.name,old_pods_name=old_pods_name,pod_name="aws-node",nameSpace="kube-system") - print("old vpc cni Pod {oldp} \t new vpc cni {newp}".format(oldp=pod.metadata.name,newp=new_pod_name)) - queue.put([cluster_name,'kube-system',new_pod_name, "aws-node",regionName]) + new_pod_name = sort_pods( + cluster_name=cluster_name, + regionName=regionName, + original_name=pod.metadata.name, + old_pods_name=old_pods_name, + pod_name="aws-node", + nameSpace="kube-system", + ) + print("old vpc cni Pod {oldp} \t new vpc cni {newp}".format(oldp=pod.metadata.name, newp=new_pod_name)) + queue.put([cluster_name, "kube-system", new_pod_name, "aws-node", regionName]) queue.join() - # if addon_status(cluster_name=cluster_name,nameSpace='kube-system',new_pod_name=new_pod_name, podName="aws-node",regionName=regionName) != True: - # raise Exception("Pod is Not Started"+pod.metadata.name) + # if addon_status(cluster_name=cluster_name,nameSpace='kube-system',new_pod_name=new_pod_name, podName="aws-node",regionName=regionName) != True: + # raise Exception("Pod is Not Started"+pod.metadata.name) except Exception as e: print(e) raise Exception(e) - # print(list_nodes()) # print(update_addons()) def list_pods(ClusterName): @@ -370,8 +465,7 @@ def list_pods(ClusterName): print("Listing pods with their IPs:") ret = v1.list_namespaced_pod("default") for i in ret.items: - print("%s\t%s\t%s" % - (i.status.pod_ip, i.metadata.namespace, i.metadata.name)) + print("%s\t%s\t%s" % (i.status.pod_ip, i.metadata.namespace, i.metadata.name)) # def pd_bd(): @@ -404,40 +498,37 @@ def list_pods(ClusterName): def delete_pd_policy(pd_name): api_cli = client.PolicyV1beta1Api() try: - api_response = api_cli.delete_namespaced_pod_disruption_budget( - name=pd_name, namespace="default") + api_response = api_cli.delete_namespaced_pod_disruption_budget(name=pd_name, namespace="default") print(api_response) except ApiException as e: print("Exception when calling PolicyV1beta1Api->delete_namespaced_pod_disruption_budget: %s\n" % e) + # addon_status(cluster_name="Prod",podName="coredns-6b4cdc67b4-4cfjs") -def is_cluster_auto_scaler_present(ClusterName,regionName): - loading_config(cluster_name=ClusterName,regionName=regionName) - v1=client.AppsV1Api() - res=v1.list_deployment_for_all_namespaces() + +def is_cluster_auto_scaler_present(ClusterName, regionName): + loading_config(cluster_name=ClusterName, regionName=regionName) + v1 = client.AppsV1Api() + res = v1.list_deployment_for_all_namespaces() for res_i in res.items: - if res_i.metadata.name=="cluster-autoscaler": - return [True,res_i.spec.replicas] - return [False,"NAN"] + if res_i.metadata.name == "cluster-autoscaler": + return [True, res_i.spec.replicas] + return [False, "NAN"] + -def clus_auto_enable_disable(ClusterName,type,mx_val,regionName): - loading_config(cluster_name=ClusterName,regionName=regionName) +def clus_auto_enable_disable(ClusterName, type, mx_val, regionName): + loading_config(cluster_name=ClusterName, regionName=regionName) api = client.AppsV1Api() - v1=client.CoreV1Api() - if type=="pause": - body = {'spec': {'replicas': 0}} - elif type=="start": - body = {'spec': {'replicas': mx_val }} + v1 = client.CoreV1Api() + if type == "pause": + body = {"spec": {"replicas": 0}} + elif type == "start": + body = {"spec": {"replicas": mx_val}} else: return "error" try: - api.patch_namespaced_deployment( - name="cluster-autoscaler", - namespace="kube-system", - body=body - ) + api.patch_namespaced_deployment(name="cluster-autoscaler", namespace="kube-system", body=body) except Exception as e: print(e) raise Exception(e) - diff --git a/eksupdate/src/latest_ami.py b/eksupdate/src/latest_ami.py index dcd5dff..45971f3 100644 --- a/eksupdate/src/latest_ami.py +++ b/eksupdate/src/latest_ami.py @@ -1,36 +1,37 @@ - import boto3 -from botocore import client -from .ekslogs import logs_pusher -def get_latestami(clustVersion,instancetype,image_to_search,region_Name): - ssm = boto3.client('ssm',region_name=region_Name) - client=boto3.client('ec2',region_name=region_Name) + + +def get_latestami(clustVersion, instancetype, image_to_search, region_Name): + ssm = boto3.client("ssm", region_name=region_Name) + client = boto3.client("ec2", region_name=region_Name) if "Amazon Linux 2" in instancetype: - names=[ - '/aws/service/eks/optimized-ami/{version}/amazon-linux-2/recommended/image_id'.format(version=clustVersion), + names = [ + "/aws/service/eks/optimized-ami/{version}/amazon-linux-2/recommended/image_id".format(version=clustVersion), ] elif "Windows" in instancetype: - names=[ - '/aws/service/ami-windows-latest/{image_to_search}-{version}/image_id'.format(image_to_search=image_to_search,version=clustVersion) + names = [ + "/aws/service/ami-windows-latest/{image_to_search}-{version}/image_id".format( + image_to_search=image_to_search, version=clustVersion + ) ] elif "bottlerocket" in instancetype.lower(): - names=[ - '/aws/service/bottlerocket/aws-k8s-{version}/x86_64/latest/image_id'.format(version=clustVersion) - ] + names = ["/aws/service/bottlerocket/aws-k8s-{version}/x86_64/latest/image_id".format(version=clustVersion)] elif "Ubuntu" in instancetype: - filters = [{'Name':'owner-id','Values':['099720109477']},{'Name': 'name', 'Values': ["ubuntu-eks/k8s_{version}*".format(version=clustVersion)]},{'Name':'is-public','Values':['true']}] - response = client.describe_images( - Filters=filters - ) - x=sorted(response['Images'],key=lambda x:x['CreationDate'],reverse=True) - if len(x)>0: - return x[0].get('ImageId') + filters = [ + {"Name": "owner-id", "Values": ["099720109477"]}, + {"Name": "name", "Values": ["ubuntu-eks/k8s_{version}*".format(version=clustVersion)]}, + {"Name": "is-public", "Values": ["true"]}, + ] + response = client.describe_images(Filters=filters) + x = sorted(response["Images"], key=lambda x: x["CreationDate"], reverse=True) + if len(x) > 0: + return x[0].get("ImageId") else: raise Exception("Couldn't Find Latest Image Retry The Script") else: return "NAN" response = ssm.get_parameters(Names=names) - if len(response.get('Parameters'))>0: - return response.get('Parameters')[0]['Value'] + if len(response.get("Parameters")) > 0: + return response.get("Parameters")[0]["Value"] else: raise Exception("Couldn't Find Latest Image Retry The Script") diff --git a/eksupdate/src/preflight_module.py b/eksupdate/src/preflight_module.py index e8f312c..eb42f7a 100644 --- a/eksupdate/src/preflight_module.py +++ b/eksupdate/src/preflight_module.py @@ -1,352 +1,460 @@ -#import libraries - import json -from platform import node -import yaml -from kubernetes import client, config -from kubernetes.client import * -import boto3 -from pprint import pprint -import base64 -import logging -import argparse -import os.path -import re -from botocore.signers import RequestSigner -import kubernetes as k8s -from kubernetes.client.rest import ApiException -import sys import time +from pprint import pprint + +import boto3 import urllib3 +import yaml +from kubernetes import client +from kubernetes.client import * + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) from .k8s_client import loading_config - # Function to upload logs onto cloud watch -def log_pusher(log_details,cluster_name,region,msg): + +def log_pusher(log_details, cluster_name, region, msg): timestamp = int(round(time.time() * 1000)) - logs = boto3.client('logs', region_name=region) - log_group = log_details['group'] - log_stream = log_details['stream'] + logs = boto3.client("logs", region_name=region) + log_group = log_details["group"] + log_stream = log_details["stream"] timestamp = int(round(time.time() * 1000)) - response = logs.describe_log_streams(logGroupName=log_group,logStreamNamePrefix=log_stream) - response = logs.put_log_events(logGroupName = log_group,logStreamName=log_stream,logEvents = [{'timestamp':timestamp,'message':msg}], sequenceToken=response['logStreams'][0]['uploadSequenceToken']) + response = logs.describe_log_streams(logGroupName=log_group, logStreamNamePrefix=log_stream) + response = logs.put_log_events( + logGroupName=log_group, + logStreamName=log_stream, + logEvents=[{"timestamp": timestamp, "message": msg}], + sequenceToken=response["logStreams"][0]["uploadSequenceToken"], + ) -def create_log_group_stream(cluster_name,region) : + +def create_log_group_stream(cluster_name, region): timestamp = int(round(time.time() * 1000)) - logs = boto3.client('logs', region_name=region) - log_group = 'cluster-' + cluster_name + '-' + region - - log_stream = 'preflight-checks-' + str(timestamp) - if len(logs.describe_log_groups(logGroupNamePrefix=log_group)['logGroups']) > 0 : - print('Log group exists') - else : - logs.create_log_group(logGroupName = log_group) - if len(logs.describe_log_streams(logGroupName=log_group,logStreamNamePrefix=log_stream)['logStreams']) > 0 : - print('Stream exists') - else : - logs.create_log_stream(logGroupName=log_group,logStreamName=log_stream) + logs = boto3.client("logs", region_name=region) + log_group = "cluster-" + cluster_name + "-" + region + + log_stream = "preflight-checks-" + str(timestamp) + if len(logs.describe_log_groups(logGroupNamePrefix=log_group)["logGroups"]) > 0: + print("Log group exists") + else: + logs.create_log_group(logGroupName=log_group) + if len(logs.describe_log_streams(logGroupName=log_group, logStreamNamePrefix=log_stream)["logStreams"]) > 0: + print("Stream exists") + else: + logs.create_log_stream(logGroupName=log_group, logStreamName=log_stream) timestamp = int(round(time.time() * 1000)) - response = logs.put_log_events(logGroupName = log_group,logStreamName=log_stream,logEvents = [{'timestamp':timestamp,'message':'Log stream create at ' + str(timestamp)}]) - print('Check logs in cloud watch group ' + log_group +' for more information' + 'in stream ' + log_stream) - return {'group' : log_group,'stream' : log_stream} + response = logs.put_log_events( + logGroupName=log_group, + logStreamName=log_stream, + logEvents=[{"timestamp": timestamp, "message": "Log stream create at " + str(timestamp)}], + ) + print("Check logs in cloud watch group " + log_group + " for more information" + "in stream " + log_stream) + return {"group": log_group, "stream": log_stream} + # Function declaration for pre flight checks # Verify IAM role for the Input -def pre_flight_checks(preflight,cluster_name,region,pass_vpc,update_version = False,email = False,force_upgrade=False): - loading_config(cluster_name,region) - report = {'preflight_status' : True} + +def pre_flight_checks( + preflight, cluster_name, region, pass_vpc, update_version=False, email=False, force_upgrade=False +): + loading_config(cluster_name, region) + report = {"preflight_status": True} customer_report = {} - log_details = create_log_group_stream(cluster_name,region) + log_details = create_log_group_stream(cluster_name, region) errors = [] - try : - if email : + try: + if email: ses_client = boto3.client("ses", region_name="ap-south-1") - identities = ses_client.list_identities() - if email not in identities['Identities']: - response = ses_client.verify_email_identity( - EmailAddress=email - ) - print('Please check your inbox to verify your email') + identities = ses_client.list_identities() + if email not in identities["Identities"]: + response = ses_client.verify_email_identity(EmailAddress=email) + print("Please check your inbox to verify your email") v1 = client.CoreV1Api() - ret = v1.list_namespaced_service('default') - print('\n') - log_pusher(log_details,cluster_name,region,'Verifying User IAM Role....') - print('Verifying User IAM Role....') - log_pusher(log_details,cluster_name,region,'IAM role for user verified') - print('IAM role for user verified') - customer_report['IAM role'] = 'IAM role for user verified' - get_cluster_version(preflight,log_details,errors,cluster_name,region,pass_vpc,update_version,report,customer_report,email,force_upgrade) - print('\n') - log_pusher(log_details,cluster_name,region,'Customer report.....') - log_pusher(log_details,cluster_name,region,str(report)) - if (len(errors)) > 0 : - if preflight : - print('Preflight unsuccessful because of following errors') + ret = v1.list_namespaced_service("default") + print("\n") + log_pusher(log_details, cluster_name, region, "Verifying User IAM Role....") + print("Verifying User IAM Role....") + log_pusher(log_details, cluster_name, region, "IAM role for user verified") + print("IAM role for user verified") + customer_report["IAM role"] = "IAM role for user verified" + get_cluster_version( + preflight, + log_details, + errors, + cluster_name, + region, + pass_vpc, + update_version, + report, + customer_report, + email, + force_upgrade, + ) + print("\n") + log_pusher(log_details, cluster_name, region, "Customer report.....") + log_pusher(log_details, cluster_name, region, str(report)) + if (len(errors)) > 0: + if preflight: + print("Preflight unsuccessful because of following errors") else: - print('Postflight unsuccessful because of following errors') - for e in errors : + print("Postflight unsuccessful because of following errors") + for e in errors: print(e) - print('\n') + print("\n") # print('Customer report.....') # pprint(customer_report) - return report['preflight_status'] - except Exception as e : - log_pusher(log_details,cluster_name,region,'IAM role verification failed {err}'.format(err=e)) - print('IAM role verification failed {err}'.format(err=e)) - report['preflight_status'] = False - customer_report['IAM role'] = 'IAM role verification failed' - + return report["preflight_status"] + except Exception as e: + log_pusher(log_details, cluster_name, region, "IAM role verification failed {err}".format(err=e)) + print("IAM role verification failed {err}".format(err=e)) + report["preflight_status"] = False + customer_report["IAM role"] = "IAM role verification failed" # Control Plane version listing -def get_cluster_version(preflight,log_details,errors,cluster_name,region,pass_vpc,update_version,report,customer_report,email,force_upgrade): - loading_config(cluster_name,region) - print('\n') - log_pusher(log_details,cluster_name,region,'Fetching cluster details .....') - print('Fetching cluster details .....') - eks = boto3.client('eks',region_name = region) - try : - cluster_details = eks.describe_cluster( - name=cluster_name - ) - report['cluster'] = {'version':cluster_details['cluster']['version'],'region':cluster_details['cluster']['arn'].split(':')[3]} - log_pusher(log_details,cluster_name,region,'Cluster control plane version ' + report['cluster']['version']) - print('Cluster control plane version ' + report['cluster']['version']) - customer_report['cluster version'] = 'Cluster control plane version ' + report['cluster']['version'] - if update_version : - if cluster_details['cluster']['version'] == update_version : - log_pusher(log_details,cluster_name,region,'Cluster already upgraded to version' + cluster_details['cluster']['version']) - print('Cluster already upgraded to version ' + cluster_details['cluster']['version']) - customer_report['cluster upgradation'] = 'Cluster already upgraded to version ' + cluster_details['cluster']['version'] - elif (round(float(update_version) - float(cluster_details['cluster']['version']),2)) == 0.01 and float(update_version) < 1.25: - log_pusher(log_details,cluster_name,region,'Cluster with verison ' + cluster_details['cluster']['version'] + " can be updated to target version " + update_version) - print('Cluster with verison ' + cluster_details['cluster']['version'] + " can be updated to target version " + update_version) - customer_report['cluster upgradation'] = 'Cluster with verison ' + cluster_details['cluster']['version'] + " can be updated to target version " + update_version - else : - customer_report['cluster upgradation'] = 'Cluster with verison ' + cluster_details['cluster']['version'] + " cannot be updated to target version " + update_version - log_pusher(log_details,cluster_name,region,'Cluster with verison ' + cluster_details['cluster']['version'] + " cannot be updated to target version " + update_version) - print('Cluster with verison ' + cluster_details['cluster']['version'] + " cannot be updated to target version " + update_version) - report['preflight_status'] = False - errors.append('Cluster with verison ' + cluster_details['cluster']['version'] + " cannot be updated to target version " + update_version) - return - cmk_key_check(log_details,errors,cluster_name,region,cluster_details,report,customer_report) - security_group_check(log_details,errors,cluster_name,region,cluster_details,report,customer_report) - pod_security_policies(log_details,errors,cluster_name,region,report,customer_report) - node_group_details = nodegroup_customami(log_details,errors,cluster_name,region,report,customer_report,update_version) - report['nodegroup_details'] = node_group_details - customer_report['nodegroup_details'] = node_group_details - subnet_details(log_details,errors,cluster_name,region,report,customer_report) - cluster_roles(preflight,log_details,errors,cluster_name,region,report,customer_report) - addon_version(log_details,errors,cluster_name,region,cluster_details,report,customer_report,pass_vpc) - pod_disruption_budget(log_details,errors,cluster_name,region,report,customer_report,force_upgrade) - horizontal_auto_scaler(log_details,errors,cluster_name,region,report,customer_report) - cluster_auto_scaler(log_details,errors,cluster_name,region,report,customer_report) - if report['cluster']['version'] != "1.21" and update_version: - depricated_api_check(log_details,errors,cluster_name,region,report,customer_report,update_version) - if(email) : - print('\n') - log_pusher(log_details,cluster_name,region,'Delivering report via Email...') - print('Delivering report via Email...') - send_email(preflight,log_details,errors,cluster_name,region,report,customer_report,email) - except Exception as e : - errors.append('Some error occured during preflight check process {err}'.format(err = e)) - customer_report['cluster version'] = 'Some error occured during preflight check process' - log_pusher(log_details,cluster_name,region,'Some error occured during preflight check process {err}'.format(err=e)) - print('Some error occured during preflight check process {err}'.format(err=e)) - report['preflight_status'] = False +def get_cluster_version( + preflight, + log_details, + errors, + cluster_name, + region, + pass_vpc, + update_version, + report, + customer_report, + email, + force_upgrade, +): + loading_config(cluster_name, region) + print("\n") + log_pusher(log_details, cluster_name, region, "Fetching cluster details .....") + print("Fetching cluster details .....") + eks = boto3.client("eks", region_name=region) + try: + cluster_details = eks.describe_cluster(name=cluster_name) + report["cluster"] = { + "version": cluster_details["cluster"]["version"], + "region": cluster_details["cluster"]["arn"].split(":")[3], + } + log_pusher(log_details, cluster_name, region, "Cluster control plane version " + report["cluster"]["version"]) + print("Cluster control plane version " + report["cluster"]["version"]) + customer_report["cluster version"] = "Cluster control plane version " + report["cluster"]["version"] + if update_version: + if cluster_details["cluster"]["version"] == update_version: + log_pusher( + log_details, + cluster_name, + region, + "Cluster already upgraded to version" + cluster_details["cluster"]["version"], + ) + print("Cluster already upgraded to version " + cluster_details["cluster"]["version"]) + customer_report["cluster upgradation"] = ( + "Cluster already upgraded to version " + cluster_details["cluster"]["version"] + ) + elif (round(float(update_version) - float(cluster_details["cluster"]["version"]), 2)) == 0.01 and float( + update_version + ) < 1.25: + log_pusher( + log_details, + cluster_name, + region, + "Cluster with verison " + + cluster_details["cluster"]["version"] + + " can be updated to target version " + + update_version, + ) + print( + "Cluster with verison " + + cluster_details["cluster"]["version"] + + " can be updated to target version " + + update_version + ) + customer_report["cluster upgradation"] = ( + "Cluster with verison " + + cluster_details["cluster"]["version"] + + " can be updated to target version " + + update_version + ) + else: + customer_report["cluster upgradation"] = ( + "Cluster with verison " + + cluster_details["cluster"]["version"] + + " cannot be updated to target version " + + update_version + ) + log_pusher( + log_details, + cluster_name, + region, + "Cluster with verison " + + cluster_details["cluster"]["version"] + + " cannot be updated to target version " + + update_version, + ) + print( + "Cluster with verison " + + cluster_details["cluster"]["version"] + + " cannot be updated to target version " + + update_version + ) + report["preflight_status"] = False + errors.append( + "Cluster with verison " + + cluster_details["cluster"]["version"] + + " cannot be updated to target version " + + update_version + ) + return + cmk_key_check(log_details, errors, cluster_name, region, cluster_details, report, customer_report) + security_group_check(log_details, errors, cluster_name, region, cluster_details, report, customer_report) + pod_security_policies(log_details, errors, cluster_name, region, report, customer_report) + node_group_details = nodegroup_customami( + log_details, errors, cluster_name, region, report, customer_report, update_version + ) + report["nodegroup_details"] = node_group_details + customer_report["nodegroup_details"] = node_group_details + subnet_details(log_details, errors, cluster_name, region, report, customer_report) + cluster_roles(preflight, log_details, errors, cluster_name, region, report, customer_report) + addon_version(log_details, errors, cluster_name, region, cluster_details, report, customer_report, pass_vpc) + pod_disruption_budget(log_details, errors, cluster_name, region, report, customer_report, force_upgrade) + horizontal_auto_scaler(log_details, errors, cluster_name, region, report, customer_report) + cluster_auto_scaler(log_details, errors, cluster_name, region, report, customer_report) + if report["cluster"]["version"] != "1.21" and update_version: + depricated_api_check(log_details, errors, cluster_name, region, report, customer_report, update_version) + if email: + print("\n") + log_pusher(log_details, cluster_name, region, "Delivering report via Email...") + print("Delivering report via Email...") + send_email(preflight, log_details, errors, cluster_name, region, report, customer_report, email) + except Exception as e: + errors.append("Some error occured during preflight check process {err}".format(err=e)) + customer_report["cluster version"] = "Some error occured during preflight check process" + log_pusher( + log_details, cluster_name, region, "Some error occured during preflight check process {err}".format(err=e) + ) + print("Some error occured during preflight check process {err}".format(err=e)) + report["preflight_status"] = False # Gather Subnet Utilization fro the current cluster -def subnet_details(log_details,errors,cluster_name,region,report,customer_report): - loading_config(cluster_name,region) + +def subnet_details(log_details, errors, cluster_name, region, report, customer_report): + loading_config(cluster_name, region) try: - print('\n') - log_pusher(log_details,cluster_name,region,'Checking Available IP for subnets associated with cluster.....') - print('Checking Available IP for subnets associated with cluster.....') - eks = boto3.client('eks',region_name = region) - cluster_details = eks.describe_cluster( - name=cluster_name - ) + print("\n") + log_pusher(log_details, cluster_name, region, "Checking Available IP for subnets associated with cluster.....") + print("Checking Available IP for subnets associated with cluster.....") + eks = boto3.client("eks", region_name=region) + cluster_details = eks.describe_cluster(name=cluster_name) subnets = [] - customer_report['subnet'] = [] + customer_report["subnet"] = [] error = [] - for subnet_id in cluster_details['cluster']['resourcesVpcConfig']['subnetIds']: - ec2 = boto3.resource('ec2',region_name = region) + for subnet_id in cluster_details["cluster"]["resourcesVpcConfig"]["subnetIds"]: + ec2 = boto3.resource("ec2", region_name=region) subnet = ec2.Subnet(subnet_id) - if subnet.available_ip_address_count < 5 : - error.append('Subnet ID ' + str(subnet_id) + ' doesnt have a minimum of 5 available IP') - log_pusher(log_details,cluster_name,region,'Subnet ID ' + str(subnet_id) + ' have a ' + str(subnet.available_ip_address_count) +' available IP address') - customer_report['subnet'].append('Subnet ID ' + str(subnet_id) + ' have a ' + str(subnet.available_ip_address_count) +' available IP address') - print('Subnet ID ' + str(subnet_id) + ' have a ' + str(subnet.available_ip_address_count) +' available IP address') - subnets.append({subnet_id : subnet.available_ip_address_count}) - #report['subnets'] = subnets - if len(error) > 0 : - report['preflight_status'] = False - errors.append('Available IPs for Subnet verification failed') - log_pusher(log_details,cluster_name,region,'Available IPs for Subnet verification failed') - print('Available IPs for Subnet verification failed') - customer_report['subnet'].append('Available IP for Subnet verification failed') + if subnet.available_ip_address_count < 5: + error.append("Subnet ID " + str(subnet_id) + " doesnt have a minimum of 5 available IP") + log_pusher( + log_details, + cluster_name, + region, + "Subnet ID " + + str(subnet_id) + + " have a " + + str(subnet.available_ip_address_count) + + " available IP address", + ) + customer_report["subnet"].append( + "Subnet ID " + + str(subnet_id) + + " have a " + + str(subnet.available_ip_address_count) + + " available IP address" + ) + print( + "Subnet ID " + + str(subnet_id) + + " have a " + + str(subnet.available_ip_address_count) + + " available IP address" + ) + subnets.append({subnet_id: subnet.available_ip_address_count}) + # report['subnets'] = subnets + if len(error) > 0: + report["preflight_status"] = False + errors.append("Available IPs for Subnet verification failed") + log_pusher(log_details, cluster_name, region, "Available IPs for Subnet verification failed") + print("Available IPs for Subnet verification failed") + customer_report["subnet"].append("Available IP for Subnet verification failed") for e in error: - customer_report['subnet'].append(e) + customer_report["subnet"].append(e) print(e) - else : - customer_report['subnet'].append('Available IP for Subnet verified') - log_pusher(log_details,cluster_name,region,'Available IPs for Subnet verified') - print('Available IPs for Subnet verified') - except Exception as e: - - errors.append('Some error occured while fetching subnet details {err}'.format(err=e)) - log_pusher(log_details,cluster_name,region,'Some error occured while fetching subnet details {err}'.format(err=e)) - print('Some error occured while fetching subnet details {err}'.format(err=e)) - report['preflight_status'] = False - + else: + customer_report["subnet"].append("Available IP for Subnet verified") + log_pusher(log_details, cluster_name, region, "Available IPs for Subnet verified") + print("Available IPs for Subnet verified") + except Exception as e: + errors.append("Some error occured while fetching subnet details {err}".format(err=e)) + log_pusher( + log_details, cluster_name, region, "Some error occured while fetching subnet details {err}".format(err=e) + ) + print("Some error occured while fetching subnet details {err}".format(err=e)) + report["preflight_status"] = False # Verification for required cluster roles -def cluster_roles(preflight,log_details,errors,cluster_name,region,report,customer_report): - loading_config(cluster_name,region) - s3 = boto3.resource('s3') + +def cluster_roles(preflight, log_details, errors, cluster_name, region, report, customer_report): + loading_config(cluster_name, region) + s3 = boto3.resource("s3") # Cluster Roles for proper functioning # cluster_roles_list = s3.Object('eks-one-click-upgrade', 'cluster_roles.json') # cluster_roles_list = cluster_roles_list.get()['Body'].read().decode('utf-8') # cluster_roles_list = json.loads(cluster_roles_list) - f = open('eksupdate/src/S3Files/cluster_roles.json',) + f = open( + "eksupdate/src/S3Files/cluster_roles.json", + ) cluster_roles_list = json.load(f) - #print(cluster_roles_list) - if preflight : - cluster_roles_list = cluster_roles_list['preflight'] - else : - cluster_roles_list = cluster_roles_list['postflight'] # print(cluster_roles_list) - try : - print('\n') - log_pusher(log_details,cluster_name,region,'Checking important cluster role are present or not .....') - print('Checking important cluster role are present or not .....') + if preflight: + cluster_roles_list = cluster_roles_list["preflight"] + else: + cluster_roles_list = cluster_roles_list["postflight"] + # print(cluster_roles_list) + try: + print("\n") + log_pusher(log_details, cluster_name, region, "Checking important cluster role are present or not .....") + print("Checking important cluster role are present or not .....") available = [] not_available = [] - customer_report['cluster role'] = [] - for role in cluster_roles_list['roles']: - try : - v1=client.RbacAuthorizationV1Api() - fs = 'metadata.name=' + role - res=v1.list_cluster_role(field_selector = fs) - if(len(res.items) > 0): + customer_report["cluster role"] = [] + for role in cluster_roles_list["roles"]: + try: + v1 = client.RbacAuthorizationV1Api() + fs = "metadata.name=" + role + res = v1.list_cluster_role(field_selector=fs) + if len(res.items) > 0: available.append(role) - #print(available) - #print(res) - else : + # print(available) + # print(res) + else: not_available.append(role) - #print(not_available) - #print('Unable to find ' + role) + # print(not_available) + # print('Unable to find ' + role) except: - customer_report['cluster role'].append('Some error occured while checking role for ' + role) - log_pusher(log_details,cluster_name,region,'Some error occured while checking role for ' + role) - print('Some error occured while checking role for ' + role) - #report['cluster-roles'] = {'available' : available,'not-available':not_available} - if report['cluster']['version'] in cluster_roles_list.keys(): - - #print(cluster_roles_list[report['cluster']['version']]) - for role in cluster_roles_list[report['cluster']['version']].keys(): - v1=client.RbacAuthorizationV1Api() - res = eval(cluster_roles_list[report['cluster']['version']][role]) - - if len(res.items) == 0 : - log_pusher(log_details,cluster_name,region,role + " is not present in the cluster") - customer_report['cluster role'].append(role + " is not present in the cluster") + customer_report["cluster role"].append("Some error occured while checking role for " + role) + log_pusher(log_details, cluster_name, region, "Some error occured while checking role for " + role) + print("Some error occured while checking role for " + role) + # report['cluster-roles'] = {'available' : available,'not-available':not_available} + if report["cluster"]["version"] in cluster_roles_list.keys(): + + # print(cluster_roles_list[report['cluster']['version']]) + for role in cluster_roles_list[report["cluster"]["version"]].keys(): + v1 = client.RbacAuthorizationV1Api() + res = eval(cluster_roles_list[report["cluster"]["version"]][role]) + + if len(res.items) == 0: + log_pusher(log_details, cluster_name, region, role + " is not present in the cluster") + customer_report["cluster role"].append(role + " is not present in the cluster") print(role + " is not present in the cluster") not_available.append(role) - else : + else: available.append(role) - if len(not_available) > 0 : - customer_report['cluster role'].append('Cluster role verification failed') - log_pusher(log_details,cluster_name,region,'Cluster role verification failed') - print('Cluster role verification failed') - report['preflight_status'] = False - errors.append('Cluster role verification failed') + if len(not_available) > 0: + customer_report["cluster role"].append("Cluster role verification failed") + log_pusher(log_details, cluster_name, region, "Cluster role verification failed") + print("Cluster role verification failed") + report["preflight_status"] = False + errors.append("Cluster role verification failed") for n in not_available: - customer_report['cluster role'].append(n + ' role is not present in the cluster') - print(n + ' role is not present in the cluster') - log_pusher(log_details,cluster_name,region,n + ' role is not present in the cluster') - - else : - customer_report['cluster role'].append('All cluster role needed sucessfully verified') - for n in available : - customer_report['cluster role'].append(n + ' role is present in cluster') - log_pusher(log_details,cluster_name,region,n + ' role is present in cluster') - print(n + ' role is present in the cluster') - log_pusher(log_details,cluster_name,region,'All cluster role needed sucessfully verified') - print('All cluster role needed sucessfully verified') - #print(report['cluster-roles']) - except Exception as e : - - errors.append('Some error occured while checking the cluster roles available {err}'.format(err=e)) - customer_report['cluster role'].append('Some error occured while checking the cluster roles available {err}'.format(err=e)) - print('Some error occured while checking the cluster roles available {err}'.format(err=e)) - report['preflight_status'] = False + customer_report["cluster role"].append(n + " role is not present in the cluster") + print(n + " role is not present in the cluster") + log_pusher(log_details, cluster_name, region, n + " role is not present in the cluster") + else: + customer_report["cluster role"].append("All cluster role needed sucessfully verified") + for n in available: + customer_report["cluster role"].append(n + " role is present in cluster") + log_pusher(log_details, cluster_name, region, n + " role is present in cluster") + print(n + " role is present in the cluster") + log_pusher(log_details, cluster_name, region, "All cluster role needed sucessfully verified") + print("All cluster role needed sucessfully verified") + # print(report['cluster-roles']) + except Exception as e: + errors.append("Some error occured while checking the cluster roles available {err}".format(err=e)) + customer_report["cluster role"].append( + "Some error occured while checking the cluster roles available {err}".format(err=e) + ) + print("Some error occured while checking the cluster roles available {err}".format(err=e)) + report["preflight_status"] = False # Check for Pod Security Policies -def pod_security_policies(log_details,errors,cluster_name,region,report,customer_report): - loading_config(cluster_name,region) - try : + +def pod_security_policies(log_details, errors, cluster_name, region, report, customer_report): + loading_config(cluster_name, region) + try: v1 = client.PolicyV1beta1Api() - print('\n') - log_pusher(log_details,cluster_name,region,'Pod Security Policies .....') - print('Pod Security Policies .....') - ret = v1.list_pod_security_policy(field_selector='metadata.name=eks.privileged') + print("\n") + log_pusher(log_details, cluster_name, region, "Pod Security Policies .....") + print("Pod Security Policies .....") + ret = v1.list_pod_security_policy(field_selector="metadata.name=eks.privileged") # pprint(ret) # report['pod-security-policy'] = ret - if len(ret.items) == 0 : - customer_report['pod security policy'] = ("Pod Security Policy with eks.privileged role doesnt exists.") - report['preflight_status'] = False + if len(ret.items) == 0: + customer_report["pod security policy"] = "Pod Security Policy with eks.privileged role doesnt exists." + report["preflight_status"] = False errors.append("Pod Security Policy with eks.privileged role doesnt exists.") - log_pusher(log_details,cluster_name,region,"Pod Security Policy with eks.privileged role doesnt exists.") + log_pusher(log_details, cluster_name, region, "Pod Security Policy with eks.privileged role doesnt exists.") print("Pod Security Policy with eks.privileged role doesnt exists.") for i in ret.items: # print(i.metadata) - if(i.metadata.name=='eks.privileged'): - customer_report['pod security policy'] = ("Pod Security Policy with eks.privileged role exists.") - log_pusher(log_details,cluster_name,region,"Pod Security Policy with eks.privileged role exists.") + if i.metadata.name == "eks.privileged": + customer_report["pod security policy"] = "Pod Security Policy with eks.privileged role exists." + log_pusher(log_details, cluster_name, region, "Pod Security Policy with eks.privileged role exists.") print("Pod Security Policy with eks.privileged role exists.") else: - customer_report['pod security policy'] = ("Pod Security Policy with eks.privileged role doesnt exists.") - report['preflight_status'] = False + customer_report["pod security policy"] = "Pod Security Policy with eks.privileged role doesnt exists." + report["preflight_status"] = False errors.append("Pod Security Policy with eks.privileged role doesnt exists.") - log_pusher(log_details,cluster_name,region,"Pod Security Policy with eks.privileged role doesnt exists.") + log_pusher( + log_details, cluster_name, region, "Pod Security Policy with eks.privileged role doesnt exists." + ) print("Pod Security Policy with eks.privileged role doesnt exists.") - #print(report['pod-security-policy']) - except Exception as e : - errors.append('Some error occured while checking for the policy security policies {err}'.format(err=e)) - customer_report['pod security policy'] = ('Some error occured while checking for the policy security policies') - log_pusher(log_details,cluster_name,region,'Some error occured while checking for the policy security policies {err}'.format(err=e)) - print('Some error occured while checking for the policy security policies {err}'.format(err=e)) - report['preflight_status'] = False - - - + # print(report['pod-security-policy']) + except Exception as e: + errors.append("Some error occured while checking for the policy security policies {err}".format(err=e)) + customer_report["pod security policy"] = "Some error occured while checking for the policy security policies" + log_pusher( + log_details, + cluster_name, + region, + "Some error occured while checking for the policy security policies {err}".format(err=e), + ) + print("Some error occured while checking for the policy security policies {err}".format(err=e)) + report["preflight_status"] = False # Check for compatibility between addon and control plane versions -def addon_version(log_details,errors,cluster_name,region,cluster_details,report,customer_report,pass_vpc): - loading_config(cluster_name,region) +def addon_version(log_details, errors, cluster_name, region, cluster_details, report, customer_report, pass_vpc): + loading_config(cluster_name, region) # Fetch data from S3 Bucket - s3 = boto3.resource('s3') + s3 = boto3.resource("s3") yaml_data = {} config_map = {} @@ -355,48 +463,72 @@ def addon_version(log_details,errors,cluster_name,region,cluster_details,report, # version_dict = s3.Object('eks-one-click-upgrade', 'version_dict.json') # version_dict = version_dict.get()['Body'].read().decode('utf-8') # version_dict = json.loads(version_dict) - #print(version_dict) + # print(version_dict) - f = open('eksupdate/src/S3Files/version_dict.json',) + f = open( + "eksupdate/src/S3Files/version_dict.json", + ) version_dict = json.load(f) # Kube Proxy config YAML # kube_proxy_config = s3.Object('eks-one-click-upgrade', 'addons/kube-proxy.json') # kube_proxy_config = kube_proxy_config.get()['Body'].read().decode('utf-8') # kube_proxy_config = json.loads(kube_proxy_config) - f = open('eksupdate/src/S3Files/kube-proxy.json',) + f = open( + "eksupdate/src/S3Files/kube-proxy.json", + ) kube_proxy_config = json.load(f) - kube_proxy_container = kube_proxy_config['spec']['template']['spec']['containers'][0] - yaml_data['kube-proxy'] = {'image':kube_proxy_container['image'],'volumeMount':kube_proxy_container['volumeMounts'],'env':None} - #print(kube_proxy_config) + kube_proxy_container = kube_proxy_config["spec"]["template"]["spec"]["containers"][0] + yaml_data["kube-proxy"] = { + "image": kube_proxy_container["image"], + "volumeMount": kube_proxy_container["volumeMounts"], + "env": None, + } + # print(kube_proxy_config) # Core DNS config YAML # core_dns_config = s3.Object('eks-one-click-upgrade', 'addons/coredns.json') # core_dns_config = core_dns_config.get()['Body'].read().decode('utf-8') # core_dns_config = json.loads(core_dns_config) - f = open('eksupdate/src/S3Files/coredns.json',) + f = open( + "eksupdate/src/S3Files/coredns.json", + ) core_dns_config = json.load(f) - coredns_container = core_dns_config['spec']['template']['spec']['containers'][0] - yaml_data['coredns'] = {'image':coredns_container['image'],'volumeMount':coredns_container['volumeMounts'],'env':None} - #print(core_dns_config) + coredns_container = core_dns_config["spec"]["template"]["spec"]["containers"][0] + yaml_data["coredns"] = { + "image": coredns_container["image"], + "volumeMount": coredns_container["volumeMounts"], + "env": None, + } + # print(core_dns_config) # VPC CNI config YAML # vpc_cni_config = s3.Object('eks-one-click-upgrade', 'addons/vpc-cni.json') # vpc_cni_config = vpc_cni_config.get()['Body'].read().decode('utf-8') # vpc_cni_config = json.loads(vpc_cni_config) - f = open('eksupdate/src/S3Files/vpc-cni.json',) + f = open( + "eksupdate/src/S3Files/vpc-cni.json", + ) vpc_cni_config = json.load(f) - vpc_cni_container = vpc_cni_config['spec']['template']['spec']['containers'][0] - yaml_data['vpc-cni'] = {'image':vpc_cni_container['image'],'volumeMount':vpc_cni_container['volumeMounts'],'env':vpc_cni_container['env']} - #print(vpc_cni_config) + vpc_cni_container = vpc_cni_config["spec"]["template"]["spec"]["containers"][0] + yaml_data["vpc-cni"] = { + "image": vpc_cni_container["image"], + "volumeMount": vpc_cni_container["volumeMounts"], + "env": vpc_cni_container["env"], + } + # print(vpc_cni_config) # Kube Proxy config map YAML # kube_proxy_config_map = s3.Object('eks-one-click-upgrade', 'configMap/kube-proxy.json') # kube_proxy_config_map = kube_proxy_config_map.get()['Body'].read().decode('utf-8') # kube_proxy_config_map = json.loads(kube_proxy_config_map) - f = open('eksupdate/src/S3Files/kube-proxy-configmap.json',) + f = open( + "eksupdate/src/S3Files/kube-proxy-configmap.json", + ) kube_proxy_config_map = json.load(f) - config_map['certificate-authority'] = yaml.safe_load(kube_proxy_config_map['data']['kubeconfig'])['clusters'][0]['cluster']['certificate-authority'] + config_map["certificate-authority"] = yaml.safe_load(kube_proxy_config_map["data"]["kubeconfig"])["clusters"][0][ + "cluster" + ]["certificate-authority"] # Core DNS config map YAML # core_dns_config_map = s3.Object('eks-one-click-upgrade', 'configMap/coredns.json') @@ -405,427 +537,597 @@ def addon_version(log_details,errors,cluster_name,region,cluster_details,report, # config_map['coredns'] = core_dns_config_map['data'] # pprint(yaml.load(core_dns_config_map['data']['Corefile']).split('.:53')[1].split(' ')) - try : + try: - print('\n') - log_pusher(log_details,cluster_name,region,'Check addon version compatibility .....') - print('Check addon version compatibility .....') + print("\n") + log_pusher(log_details, cluster_name, region, "Check addon version compatibility .....") + print("Check addon version compatibility .....") addons = [] - report['addon_params'] = {} - customer_report['addons'] = {'vpc-cni':{},'kube-proxy':{},'coredns':{}} - v1=client.AppsV1Api() - daemon_set = v1.list_namespaced_daemon_set('kube-system') - deployment = v1.list_namespaced_deployment('kube-system') - calico = v1.list_namespaced_daemon_set('calico-system') + report["addon_params"] = {} + customer_report["addons"] = {"vpc-cni": {}, "kube-proxy": {}, "coredns": {}} + v1 = client.AppsV1Api() + daemon_set = v1.list_namespaced_daemon_set("kube-system") + deployment = v1.list_namespaced_deployment("kube-system") + calico = v1.list_namespaced_daemon_set("calico-system") if len(calico.items) != 0: - for cal in calico.items : + for cal in calico.items: print("Calico addon is present in cluster") - check_pods_running('calico',log_details,cluster_name,region,report,errors,"calico-system") - for ds in daemon_set.items : - #print(ds.metadata.name) - if(ds.metadata.name == 'aws-node' and not pass_vpc): - version_str = ds.spec.template.spec.containers[0].image.split('amazon-k8s-cni:v')[1].split('-')[0] - config = {'image':ds.spec.template.spec.containers[0].image,'volumeMount':ds.spec.template.spec.containers[0].volume_mounts,'env':ds.spec.template.spec.containers[0].env} - target_version = version_dict[report['cluster']['version']]['vpc-cni'].split('.') - version = version_str.split('.') - check_pods_running('aws-node',log_details,cluster_name,region,report,errors) - if(int(''.join(version)) >= int('170')) : - addons.append({'name' :'vpc-cni','version' : version_str, 'update' : False}) - customer_report['addons']['vpc-cni']['version'] = 'Up to date' - log_pusher(log_details,cluster_name,region, 'vpc-cni version up to date' ) - print('vpc-cni version up to date') - check_addons_params(log_details,config,'vpc-cni',cluster_details,config_map,yaml_data,report,customer_report,cluster_name,region,errors) - else : - addons.append({'name' :'vpc-cni','version' :version_str, 'update' : True}) - log_pusher(log_details,cluster_name,region, 'vpc-cni version is not compatible' ) - print('vpc-cni version is not compatible' ) - customer_report['addons']['vpc-cni']['version'] = 'Version Not Compatible with current cluster version' - #print(ds.metadata.name + ':' + version) - elif(ds.metadata.name == 'kube-proxy') : - version = ds.spec.template.spec.containers[0].image.split(ds.metadata.name+':v')[1].split('-')[0] - config = {'image':ds.spec.template.spec.containers[0].image,'volumeMount':ds.spec.template.spec.containers[0].volume_mounts,'env':ds.spec.template.spec.containers[0].env} - check_pods_running('kube-proxy',log_details,cluster_name,region,report,errors) - print(version_dict[report['cluster']['version']][ds.metadata.name],version) - if(version_dict[report['cluster']['version']][ds.metadata.name] == version) : - addons.append({'name' :ds.metadata.name,'version' : version, 'update' : False}) - log_pusher(log_details,cluster_name,region, 'kube-proxy version up to date' ) - print('kube-proxy version up to date') - customer_report['addons'][ds.metadata.name]['version'] = 'Up to date' - check_addons_params(log_details,config,ds.metadata.name,cluster_details,config_map,yaml_data,report,customer_report,cluster_name,region,errors) - else : - addons.append({'name' :ds.metadata.name,'version' : version, 'update' : True}) - print('kube-proxy version not compatible') - log_pusher(log_details,cluster_name,region, 'kube-proxy version not compatible' ) - customer_report['addons'][ds.metadata.name]['version'] = 'Version Not Compatible with current cluster version' - #print(ds.metadata.name + ':' + version) - #print(addons) - for dp in deployment.items : + check_pods_running("calico", log_details, cluster_name, region, report, errors, "calico-system") + for ds in daemon_set.items: + # print(ds.metadata.name) + if ds.metadata.name == "aws-node" and not pass_vpc: + version_str = ds.spec.template.spec.containers[0].image.split("amazon-k8s-cni:v")[1].split("-")[0] + config = { + "image": ds.spec.template.spec.containers[0].image, + "volumeMount": ds.spec.template.spec.containers[0].volume_mounts, + "env": ds.spec.template.spec.containers[0].env, + } + target_version = version_dict[report["cluster"]["version"]]["vpc-cni"].split(".") + version = version_str.split(".") + check_pods_running("aws-node", log_details, cluster_name, region, report, errors) + if int("".join(version)) >= int("170"): + addons.append({"name": "vpc-cni", "version": version_str, "update": False}) + customer_report["addons"]["vpc-cni"]["version"] = "Up to date" + log_pusher(log_details, cluster_name, region, "vpc-cni version up to date") + print("vpc-cni version up to date") + check_addons_params( + log_details, + config, + "vpc-cni", + cluster_details, + config_map, + yaml_data, + report, + customer_report, + cluster_name, + region, + errors, + ) + else: + addons.append({"name": "vpc-cni", "version": version_str, "update": True}) + log_pusher(log_details, cluster_name, region, "vpc-cni version is not compatible") + print("vpc-cni version is not compatible") + customer_report["addons"]["vpc-cni"][ + "version" + ] = "Version Not Compatible with current cluster version" + # print(ds.metadata.name + ':' + version) + elif ds.metadata.name == "kube-proxy": + version = ds.spec.template.spec.containers[0].image.split(ds.metadata.name + ":v")[1].split("-")[0] + config = { + "image": ds.spec.template.spec.containers[0].image, + "volumeMount": ds.spec.template.spec.containers[0].volume_mounts, + "env": ds.spec.template.spec.containers[0].env, + } + check_pods_running("kube-proxy", log_details, cluster_name, region, report, errors) + print(version_dict[report["cluster"]["version"]][ds.metadata.name], version) + if version_dict[report["cluster"]["version"]][ds.metadata.name] == version: + addons.append({"name": ds.metadata.name, "version": version, "update": False}) + log_pusher(log_details, cluster_name, region, "kube-proxy version up to date") + print("kube-proxy version up to date") + customer_report["addons"][ds.metadata.name]["version"] = "Up to date" + check_addons_params( + log_details, + config, + ds.metadata.name, + cluster_details, + config_map, + yaml_data, + report, + customer_report, + cluster_name, + region, + errors, + ) + else: + addons.append({"name": ds.metadata.name, "version": version, "update": True}) + print("kube-proxy version not compatible") + log_pusher(log_details, cluster_name, region, "kube-proxy version not compatible") + customer_report["addons"][ds.metadata.name][ + "version" + ] = "Version Not Compatible with current cluster version" + # print(ds.metadata.name + ':' + version) + # print(addons) + for dp in deployment.items: # print(dp.metadata.name) - if (dp.metadata.name == 'coredns'): - version = dp.spec.template.spec.containers[0].image.split(dp.metadata.name+':v')[1].split('-')[0] - config = {'image':dp.spec.template.spec.containers[0].image,'volumeMount':dp.spec.template.spec.containers[0].volume_mounts,'env':dp.spec.template.spec.containers[0].env} - check_pods_running('coredns',log_details,cluster_name,region,report,errors) - if(version_dict[report['cluster']['version']][dp.metadata.name] == version) : - addons.append({'name' :dp.metadata.name,'version' : version, 'update' : False}) - log_pusher(log_details,cluster_name,region, 'core-dns version up to date' ) - customer_report['addons'][dp.metadata.name]['version'] = 'Up to date' - #print(config) - print( 'core-dns version up to date') - check_addons_params(log_details,config,dp.metadata.name,cluster_details,config_map,yaml_data,report,customer_report,cluster_name,region,errors) - else : - addons.append({'name' :dp.metadata.name,'version' : version, 'update' : True}) - print( 'core-dns version up not compatible' ) - log_pusher(log_details,cluster_name,region, 'core-dns version up not compatible' ) - customer_report['addons'][dp.metadata.name]['version'] = 'Version Not Compatible with current cluster version' + if dp.metadata.name == "coredns": + version = dp.spec.template.spec.containers[0].image.split(dp.metadata.name + ":v")[1].split("-")[0] + config = { + "image": dp.spec.template.spec.containers[0].image, + "volumeMount": dp.spec.template.spec.containers[0].volume_mounts, + "env": dp.spec.template.spec.containers[0].env, + } + check_pods_running("coredns", log_details, cluster_name, region, report, errors) + if version_dict[report["cluster"]["version"]][dp.metadata.name] == version: + addons.append({"name": dp.metadata.name, "version": version, "update": False}) + log_pusher(log_details, cluster_name, region, "core-dns version up to date") + customer_report["addons"][dp.metadata.name]["version"] = "Up to date" + # print(config) + print("core-dns version up to date") + check_addons_params( + log_details, + config, + dp.metadata.name, + cluster_details, + config_map, + yaml_data, + report, + customer_report, + cluster_name, + region, + errors, + ) + else: + addons.append({"name": dp.metadata.name, "version": version, "update": True}) + print("core-dns version up not compatible") + log_pusher(log_details, cluster_name, region, "core-dns version up not compatible") + customer_report["addons"][dp.metadata.name][ + "version" + ] = "Version Not Compatible with current cluster version" # print(addons) - report['addons'] = addons - customer_report['addons_version'] = addons - except Exception as e : - errors.append('Some error occured while checking the addon version {err}'.format(err=e)) - log_pusher(log_details,cluster_name,region,'Some error occured while checking the addon version {err}'.format(err=e)) - print('Some error occured while checking the addon version {err}'.format(err=e)) - report['preflight_status'] = False + report["addons"] = addons + customer_report["addons_version"] = addons + except Exception as e: + errors.append("Some error occured while checking the addon version {err}".format(err=e)) + log_pusher( + log_details, cluster_name, region, "Some error occured while checking the addon version {err}".format(err=e) + ) + print("Some error occured while checking the addon version {err}".format(err=e)) + report["preflight_status"] = False # Function to check for addons pod to be in running state -def check_pods_running(addon,log_details,cluster_name,region,report,errors,namespace="kube-system"): + +def check_pods_running(addon, log_details, cluster_name, region, report, errors, namespace="kube-system"): try: v1 = client.CoreV1Api() count = 0 rep = v1.list_namespaced_pod(namespace) for pod in rep.items: - if addon in pod.metadata.name : + if addon in pod.metadata.name: count = count + 1 - if (pod.status.phase == "Running"): + if pod.status.phase == "Running": print(addon + " pod is running") - log_pusher(log_details,cluster_name,region,addon + " pod is running") + log_pusher(log_details, cluster_name, region, addon + " pod is running") else: print(addon + " Pod is not running, it is in " + pod.status.phase) - log_pusher(log_details,cluster_name,region,addon + " Pod is not running, it is in " + pod.status.phase) + log_pusher( + log_details, cluster_name, region, addon + " Pod is not running, it is in " + pod.status.phase + ) errors.append(addon + " Pod is not running, it is in " + pod.status.phase) - report['preflight_status'] = False - + report["preflight_status"] = False + if count == 0: print(addon + " pod is not present in the cluster") - log_pusher(log_details,cluster_name,region,addon + " pod is not present in the cluster") - report['preflight_status'] = False + log_pusher(log_details, cluster_name, region, addon + " pod is not present in the cluster") + report["preflight_status"] = False errors.append(addon + " pod is not present in the cluster") - except Exception as e : - errors.append('Some error occured while checking for addon pods to be running {err}'.format(err=e)) - log_pusher(log_details,cluster_name,region,'Some error occured while checking for addon pods to be running {err}'.format(err=e)) - print('Some error occured while checking for addon pods to be running {err}'.format(err=e)) - report['preflight_status'] = False - - + except Exception as e: + errors.append("Some error occured while checking for addon pods to be running {err}".format(err=e)) + log_pusher( + log_details, + cluster_name, + region, + "Some error occured while checking for addon pods to be running {err}".format(err=e), + ) + print("Some error occured while checking for addon pods to be running {err}".format(err=e)) + report["preflight_status"] = False # Function to check the volume mount and env in cluster and original YAML files for addons -def check_addons_params(log_details,config,name,cluster_details,config_map,yaml_data,report,customer_report,cluster_name,region,errors): + +def check_addons_params( + log_details, + config, + name, + cluster_details, + config_map, + yaml_data, + report, + customer_report, + cluster_name, + region, + errors, +): # loading_config(cluster_name,region) s3_config = yaml_data[name] - log_pusher(log_details,cluster_name,region,'************* Parameter check for '+name + ' *************') - print('************* Parameter check for '+name + ' *************') + log_pusher(log_details, cluster_name, region, "************* Parameter check for " + name + " *************") + print("************* Parameter check for " + name + " *************") # Compare image name - image_part_1 = config['image'].split('.ecr.')[0] == s3_config['image'].split('.ecr.')[0] - image_part_2 = config['image'].split('amazonaws.com/')[1].split(':v')[0] == s3_config['image'].split('amazonaws.com/')[1].split(':v')[0] + image_part_1 = config["image"].split(".ecr.")[0] == s3_config["image"].split(".ecr.")[0] + image_part_2 = ( + config["image"].split("amazonaws.com/")[1].split(":v")[0] + == s3_config["image"].split("amazonaws.com/")[1].split(":v")[0] + ) if image_part_1 and image_part_2: - report['addon_params'][name] = {'image':config['image']} - customer_report['addons'][name]["image"] = "Image Verified" - log_pusher(log_details,cluster_name,region,'Image verified') - print('Image verified') - else : - customer_report['addons'][name]["image"] = "Image Verification Failed" - log_pusher(log_details,cluster_name,region,'Image verification failed') - print('Image verification failed') + report["addon_params"][name] = {"image": config["image"]} + customer_report["addons"][name]["image"] = "Image Verified" + log_pusher(log_details, cluster_name, region, "Image verified") + print("Image verified") + else: + customer_report["addons"][name]["image"] = "Image Verification Failed" + log_pusher(log_details, cluster_name, region, "Image verification failed") + print("Image verification failed") # Compare Volume Mounts mount_paths = [] - customer_report['addons'][name]["mount_paths"] = {} - report['addon_params'][name]['mount_paths'] = {} + customer_report["addons"][name]["mount_paths"] = {} + report["addon_params"][name]["mount_paths"] = {} remaining = [] - for i in range(len(s3_config['volumeMount'])): - mount_paths.append(s3_config['volumeMount'][i]['mountPath']) - for i in range(len(config['volumeMount'])): - if config['volumeMount'][i].mount_path in mount_paths: - mount_paths.remove(config['volumeMount'][i].mount_path) - else : - remaining.append(config['volumeMount'][i].mount_path) - if len(mount_paths) > 0 : - customer_report['addons'][name]["mount_paths"]['message'] = 'Default mount paths are missing' - report['addon_params'][name]['mount_paths']['custom'] = True - report['addon_params'][name]['mount_paths']['default'] = ' '.join(map(str, mount_paths)) - customer_report['addons'][name]["mount_paths"]['default-mountpaths'] = ' '.join(map(str, mount_paths)) - log_pusher(log_details,cluster_name,region,'These mount paths are not present ' + ' '.join(map(str, mount_paths))) - print('These mount paths are not present',' '.join(map(str, mount_paths))) - if len(remaining) > 0 : - customer_report['addons'][name]["mount_paths"]['message'] = 'There are additional mount paths present' - report['addon_params'][name]['mount_paths']['custom'] = True - report['addon_params'][name]['mount_paths']['user-defined'] = ' '.join(map(str, mount_paths)) - customer_report['addons'][name]["mount_paths"]['userdefined-mountpaths'] = ' '.join(map(str, mount_paths)) - log_pusher(log_details,cluster_name,region,'These user defined mount paths are present' + ' '.join(map(str, mount_paths)) ) - print('These user defined mount paths are present',' '.join(map(str, mount_paths)) ) + for i in range(len(s3_config["volumeMount"])): + mount_paths.append(s3_config["volumeMount"][i]["mountPath"]) + for i in range(len(config["volumeMount"])): + if config["volumeMount"][i].mount_path in mount_paths: + mount_paths.remove(config["volumeMount"][i].mount_path) + else: + remaining.append(config["volumeMount"][i].mount_path) + if len(mount_paths) > 0: + customer_report["addons"][name]["mount_paths"]["message"] = "Default mount paths are missing" + report["addon_params"][name]["mount_paths"]["custom"] = True + report["addon_params"][name]["mount_paths"]["default"] = " ".join(map(str, mount_paths)) + customer_report["addons"][name]["mount_paths"]["default-mountpaths"] = " ".join(map(str, mount_paths)) + log_pusher( + log_details, cluster_name, region, "These mount paths are not present " + " ".join(map(str, mount_paths)) + ) + print("These mount paths are not present", " ".join(map(str, mount_paths))) + if len(remaining) > 0: + customer_report["addons"][name]["mount_paths"]["message"] = "There are additional mount paths present" + report["addon_params"][name]["mount_paths"]["custom"] = True + report["addon_params"][name]["mount_paths"]["user-defined"] = " ".join(map(str, mount_paths)) + customer_report["addons"][name]["mount_paths"]["userdefined-mountpaths"] = " ".join(map(str, mount_paths)) + log_pusher( + log_details, + cluster_name, + region, + "These user defined mount paths are present" + " ".join(map(str, mount_paths)), + ) + print("These user defined mount paths are present", " ".join(map(str, mount_paths))) if len(mount_paths) == 0 and len(remaining) == 0: - report['addon_params'][name]["mount_paths"]['custom'] = False - customer_report['addons'][name]["mount_paths"]['message'] = 'Mount paths verified successfully' - log_pusher(log_details,cluster_name,region,'Mount path verification successful') - print('Mount path verification successful') + report["addon_params"][name]["mount_paths"]["custom"] = False + customer_report["addons"][name]["mount_paths"]["message"] = "Mount paths verified successfully" + log_pusher(log_details, cluster_name, region, "Mount path verification successful") + print("Mount path verification successful") # Compare env - if name == 'vpc-cni': - customer_report['addons'][name]["env"] = {} - report['addon_params'][name]["envs"] = {} + if name == "vpc-cni": + customer_report["addons"][name]["env"] = {} + report["addon_params"][name]["envs"] = {} envs = [] extra_envs = [] - for i in range(len(s3_config['env'])): - #print(s3_config['env'][i]['name']) - envs.append(s3_config['env'][i]['name']) - for i in range(len(config['env'])): - if config['env'][i].name in envs : - envs.remove(config['env'][i].name) - else : - extra_envs.append(config['env'][i].name) - if len(envs) > 0 : - #customer_report['addons'][name]["mount_paths"].append('These mount paths are not present',mount_paths) - customer_report['addons'][name]["env"]['message'] = 'Default envs are missing' - report['addon_params'][name]['envs']['custom'] = True - report['addon_params'][name]['envs']['default'] = ' '.join(map(str, envs)) - customer_report['addons'][name]["env"]['default-envs'] = ' '.join(map(str, envs)) - log_pusher(log_details,cluster_name,region,'These envs are not present' + ' '.join(map(str, envs)) ) - - print('These envs are not present' + ' '.join(map(str, envs)) ) - - if len(extra_envs) > 0 : - #customer_report['addons'][name]["mount_paths"].append('these user defined mount paths are not present',remaining) - report['addon_params'][name]['envs']['custom'] = True - report['addon_params'][name]['envs']['user-defined'] = ' '.join(map(str, extra_envs)) - customer_report['addons'][name]["env"]['message'] = 'There are additional envs present' - log_pusher(log_details,cluster_name,region,'These user defined envs are present'+' '.join(map(str, extra_envs)) ) - print('These user defined envs are present',' '.join(map(str, extra_envs)) ) - - - customer_report['addons'][name]["env"]['userdefined-envs'] =' '.join(map(str, extra_envs)) + for i in range(len(s3_config["env"])): + # print(s3_config['env'][i]['name']) + envs.append(s3_config["env"][i]["name"]) + for i in range(len(config["env"])): + if config["env"][i].name in envs: + envs.remove(config["env"][i].name) + else: + extra_envs.append(config["env"][i].name) + if len(envs) > 0: + # customer_report['addons'][name]["mount_paths"].append('These mount paths are not present',mount_paths) + customer_report["addons"][name]["env"]["message"] = "Default envs are missing" + report["addon_params"][name]["envs"]["custom"] = True + report["addon_params"][name]["envs"]["default"] = " ".join(map(str, envs)) + customer_report["addons"][name]["env"]["default-envs"] = " ".join(map(str, envs)) + log_pusher(log_details, cluster_name, region, "These envs are not present" + " ".join(map(str, envs))) + + print("These envs are not present" + " ".join(map(str, envs))) + + if len(extra_envs) > 0: + # customer_report['addons'][name]["mount_paths"].append('these user defined mount paths are not present',remaining) + report["addon_params"][name]["envs"]["custom"] = True + report["addon_params"][name]["envs"]["user-defined"] = " ".join(map(str, extra_envs)) + customer_report["addons"][name]["env"]["message"] = "There are additional envs present" + log_pusher( + log_details, + cluster_name, + region, + "These user defined envs are present" + " ".join(map(str, extra_envs)), + ) + print("These user defined envs are present", " ".join(map(str, extra_envs))) + + customer_report["addons"][name]["env"]["userdefined-envs"] = " ".join(map(str, extra_envs)) if len(envs) == 0 and len(extra_envs) == 0: - report['addon_params'][name]["envs"]['custom'] = False - customer_report['addons'][name]["env"]['message'] = 'Envs verified successfully' - log_pusher(log_details,cluster_name,region,'Envs verification successful') - print('Envs verification successful') - if name == 'coredns': - customer_report['addons'][name]["corefile"] = {} - report['addon_params'][name]["corefile"] = {} - arr = ['errors','health','kubernetes cluster.local in-addr.arpa ip6.arpa { pods insecure fallthrough in-addr.arpa ip6.arpa }','prometheus :9153','forward . /etc/resolv.conf','cache 30','loop','reload','loadbalance','{','}'] + report["addon_params"][name]["envs"]["custom"] = False + customer_report["addons"][name]["env"]["message"] = "Envs verified successfully" + log_pusher(log_details, cluster_name, region, "Envs verification successful") + print("Envs verification successful") + if name == "coredns": + customer_report["addons"][name]["corefile"] = {} + report["addon_params"][name]["corefile"] = {} + arr = [ + "errors", + "health", + "kubernetes cluster.local in-addr.arpa ip6.arpa { pods insecure fallthrough in-addr.arpa ip6.arpa }", + "prometheus :9153", + "forward . /etc/resolv.conf", + "cache 30", + "loop", + "reload", + "loadbalance", + "{", + "}", + ] v1 = client.CoreV1Api() default = [] - ret = v1.list_config_map_for_all_namespaces(field_selector = 'metadata.name=coredns') - corefile = yaml.safe_load(ret.items[0].data['Corefile']).split('.:53')[1] + ret = v1.list_config_map_for_all_namespaces(field_selector="metadata.name=coredns") + corefile = yaml.safe_load(ret.items[0].data["Corefile"]).split(".:53")[1] for i in arr: - if corefile.find(i) == -1 : + if corefile.find(i) == -1: default.append(i) - log_pusher(log_details,cluster_name,region,i + 'doesnt exist in corefile') - print(i + 'doesnt exist in corefile') - + log_pusher(log_details, cluster_name, region, i + "doesnt exist in corefile") + print(i + "doesnt exist in corefile") + else: - corefile = corefile.replace(i,'') - corefile = corefile.replace(' ','') - if(len(default) > 0): - customer_report['addons'][name]["corefile"]['message'] = 'Default corefile fields are not present' - report['addon_params'][name]["corefile"]['custom'] = True - report['addon_params'][name]["corefile"]['default'] = ' '.join(map(str, default)) - customer_report['addons'][name]["corefile"]['default-corefile-fields'] = ' '.join(map(str, default)) - print('Default corefile fields are not present',' '.join(map(str, default)) ) + corefile = corefile.replace(i, "") + corefile = corefile.replace(" ", "") + if len(default) > 0: + customer_report["addons"][name]["corefile"]["message"] = "Default corefile fields are not present" + report["addon_params"][name]["corefile"]["custom"] = True + report["addon_params"][name]["corefile"]["default"] = " ".join(map(str, default)) + customer_report["addons"][name]["corefile"]["default-corefile-fields"] = " ".join(map(str, default)) + print("Default corefile fields are not present", " ".join(map(str, default))) if len(corefile) > 0: - customer_report['addons'][name]["corefile"]['message'] = 'There are additional fields present in corefile' - report['addon_params'][name]["corefile"]['custom'] = True - report['addon_params'][name]["corefile"]['userdefined'] = ' '.join(map(str, corefile)) - customer_report['addons'][name]["corefile"]['userdefined-corefile-fields'] = ' '.join(map(str, corefile)) - log_pusher(log_details,cluster_name,region,'Additional fields in corefile '+' '.join(map(str, corefile))) - print('Additional fields in corefile ',' '.join(map(str, corefile))) - if len(corefile) == 0 and len(default) == 0 : - report['addon_params'][name]["corefile"]['custom'] = False - customer_report['addons'][name]["corefile"]['message'] = 'Corefile fields verified successfully' - log_pusher(log_details,cluster_name,region,'Corefile verified successfully') - print('Corefile verified successfully') - if name == 'kube-proxy' : - report['addon_params'][name]["certificate-authority"] = {} - report['addon_params'][name]["server-endpoint"] = {} - customer_report['addons'][name]["certificate-authority"] = {} - customer_report['addons'][name]["server-endpoint"] = {} + customer_report["addons"][name]["corefile"]["message"] = "There are additional fields present in corefile" + report["addon_params"][name]["corefile"]["custom"] = True + report["addon_params"][name]["corefile"]["userdefined"] = " ".join(map(str, corefile)) + customer_report["addons"][name]["corefile"]["userdefined-corefile-fields"] = " ".join(map(str, corefile)) + log_pusher( + log_details, cluster_name, region, "Additional fields in corefile " + " ".join(map(str, corefile)) + ) + print("Additional fields in corefile ", " ".join(map(str, corefile))) + if len(corefile) == 0 and len(default) == 0: + report["addon_params"][name]["corefile"]["custom"] = False + customer_report["addons"][name]["corefile"]["message"] = "Corefile fields verified successfully" + log_pusher(log_details, cluster_name, region, "Corefile verified successfully") + print("Corefile verified successfully") + if name == "kube-proxy": + report["addon_params"][name]["certificate-authority"] = {} + report["addon_params"][name]["server-endpoint"] = {} + customer_report["addons"][name]["certificate-authority"] = {} + customer_report["addons"][name]["server-endpoint"] = {} v1 = client.CoreV1Api() - ret = v1.list_config_map_for_all_namespaces(field_selector = 'metadata.name=kube-proxy') - if yaml.safe_load(ret.items[0].data['kubeconfig'])['clusters'][0]['cluster']['certificate-authority'] == config_map['certificate-authority'] : - report['addon_params'][name]["certificate-authority"]['verified'] = True - customer_report['addons'][name]["certificate-authority"]['message'] = 'Certificate Authority Verified in kube config' - report['addon_params'][name]["certificate-authority"]['certificate'] = config_map['certificate-authority'] - log_pusher(log_details,cluster_name,region,'Certificate Authority Verified in kube config') - print('Certificate Authority Verified in kube config') - else : - customer_report['addons'][name]["certificate-authority"]['message'] = 'Certificate Verification failed in kube config' - report['addon_params'][name]["certificate-authority"]['verified'] = False - report['addon_params'][name]["certificate-authority"]['certificate'] = yaml.safe_load(ret.items[0].data['kubeconfig'])['clusters'][0]['cluster']['certificate-authority'] - log_pusher(log_details,cluster_name,region,'Certificate Verification failed in kube config') - print('Certificate Verification failed in kube config') + ret = v1.list_config_map_for_all_namespaces(field_selector="metadata.name=kube-proxy") + if ( + yaml.safe_load(ret.items[0].data["kubeconfig"])["clusters"][0]["cluster"]["certificate-authority"] + == config_map["certificate-authority"] + ): + report["addon_params"][name]["certificate-authority"]["verified"] = True + customer_report["addons"][name]["certificate-authority"][ + "message" + ] = "Certificate Authority Verified in kube config" + report["addon_params"][name]["certificate-authority"]["certificate"] = config_map["certificate-authority"] + log_pusher(log_details, cluster_name, region, "Certificate Authority Verified in kube config") + print("Certificate Authority Verified in kube config") + else: + customer_report["addons"][name]["certificate-authority"][ + "message" + ] = "Certificate Verification failed in kube config" + report["addon_params"][name]["certificate-authority"]["verified"] = False + report["addon_params"][name]["certificate-authority"]["certificate"] = yaml.safe_load( + ret.items[0].data["kubeconfig"] + )["clusters"][0]["cluster"]["certificate-authority"] + log_pusher(log_details, cluster_name, region, "Certificate Verification failed in kube config") + print("Certificate Verification failed in kube config") # pprint(yaml.load(ret.items[0].data['kubeconfig'])['clusters'][0]['cluster']['server']) # pprint(yaml.load(ret.items[0].data['kubeconfig'])['clusters'][0]['cluster']['certificate-authority']) # pprint(config_map['certificate-authority']) - server_endpoint = cluster_details['cluster']['endpoint'] - if yaml.safe_load(ret.items[0].data['kubeconfig'])['clusters'][0]['cluster']['server'] == cluster_details['cluster']['endpoint'].lower(): - customer_report['addons'][name]["server-endpoint"]['message'] = 'Server end point verified' - report['addon_params'][name]["server-endpoint"]['verified'] = True - report['addon_params'][name]["server-endpoint"]['server-endpoint'] = cluster_details['cluster']['endpoint'].lower() - log_pusher(log_details,cluster_name,region,'Server end point verified') - print('Server end point verified') - else : - customer_report['addons'][name]["server-endpoint"]['message'] = 'Server end point verification failed' - report['addon_params'][name]["certificate-authority"]['verified'] = False - report['addon_params'][name]["certificate-authority"]['server-endpoint'] = yaml.safe_load(ret.items[0].data['kubeconfig'])['clusters'][0]['cluster']['server'] - log_pusher(log_details,cluster_name,region,' Server end point verification failed') - print(' Server end point verification failed') - - -def pod_disruption_budget(log_details,errors,cluster_name,region,report,customer_report,force_upgrade): - loading_config(cluster_name,region) - print('\n') - log_pusher(log_details,cluster_name,region,"Fetching Pod Disruption Budget Details....") + server_endpoint = cluster_details["cluster"]["endpoint"] + if ( + yaml.safe_load(ret.items[0].data["kubeconfig"])["clusters"][0]["cluster"]["server"] + == cluster_details["cluster"]["endpoint"].lower() + ): + customer_report["addons"][name]["server-endpoint"]["message"] = "Server end point verified" + report["addon_params"][name]["server-endpoint"]["verified"] = True + report["addon_params"][name]["server-endpoint"]["server-endpoint"] = cluster_details["cluster"][ + "endpoint" + ].lower() + log_pusher(log_details, cluster_name, region, "Server end point verified") + print("Server end point verified") + else: + customer_report["addons"][name]["server-endpoint"]["message"] = "Server end point verification failed" + report["addon_params"][name]["certificate-authority"]["verified"] = False + report["addon_params"][name]["certificate-authority"]["server-endpoint"] = yaml.safe_load( + ret.items[0].data["kubeconfig"] + )["clusters"][0]["cluster"]["server"] + log_pusher(log_details, cluster_name, region, " Server end point verification failed") + print(" Server end point verification failed") + + +def pod_disruption_budget(log_details, errors, cluster_name, region, report, customer_report, force_upgrade): + loading_config(cluster_name, region) + print("\n") + log_pusher(log_details, cluster_name, region, "Fetching Pod Disruption Budget Details....") print("Fetching Pod Disruption Budget Details....") - try : + try: v1 = client.PolicyV1beta1Api() ret = v1.list_pod_disruption_budget_for_all_namespaces() - if len(ret.items) == 0 : - customer_report['pod disruption budget'] = 'No Pod Disruption Budget exists in cluster' - log_pusher(log_details,cluster_name,region,'No Pod Disruption Budget exists in cluster') - print('No Pod Disruption Budget exists in cluster') + if len(ret.items) == 0: + customer_report["pod disruption budget"] = "No Pod Disruption Budget exists in cluster" + log_pusher(log_details, cluster_name, region, "No Pod Disruption Budget exists in cluster") + print("No Pod Disruption Budget exists in cluster") else: print("Pod Disruption Budget exists in cluster therefore force upgrade is required to upgrade the cluster") - if not force_upgrade : - print("Pod Disruption Budget exists in cluster therefore force upgrade is required to upgrade the cluster, To upgrade please run the code with --force flag ") + if not force_upgrade: + print( + "Pod Disruption Budget exists in cluster therefore force upgrade is required to upgrade the cluster, To upgrade please run the code with --force flag " + ) errors.append("To upgrade please run the code with --force flag ") report["preflight_status"] = False - for pdb in ret.items : - max_available = pdb.spec.max_unavailable - min_available = pdb.spec.min_available - #print(max_available,min_available) - report['pdb'] = {'max_unavailable':max_available,'min_available':min_available} - customer_report['pod disruption budget'] = 'Pod disruption budget exists in the cluster' - log_pusher(log_details,cluster_name,region,"Pod disruption budget exists with max unavailable as " + str(max_available) + " and min available as " + str(min_available)) - print("Pod disruption budget exists with max unavailable as " + str(max_available) + " and min available as " + str(min_available)) + for pdb in ret.items: + max_available = pdb.spec.max_unavailable + min_available = pdb.spec.min_available + # print(max_available,min_available) + report["pdb"] = {"max_unavailable": max_available, "min_available": min_available} + customer_report["pod disruption budget"] = "Pod disruption budget exists in the cluster" + log_pusher( + log_details, + cluster_name, + region, + "Pod disruption budget exists with max unavailable as " + + str(max_available) + + " and min available as " + + str(min_available), + ) + print( + "Pod disruption budget exists with max unavailable as " + + str(max_available) + + " and min available as " + + str(min_available) + ) v1 = client.CoreV1Api() pods_and_nodes = [] ret = v1.list_pod_for_all_namespaces(watch=False) - #pprint(ret.items[0].spec.node_name) + # pprint(ret.items[0].spec.node_name) for i in ret.items: - pods_and_nodes.append({'name':i.metadata.name,'namespace':i.metadata.namespace,'nodename':i.spec.node_name}) - report['pdb']['pods'] = pods_and_nodes + pods_and_nodes.append( + {"name": i.metadata.name, "namespace": i.metadata.namespace, "nodename": i.spec.node_name} + ) + report["pdb"]["pods"] = pods_and_nodes pprint(pods_and_nodes) - #pprint(pods_and_nodes) - except Exception as e : + # pprint(pods_and_nodes) + except Exception as e: errors.append("Error ocurred while checking for pod disruption budget {err}".format(err=e)) - customer_report['pod disruption budget'] = "Error ocurred while checking for pod disruption budget" - log_pusher(log_details,cluster_name,region,"Error ocurred while checking for pod disruption budget {err}".format(err=e)) + customer_report["pod disruption budget"] = "Error ocurred while checking for pod disruption budget" + log_pusher( + log_details, + cluster_name, + region, + "Error ocurred while checking for pod disruption budget {err}".format(err=e), + ) print("Error ocurred while checking for pod disruption budget {err}".format(err=e)) - report['preflight_status'] = False - + report["preflight_status"] = False + -def cluster_auto_scaler(log_details,errors,cluster_name,region,report,customer_report): - loading_config(cluster_name,region) - print('\n') - log_pusher(log_details,cluster_name,region,"Fetching Cluster Auto Scaler Details....") +def cluster_auto_scaler(log_details, errors, cluster_name, region, report, customer_report): + loading_config(cluster_name, region) + print("\n") + log_pusher(log_details, cluster_name, region, "Fetching Cluster Auto Scaler Details....") print("Fetching Cluster Auto Scaler Details....") - try: - eks = boto3.client('eks',region_name = region) + try: + eks = boto3.client("eks", region_name=region) cluster_details = eks.describe_cluster(name=cluster_name) - val= cluster_details['cluster']['version'] - l=val.split('.') - v1=client.AppsV1Api() - res=v1.list_deployment_for_all_namespaces() + val = cluster_details["cluster"]["version"] + l = val.split(".") + v1 = client.AppsV1Api() + res = v1.list_deployment_for_all_namespaces() for i in res.items: - x=i.metadata.name - if(x=='cluster-autoscaler'): - log_pusher(log_details,cluster_name,region,"Cluster Autoscaler exists") + x = i.metadata.name + if x == "cluster-autoscaler": + log_pusher(log_details, cluster_name, region, "Cluster Autoscaler exists") print("Cluster Autoscaler exists") - check_pods_running('cluster-autoscaler',log_details,cluster_name,region,report,errors) - version = i.spec.template.spec.containers[0].image.split('k8s.gcr.io/autoscaling/cluster-autoscaler:v')[1].split('-')[0] - l1=version.split('.') - if(l[0]==l1[0] and l[1]==l1[1]): - report['cluster_auto_scaler'] = {'image':i.spec.template.spec.containers[0].image} - customer_report['cluster autoscaler'] = "Auto scaler version is compatible with cluster version!" - log_pusher(log_details,cluster_name,region,"Auto scaler version is compatible with cluster version!") + check_pods_running("cluster-autoscaler", log_details, cluster_name, region, report, errors) + version = ( + i.spec.template.spec.containers[0] + .image.split("k8s.gcr.io/autoscaling/cluster-autoscaler:v")[1] + .split("-")[0] + ) + l1 = version.split(".") + if l[0] == l1[0] and l[1] == l1[1]: + report["cluster_auto_scaler"] = {"image": i.spec.template.spec.containers[0].image} + customer_report["cluster autoscaler"] = "Auto scaler version is compatible with cluster version!" + log_pusher( + log_details, cluster_name, region, "Auto scaler version is compatible with cluster version!" + ) print("Auto scaler version is compatible with cluster version!") - else : + else: print("Auto scaler version is not compatible with cluster version") - customer_report['cluster autoscaler'] = "Auto scaler version is not compatible with cluster version" - log_pusher(log_details,cluster_name,region,"Auto scaler version is not compatible with cluster version!") + customer_report["cluster autoscaler"] = "Auto scaler version is not compatible with cluster version" + log_pusher( + log_details, cluster_name, region, "Auto scaler version is not compatible with cluster version!" + ) return - + else: continue - customer_report['cluster autoscaler'] = "Cluster Autoscaler doesn't exists" - log_pusher(log_details,cluster_name,region,"Cluster Autoscaler doesn't exists") + customer_report["cluster autoscaler"] = "Cluster Autoscaler doesn't exists" + log_pusher(log_details, cluster_name, region, "Cluster Autoscaler doesn't exists") print("Cluster Autoscaler doesn't exists") - except Exception as e : - errors.append('Error occured while checking for the cluster autoscaler {err}'.format(err=e)) - customer_report['cluster autoscaler'] = 'Error occured while checking for the cluster autoscaler' + e - log_pusher(log_details,cluster_name,region,'Error occured while checking for the cluster autoscaler {err}'.format(err=e)) - print('Error occured while checking for the cluster autoscaler {err}'.format(err=e)) - report['preflight_status'] = False - -def horizontal_auto_scaler(log_details,errors,cluster_name,region,report,customer_report): - loading_config(cluster_name,region) - print('\n') - log_pusher(log_details,cluster_name,region,"Fetching Horizontal Autoscaler Details....") + except Exception as e: + errors.append("Error occured while checking for the cluster autoscaler {err}".format(err=e)) + customer_report["cluster autoscaler"] = "Error occured while checking for the cluster autoscaler" + e + log_pusher( + log_details, + cluster_name, + region, + "Error occured while checking for the cluster autoscaler {err}".format(err=e), + ) + print("Error occured while checking for the cluster autoscaler {err}".format(err=e)) + report["preflight_status"] = False + + +def horizontal_auto_scaler(log_details, errors, cluster_name, region, report, customer_report): + loading_config(cluster_name, region) + print("\n") + log_pusher(log_details, cluster_name, region, "Fetching Horizontal Autoscaler Details....") print("Fetching Horizontal Autoscaler Details....") - try : - v1=client.AutoscalingV1Api() - ret=v1.list_horizontal_pod_autoscaler_for_all_namespaces() - if len(ret.items) == 0 : - customer_report['horizontal auto scaler'] = 'No Horizontal Auto Scaler exists in cluster' - log_pusher(log_details,cluster_name,region,'No Horizontal Auto Scaler exists in cluster') - print('No Horizontal Auto Scaler exists in cluster') + try: + v1 = client.AutoscalingV1Api() + ret = v1.list_horizontal_pod_autoscaler_for_all_namespaces() + if len(ret.items) == 0: + customer_report["horizontal auto scaler"] = "No Horizontal Auto Scaler exists in cluster" + log_pusher(log_details, cluster_name, region, "No Horizontal Auto Scaler exists in cluster") + print("No Horizontal Auto Scaler exists in cluster") else: - customer_report['horizontal auto scaler'] = 'Horizontal Pod Auto scaler exists in cluster' - log_pusher(log_details,cluster_name,region,'Horizontal Pod Auto scaler exists in cluster') - print('Horizontal Pod Auto scaler exists in cluster') - report['horizontal_autoscaler'] = ret.items[0] - #pprint(ret.items[0]) + customer_report["horizontal auto scaler"] = "Horizontal Pod Auto scaler exists in cluster" + log_pusher(log_details, cluster_name, region, "Horizontal Pod Auto scaler exists in cluster") + print("Horizontal Pod Auto scaler exists in cluster") + report["horizontal_autoscaler"] = ret.items[0] + # pprint(ret.items[0]) except Exception as e: - errors.append('Error occured while checking for horizontal autoscaler {err}'.format(err=e)) - log_pusher(log_details,cluster_name,region,'Error occured while checking for horizontal autoscaler {err}'.format(err=e)) - print('Error occured while checking for horizontal autoscaler {err}'.format(err=e)) - customer_report['horizontal auto scaler'] = 'Error occured while checking for horizontal autoscaler' - report['preflight_status'] = False - -def depricated_api_check(log_details,errors,cluster_name,region,report,customer_report,update_version): - loading_config(cluster_name,region) - s3 = boto3.resource('s3') + errors.append("Error occured while checking for horizontal autoscaler {err}".format(err=e)) + log_pusher( + log_details, + cluster_name, + region, + "Error occured while checking for horizontal autoscaler {err}".format(err=e), + ) + print("Error occured while checking for horizontal autoscaler {err}".format(err=e)) + customer_report["horizontal auto scaler"] = "Error occured while checking for horizontal autoscaler" + report["preflight_status"] = False + + +def depricated_api_check(log_details, errors, cluster_name, region, report, customer_report, update_version): + loading_config(cluster_name, region) + s3 = boto3.resource("s3") # Depricated API dictionary # depricated_api = s3.Object('eks-one-click-upgrade', 'depricatedApi') # depricated_api = depricated_api.get()['Body'].read().decode('utf-8') # depricated_api = json.loads(depricated_api) - f = open('eksupdate/src/S3Files/depricatedApi',) + f = open( + "eksupdate/src/S3Files/depricatedApi", + ) depricated_api = json.load(f) - #print(depricated_api) - print('\n') - log_pusher(log_details,cluster_name,region,"Checking for any depricated API being used....") + # print(depricated_api) + print("\n") + log_pusher(log_details, cluster_name, region, "Checking for any depricated API being used....") print("Checking for any depricated API being used....") - customer_report['depricated Api'] = [] - try : + customer_report["depricated Api"] = [] + try: dict = depricated_api[update_version] # print(dict) - + for key in dict.keys(): # print(key) - if key == 'all-resources': + if key == "all-resources": for k in dict[key].keys(): - if (dict[key][k] == "permanent") : - customer_report['depricated Api'].append(k + ' API has been depricated permanently ') - log_pusher(log_details,cluster_name,region,k + ' API has been depricated permanently ') - print(k + ' API has been depricated permanently ') - else : - customer_report['depricated Api'].append(k + ' API has been depricated use ' + dict[key][k] + ' instead') - log_pusher(log_details,cluster_name,region,k + ' API has been depricated use ' + dict[key][k] + ' instead') - print(k + ' API has been depricated use ' + dict[key][k] + ' instead') - else : + if dict[key][k] == "permanent": + customer_report["depricated Api"].append(k + " API has been depricated permanently ") + log_pusher(log_details, cluster_name, region, k + " API has been depricated permanently ") + print(k + " API has been depricated permanently ") + else: + customer_report["depricated Api"].append( + k + " API has been depricated use " + dict[key][k] + " instead" + ) + log_pusher( + log_details, + cluster_name, + region, + k + " API has been depricated use " + dict[key][k] + " instead", + ) + print(k + " API has been depricated use " + dict[key][k] + " instead") + else: depricated_resource = [] new_resource = [] v1 = eval(key) @@ -837,124 +1139,190 @@ def depricated_api_check(log_details,errors,cluster_name,region,report,customer_ ret = v2.get_api_resources() for resource in ret.resources: new_resource.append(resource.name) - if (dict[key][k] in depricated_resource and dict[key][k] not in new_resource): - customer_report['depricated Api'].append("Resource "+ dict[key][k] + " is present in depricated API "+ key + " to be shifted to " + k) - errors.append("Resource "+ dict[key][k] + " is present in depricated API "+ key + " to be shifted to " + k) - log_pusher(log_details,cluster_name,region,"Resource "+ dict[key][k] + " is present in depricated API "+ key + " to be shifted to " + k) - print("Resource "+ dict[key][k] + " is present in depricated API "+ key + " to be shifted to " + k) - log_pusher(log_details,cluster_name,region,"Depricated Api check completed") + if dict[key][k] in depricated_resource and dict[key][k] not in new_resource: + customer_report["depricated Api"].append( + "Resource " + + dict[key][k] + + " is present in depricated API " + + key + + " to be shifted to " + + k + ) + errors.append( + "Resource " + + dict[key][k] + + " is present in depricated API " + + key + + " to be shifted to " + + k + ) + log_pusher( + log_details, + cluster_name, + region, + "Resource " + + dict[key][k] + + " is present in depricated API " + + key + + " to be shifted to " + + k, + ) + print( + "Resource " + + dict[key][k] + + " is present in depricated API " + + key + + " to be shifted to " + + k + ) + log_pusher(log_details, cluster_name, region, "Depricated Api check completed") print("Depricated Api check completed") except Exception as e: errors.append("Depricated API check failed {err}".format(err=e)) - customer_report['depricated Api'].append("Depricated API check failed") - log_pusher(log_details,cluster_name,region,"Depricated API check failed {err}".format(err=e)) + customer_report["depricated Api"].append("Depricated API check failed") + log_pusher(log_details, cluster_name, region, "Depricated API check failed {err}".format(err=e)) print("Depricated API check failed {err}".format(err=e)) - report['preflight_status'] = False - -def cmk_key_check(log_details,errors,cluster_name,region,cluster,report,customer_report): - loading_config(cluster_name,region) - cmk = boto3.client('kms',region_name = region) - print('\n') - log_pusher(log_details,cluster_name,region,'Checking if customer management key exists....') - print('Checking if customer management key exists....') - try : - if 'encryptionConfig' in cluster['cluster'].keys(): - cmk_key = cluster['cluster']['encryptionConfig'][0]['provider']['keyArn'] - customer_report['CMK Key'] = 'CMK Key with id ' + cmk_key + ' is present' - log_pusher(log_details,cluster_name,region,'CMK Key with id ' + cmk_key + ' is present') - print('CMK Key with id ' + cmk_key + ' is present') + report["preflight_status"] = False + + +def cmk_key_check(log_details, errors, cluster_name, region, cluster, report, customer_report): + loading_config(cluster_name, region) + cmk = boto3.client("kms", region_name=region) + print("\n") + log_pusher(log_details, cluster_name, region, "Checking if customer management key exists....") + print("Checking if customer management key exists....") + try: + if "encryptionConfig" in cluster["cluster"].keys(): + cmk_key = cluster["cluster"]["encryptionConfig"][0]["provider"]["keyArn"] + customer_report["CMK Key"] = "CMK Key with id " + cmk_key + " is present" + log_pusher(log_details, cluster_name, region, "CMK Key with id " + cmk_key + " is present") + print("CMK Key with id " + cmk_key + " is present") response = cmk.describe_key( - KeyId=cmk_key, + KeyId=cmk_key, ) - try : + try: response = cmk.describe_key( - KeyId=cmk_key, + KeyId=cmk_key, + ) + if "KeyId" in response["KeyMetadata"].keys(): + log_pusher(log_details, cluster_name, region, "Key with id " + cmk_key + " exist in user account") + customer_report["CMK Key"] = "Key with id " + cmk_key + " exist in user account" + print("Key with id " + cmk_key + " exist in user account") + else: + report["preflight_status"] = False + errors.append("Key with id " + cmk_key + " doesnt exist in user account") + log_pusher( + log_details, cluster_name, region, "Key with id " + cmk_key + " doesnt exist in user account" + ) + customer_report["CMK Key"] = "Key with id " + cmk_key + " doesnt exist in user account" + print("Key with id " + cmk_key + " doesnt exist in user account") + except: + report["preflight_status"] = False + errors.append("Key with id " + cmk_key + " doesnt exist in user account") + log_pusher( + log_details, cluster_name, region, "Key with id " + cmk_key + " doesnt exist in user account" ) - if 'KeyId' in response['KeyMetadata'].keys() : - log_pusher(log_details,cluster_name,region,'Key with id ' + cmk_key + " exist in user account") - customer_report['CMK Key'] = 'Key with id ' + cmk_key + " exist in user account" - print('Key with id ' + cmk_key + " exist in user account") - else : - report['preflight_status'] = False - errors.append('Key with id ' + cmk_key + " doesnt exist in user account") - log_pusher(log_details,cluster_name,region,'Key with id ' + cmk_key + " doesnt exist in user account") - customer_report['CMK Key'] ='Key with id ' + cmk_key + " doesnt exist in user account" - print('Key with id ' + cmk_key + " doesnt exist in user account") - except : - report['preflight_status'] = False - errors.append('Key with id ' + cmk_key + " doesnt exist in user account") - log_pusher(log_details,cluster_name,region,'Key with id ' + cmk_key + " doesnt exist in user account") - customer_report['CMK Key'] ='Key with id ' + cmk_key + " doesnt exist in user account" - print('Key with id ' + cmk_key + " doesnt exist in user account") - else : - customer_report['CMK Key'] = 'No CMK Key associated with the cluster' - log_pusher(log_details,cluster_name,region,'No CMK Key associated with the cluster') - print('No CMK Key associated with the cluster') - except Exception as e : - - errors.append('Error while checking for cluster CMK key {err}'.format(err=e)) - customer_report['CMK Key'] = 'Error while checking for cluster CMK key' - log_pusher(log_details,cluster_name,region,'Error while checking for cluster CMK key {err}'.format(err=e)) - print('Error while checking for cluster CMK key {err}'.format(err=e)) - report['preflight_status'] = False - - -def security_group_check(log_details,errors,cluster_name,region,cluster,report,customer_report): - loading_config(cluster_name,region) - print('\n') - log_pusher(log_details,cluster_name,region,'Fetching security group details .....') - print('Fetching security group details .....') - try : - security_groups = cluster['cluster']['resourcesVpcConfig']['securityGroupIds'] - if len(security_groups)== 0: - log_pusher(log_details,cluster_name,region,'No security groups available with cluster') - customer_report['security group'] = 'No security groups available with cluster' - print('No security groups available with cluster') - else : + customer_report["CMK Key"] = "Key with id " + cmk_key + " doesnt exist in user account" + print("Key with id " + cmk_key + " doesnt exist in user account") + else: + customer_report["CMK Key"] = "No CMK Key associated with the cluster" + log_pusher(log_details, cluster_name, region, "No CMK Key associated with the cluster") + print("No CMK Key associated with the cluster") + except Exception as e: + + errors.append("Error while checking for cluster CMK key {err}".format(err=e)) + customer_report["CMK Key"] = "Error while checking for cluster CMK key" + log_pusher(log_details, cluster_name, region, "Error while checking for cluster CMK key {err}".format(err=e)) + print("Error while checking for cluster CMK key {err}".format(err=e)) + report["preflight_status"] = False + + +def security_group_check(log_details, errors, cluster_name, region, cluster, report, customer_report): + loading_config(cluster_name, region) + print("\n") + log_pusher(log_details, cluster_name, region, "Fetching security group details .....") + print("Fetching security group details .....") + try: + security_groups = cluster["cluster"]["resourcesVpcConfig"]["securityGroupIds"] + if len(security_groups) == 0: + log_pusher(log_details, cluster_name, region, "No security groups available with cluster") + customer_report["security group"] = "No security groups available with cluster" + print("No security groups available with cluster") + else: for s in security_groups: - try : - ec2 = boto3.resource('ec2',region_name = region) + try: + ec2 = boto3.resource("ec2", region_name=region) security_group = ec2.SecurityGroup(s) y = security_group.description - customer_report['security group'] = ("Security Group " + security_group.id + " is present in VPC with ID" + security_group.vpc_id) - log_pusher(log_details,cluster_name,region,"Security Group " + security_group.id + " is present in VPC with ID " + security_group.vpc_id) + customer_report["security group"] = ( + "Security Group " + security_group.id + " is present in VPC with ID" + security_group.vpc_id + ) + log_pusher( + log_details, + cluster_name, + region, + "Security Group " + security_group.id + " is present in VPC with ID " + security_group.vpc_id, + ) print("Security Group " + security_group.id + " is present in VPC with ID " + security_group.vpc_id) - except : - customer_report['security group'] = ('The security group with id ' + s + " is not present") - report['preflight_status'] = False - errors.append('The security group with id ' + s + " is not present") - log_pusher(log_details,cluster_name,region,'The security group with id ' + s + " is not present") - print('The security group with id ' + s + " is not present") - except Exception as e: - - errors.append(' Error retireving security group of cluster {err}'.format(err=e)) - customer_report['security group'] = (' Error retireving security group of cluster {err}'.format(err=e)) - log_pusher(log_details,cluster_name,region,' Error retireving security group of cluster {err}'.format(err=e)) - print(' Error retireving security group of cluster {err}'.format(err=e)) - report['preflight_status'] = False - -#Check if the AMI is custom - -def iscustomami(node_type,Presentversion,image_id,region): + except: + customer_report["security group"] = "The security group with id " + s + " is not present" + report["preflight_status"] = False + errors.append("The security group with id " + s + " is not present") + log_pusher(log_details, cluster_name, region, "The security group with id " + s + " is not present") + print("The security group with id " + s + " is not present") + except Exception as e: + + errors.append(" Error retireving security group of cluster {err}".format(err=e)) + customer_report["security group"] = " Error retireving security group of cluster {err}".format(err=e) + log_pusher( + log_details, cluster_name, region, " Error retireving security group of cluster {err}".format(err=e) + ) + print(" Error retireving security group of cluster {err}".format(err=e)) + report["preflight_status"] = False + + +# Check if the AMI is custom + + +def iscustomami(node_type, Presentversion, image_id, region): # print(node_type) - all_images=[] - if node_type=="Amazon Linux 2": - filters = [{'Name':'owner-id','Values':['602401143452']},{'Name': 'name', 'Values': ["amazon-eks-node-{version}*".format(version=Presentversion)]},{'Name':'is-public','Values':['true']}] + all_images = [] + if node_type == "Amazon Linux 2": + filters = [ + {"Name": "owner-id", "Values": ["602401143452"]}, + {"Name": "name", "Values": ["amazon-eks-node-{version}*".format(version=Presentversion)]}, + {"Name": "is-public", "Values": ["true"]}, + ] elif "ubuntu" in node_type.lower(): - filters = [{'Name':'owner-id','Values':['099720109477']},{'Name': 'name', 'Values': ["ubuntu-eks/k8s_{version}*".format(version=Presentversion)]},{'Name':'is-public','Values':['true']}] + filters = [ + {"Name": "owner-id", "Values": ["099720109477"]}, + {"Name": "name", "Values": ["ubuntu-eks/k8s_{version}*".format(version=Presentversion)]}, + {"Name": "is-public", "Values": ["true"]}, + ] elif "bottlerocket" in node_type.lower(): - filters = [{'Name':'owner-id','Values':['092701018921']},{'Name': 'name', 'Values': ["bottlerocket-aws-k8s-{version}*".format(version=Presentversion)]},{'Name':'is-public','Values':['true']}] + filters = [ + {"Name": "owner-id", "Values": ["092701018921"]}, + {"Name": "name", "Values": ["bottlerocket-aws-k8s-{version}*".format(version=Presentversion)]}, + {"Name": "is-public", "Values": ["true"]}, + ] elif "windows" in node_type.lower(): - filters = [{'Name':'owner-id','Values':['801119661308']},{'Name': 'name', 'Values': ["Windows_Server-*-English-*-EKS_Optimized-{version}*".format(version=Presentversion)]},{'Name':'is-public','Values':['true']}] + filters = [ + {"Name": "owner-id", "Values": ["801119661308"]}, + { + "Name": "name", + "Values": ["Windows_Server-*-English-*-EKS_Optimized-{version}*".format(version=Presentversion)], + }, + {"Name": "is-public", "Values": ["true"]}, + ] else: return True - ec2Client = boto3.client('ec2',region_name = region) + ec2Client = boto3.client("ec2", region_name=region) images = ec2Client.describe_images(Filters=filters) - instances_list=[] - for i in images.get('Images'): - instances_list.append([i.get('ImageId'),i.get('ImageLocation')]) - #print(instances_list) + instances_list = [] + for i in images.get("Images"): + instances_list.append([i.get("ImageId"), i.get("ImageLocation")]) + # print(instances_list) for i in instances_list: if image_id in i[0]: return False @@ -970,285 +1338,441 @@ def iscustomami(node_type,Presentversion,image_id,region): # Print nodegroup details -def nodegroup_customami(log_details,errors,cluster_name,region,report,customer_report,update_version): - loading_config(cluster_name,region) - final_dict={'self-managed':{},'managed':{},'fargate':{}} - print('\n') - log_pusher(log_details,cluster_name,region,'Fetching node group details ......') - print('Fetching node group details ......') - try : + +def nodegroup_customami(log_details, errors, cluster_name, region, report, customer_report, update_version): + loading_config(cluster_name, region) + final_dict = {"self-managed": {}, "managed": {}, "fargate": {}} + print("\n") + log_pusher(log_details, cluster_name, region, "Fetching node group details ......") + print("Fetching node group details ......") + try: v1 = client.CoreV1Api() ret = v1.list_node() - #pprint(ret.items) + # pprint(ret.items) if len(ret.items) == 0: - raise Exception('No running nodes present in the cluster') + raise Exception("No running nodes present in the cluster") for i in ret.items: - x=i.metadata.labels - ver=i.status.node_info.kubelet_version.split('-')[0][1:5] - if('eks.amazonaws.com/compute-type' in x): - final_dict['fargate'][i.metadata.name]={ - 'version':ver, - 'node_type':'fargate', - 'version_compatibility': ver == report['cluster']['version'] + x = i.metadata.labels + ver = i.status.node_info.kubelet_version.split("-")[0][1:5] + if "eks.amazonaws.com/compute-type" in x: + final_dict["fargate"][i.metadata.name] = { + "version": ver, + "node_type": "fargate", + "version_compatibility": ver == report["cluster"]["version"], } - else : - instance_id=i.spec.provider_id.split('/')[-1] - #print(instance_id) - node_type=i.status.node_info.os_image - #print(node_type) - if 'windows' in (node_type).lower() : - node_type = 'windows' - ec2Client = boto3.client('ec2',region_name = region) - res=ec2Client.describe_instances(InstanceIds=[instance_id]) + else: + instance_id = i.spec.provider_id.split("/")[-1] + # print(instance_id) + node_type = i.status.node_info.os_image + # print(node_type) + if "windows" in (node_type).lower(): + node_type = "windows" + ec2Client = boto3.client("ec2", region_name=region) + res = ec2Client.describe_instances(InstanceIds=[instance_id]) - ami=res.get('Reservations')[0]['Instances'][0]['ImageId'] - #print(ami) + ami = res.get("Reservations")[0]["Instances"][0]["ImageId"] + # print(ami) - hd=res['Reservations'][0]['Instances'][0]['Tags'] + hd = res["Reservations"][0]["Instances"][0]["Tags"] for m in hd: - - if(m['Key']=='aws:autoscaling:groupName'): - autoscale_group_name=m['Value'] - #print(autoscale_group_name) - - custom_ami=iscustomami(node_type,ver,ami,region) - #print(custom_ami) - if(ver==report['cluster']['version']) : - version_compatibility=True - elif update_version and round(float(report['cluster']['version']) - float(ver),2) == 0.01 : + + if m["Key"] == "aws:autoscaling:groupName": + autoscale_group_name = m["Value"] + # print(autoscale_group_name) + + custom_ami = iscustomami(node_type, ver, ami, region) + # print(custom_ami) + if ver == report["cluster"]["version"]: + version_compatibility = True + elif update_version and round(float(report["cluster"]["version"]) - float(ver), 2) == 0.01: version_compatibility = True else: - version_compatibility=False - if custom_ami : - #errors.append(instance_id + ' cannot be upgraded as it has custom ami') - log_pusher(log_details,cluster_name,region,instance_id + ' cannot be upgraded as it has custom ami') - print(instance_id + ' cannot be upgraded as it has custom ami') + version_compatibility = False + if custom_ami: + # errors.append(instance_id + ' cannot be upgraded as it has custom ami') + log_pusher( + log_details, cluster_name, region, instance_id + " cannot be upgraded as it has custom ami" + ) + print(instance_id + " cannot be upgraded as it has custom ami") if not version_compatibility: - report['preflight_status'] = False - #errors.append(instance_id + ' cannot be upgraded as cluster version is not compatible with node version') - log_pusher(log_details,cluster_name,region,instance_id + ' cannot be upgraded as cluster version is not compatible with node version') - print(instance_id + ' cannot be upgraded as cluster version is not compatible with node version') - if('alpha.eksctl.io/instance-id' in x or 'eks.amazonaws.com/nodegroup' not in x): - if(autoscale_group_name not in final_dict['self-managed'].keys()): - final_dict['self-managed'][autoscale_group_name]={ - 'instances' : [{ - 'version':ver, - 'ami':ami, - 'node_type':node_type, - 'version_compatibility':version_compatibility, - 'custom_ami':custom_ami, - }] + report["preflight_status"] = False + # errors.append(instance_id + ' cannot be upgraded as cluster version is not compatible with node version') + log_pusher( + log_details, + cluster_name, + region, + instance_id + " cannot be upgraded as cluster version is not compatible with node version", + ) + print(instance_id + " cannot be upgraded as cluster version is not compatible with node version") + if "alpha.eksctl.io/instance-id" in x or "eks.amazonaws.com/nodegroup" not in x: + if autoscale_group_name not in final_dict["self-managed"].keys(): + final_dict["self-managed"][autoscale_group_name] = { + "instances": [ + { + "version": ver, + "ami": ami, + "node_type": node_type, + "version_compatibility": version_compatibility, + "custom_ami": custom_ami, + } + ] } - else : - instances = final_dict['self-managed'][autoscale_group_name]['instances'] - instances.append({ - 'version':ver, - 'ami':ami, - 'node_type':node_type, - 'version_compatibility':version_compatibility, - 'custom_ami':custom_ami, - }) - final_dict['self-managed'][autoscale_group_name]['instances'] = instances + else: + instances = final_dict["self-managed"][autoscale_group_name]["instances"] + instances.append( + { + "version": ver, + "ami": ami, + "node_type": node_type, + "version_compatibility": version_compatibility, + "custom_ami": custom_ami, + } + ) + final_dict["self-managed"][autoscale_group_name]["instances"] = instances else: - if(autoscale_group_name not in final_dict['managed'].keys()): + if autoscale_group_name not in final_dict["managed"].keys(): node_group_name = "" - if 'alpha.eksctl.io/nodegroup-name' in x : - node_group_name = x['alpha.eksctl.io/nodegroup-name'] - else : - node_group_name = x['eks.amazonaws.com/nodegroup'] - eks = boto3.client('eks',region_name = region) - response = eks.describe_nodegroup(clusterName=cluster_name,nodegroupName=node_group_name) - if response['nodegroup']['amiType']=='CUSTOM': - print('Nodegroup cannot be upgraded as it has custom launch template') - #errors.append('Nodegroup cannot be upgraded as it has custom launch template') - final_dict['managed'][autoscale_group_name]={ - 'nodegroup_name':node_group_name, - 'custom_launch_template':response['nodegroup']['amiType']=='CUSTOM', - 'instances' : [{ - 'version':ver, - 'ami':ami, - 'node_type':node_type, - 'version_compatibility':version_compatibility, - 'custom_ami':custom_ami, - }] + if "alpha.eksctl.io/nodegroup-name" in x: + node_group_name = x["alpha.eksctl.io/nodegroup-name"] + else: + node_group_name = x["eks.amazonaws.com/nodegroup"] + eks = boto3.client("eks", region_name=region) + response = eks.describe_nodegroup(clusterName=cluster_name, nodegroupName=node_group_name) + if response["nodegroup"]["amiType"] == "CUSTOM": + print("Nodegroup cannot be upgraded as it has custom launch template") + # errors.append('Nodegroup cannot be upgraded as it has custom launch template') + final_dict["managed"][autoscale_group_name] = { + "nodegroup_name": node_group_name, + "custom_launch_template": response["nodegroup"]["amiType"] == "CUSTOM", + "instances": [ + { + "version": ver, + "ami": ami, + "node_type": node_type, + "version_compatibility": version_compatibility, + "custom_ami": custom_ami, + } + ], } - else : - instances = final_dict['managed'][autoscale_group_name]['instances'] - instances.append({ - 'version':ver, - 'ami':ami, - 'node_type':node_type, - 'version_compatibility':version_compatibility, - 'custom_ami':custom_ami, - }) - final_dict['managed'][autoscale_group_name]['instances'] = instances + else: + instances = final_dict["managed"][autoscale_group_name]["instances"] + instances.append( + { + "version": ver, + "ami": ami, + "node_type": node_type, + "version_compatibility": version_compatibility, + "custom_ami": custom_ami, + } + ) + final_dict["managed"][autoscale_group_name]["instances"] = instances return final_dict - except Exception as e : - - errors.append('Error ocurred while checking node group details {err}'.format(err=e)) - log_pusher(log_details,cluster_name,region,'Error ocurred while checking node group details {err}'.format(err=e)) - print('Error ocurred while checking node group details {err}'.format(err=e)) - customer_report['node group details'] = 'Error ocurred while checking node group details' - report['preflight_status'] = False + except Exception as e: + errors.append("Error ocurred while checking node group details {err}".format(err=e)) + log_pusher( + log_details, cluster_name, region, "Error ocurred while checking node group details {err}".format(err=e) + ) + print("Error ocurred while checking node group details {err}".format(err=e)) + customer_report["node group details"] = "Error ocurred while checking node group details" + report["preflight_status"] = False + + +# Publish a preflight report via SES -# Publish a preflight report via SES -def send_email(preflight,log_details,errors,cluster_name,region,report,customer_report,email): - try : +def send_email(preflight, log_details, errors, cluster_name, region, report, customer_report, email): + try: ses_client = boto3.client("ses", region_name="ap-south-1") # response = ses_client.list_identities() htmlStart = 'Document' - htmlend = '' - if preflight : - htmlStart = htmlStart + '

Preflight check for cluster ' + cluster_name + ' in region ' + region + ' : ' + str(report['preflight_status'])+ '

' - else : - htmlStart = htmlStart + '

Postflight check for cluster ' + cluster_name + ' in region ' + region + ' : ' + str(report['preflight_status'])+ '

' - if report['preflight_status']: - if preflight : - htmlStart = htmlStart + '

The preflight check for your cluster completed successfully

' - else : - htmlStart = htmlStart + '

The postflight check for your cluster completed successfully

' - htmlStart = htmlStart + 'All things upgraded successfully' - - else : - if preflight : - htmlStart = htmlStart + '

The preflight check for your cluster failed, please check the below report for more details

' - else : - htmlStart = htmlStart + '

The postflight check for your cluster failed, please check the below report for more details

' - htmlStart = htmlStart + 'Certain things couldn’t be upgraded during the upgrade' - - htmlStart = htmlStart + '

General Check details

' - - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - if 'cluster upgradation' in customer_report.keys() : - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - if 'depricated Api' in customer_report.keys() : - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '
NameStatus
CMK Key' + customer_report['CMK Key']+'
IAM role'+ customer_report['IAM role']+'
cluster autoscaler' + customer_report['cluster autoscaler']+'
cluster upgradation' + customer_report['cluster upgradation']+'
Cluster Roles
    ' - for s in customer_report['cluster role'] : - htmlStart = htmlStart + '
  • ' + str(s) + '
    • ' - htmlStart = htmlStart +'
cluster version' + customer_report['cluster version']+'
depricated Api' + str(customer_report['depricated Api'])+'
horizontal auto scaler' + customer_report['horizontal auto scaler']+'
pod disruption budget' + str(customer_report['pod disruption budget'])+'
pod security policy' + customer_report['pod security policy']+'
security group' + customer_report['security group']+'
subnet
    ' - for s in customer_report['subnet'] : - htmlStart = htmlStart + '
  • ' + str(s) + '
    • ' - htmlStart = htmlStart +'
' - - - htmlStart = htmlStart + '

Addons details

' - if 'coredns' in customer_report['addons'].keys() and len(customer_report['addons']['coredns'].keys()) != 0 : - if 'image' in customer_report['addons']['coredns'].keys(): - htmlStart = htmlStart + '

Core DNS

' + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '
CorefileImageMount PathVersion
' + customer_report['addons']['coredns']['corefile']['message'] + '
    ' - if 'default-corefile-fields' in customer_report['addons']['coredns']['corefile'].keys() : - htmlStart = htmlStart + '
  • default-corefile-fields ' + str(customer_report['addons']['coredns']['corefile']['default-corefile-fields']) + '
    • ' - if 'userdefined-corefile-fields' in customer_report['addons']['coredns']['corefile'].keys() : - htmlStart = htmlStart + '
  • userdefined-corefile-fields ' + str(customer_report['addons']['coredns']['corefile']['userdefined-corefile-fields']) + '
    • ' - htmlStart = htmlStart + '
' + customer_report['addons']['coredns']['image'] + '' + customer_report['addons']['coredns']['mount_paths']['message'] + '
    ' - if 'default-mountpaths' in customer_report['addons']['coredns']['mount_paths'].keys() : - htmlStart = htmlStart + '
  • default-mountpaths ' + str(customer_report['addons']['coredns']['mount_paths']['default-mountpaths']) + '
    • ' - if 'userdefined-mountpaths' in customer_report['addons']['coredns']['mount_paths'].keys() : - htmlStart = htmlStart + '
  • userdefined-mountpaths ' + str(customer_report['addons']['coredns']['mount_paths']['userdefined-mountpaths']) + '
    • ' - htmlStart = htmlStart + '
' + customer_report['addons']['coredns']['version'] + '
' - else : - htmlStart = htmlStart + '

Core DNS

' - htmlStart = htmlStart + '

' + customer_report['addons']['coredns']['version'] + '

' - - - if 'kube-proxy' in customer_report['addons'].keys() and len(customer_report['addons']['kube-proxy'].keys()) != 0 : - if 'image' in customer_report['addons']['kube-proxy'].keys(): - htmlStart = htmlStart + '

Kube Proxy

' + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '
Certificate AuthorityImageMount PathVersionServer Endpoint
' + customer_report['addons']['kube-proxy']['certificate-authority']['message'] + '' + customer_report['addons']['kube-proxy']['image'] + '' + customer_report['addons']['kube-proxy']['mount_paths']['message'] + '
    ' - if 'default-mountpaths' in customer_report['addons']['kube-proxy']['mount_paths'].keys() : - htmlStart = htmlStart + '
  • default-mountpaths ' + str(customer_report['addons']['kube-proxy']['mount_paths']['default-mountpaths']) + '
    • ' - if 'userdefined-mountpaths' in customer_report['addons']['kube-proxy']['mount_paths'].keys() : - htmlStart = htmlStart + '
  • userdefined-mountpaths ' + str(customer_report['addons']['kube-proxy']['mount_paths']['userdefined-mountpaths']) + '
    • ' - htmlStart = htmlStart + '
' + customer_report['addons']['kube-proxy']['version'] + '' + customer_report['addons']['kube-proxy']['server-endpoint']['message'] + '
' - else : - htmlStart = htmlStart + '

Kube Proxy

' - htmlStart = htmlStart + '

' + customer_report['addons']['kube-proxy']['version'] + '

' - - - if 'vpc-cni' in customer_report['addons'].keys() and len(customer_report['addons']['vpc-cni'].keys()) != 0 : - if 'image' in customer_report['addons']['vpc-cni'] : - htmlStart = htmlStart + '

VPC Cni

' + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '
ImageMount PathVersionEnv
' + customer_report['addons']['vpc-cni']['image'] + '' + customer_report['addons']['vpc-cni']['mount_paths']['message'] + '
    ' - if 'default-mountpaths' in customer_report['addons']['vpc-cni']['mount_paths'].keys() : - htmlStart = htmlStart + '
  • default-mountpaths ' + str(customer_report['addons']['vpc-cni']['mount_paths']['default-mountpaths']) + '
    • ' - if 'userdefined-mountpaths' in customer_report['addons']['vpc-cni']['mount_paths'].keys() : - htmlStart = htmlStart + '
  • userdefined-mountpaths ' + str(customer_report['addons']['vpc-cni']['mount_paths']['userdefined-mountpaths']) + '
    • ' - htmlStart = htmlStart + '
' + customer_report['addons']['vpc-cni']['version'] + '' + customer_report['addons']['vpc-cni']['env']['message'] + '
    ' - if 'default-envs' in customer_report['addons']['vpc-cni']['env'].keys() : - htmlStart = htmlStart + '
  • default-envs ' + str(customer_report['addons']['vpc-cni']['env']['default-envs']) + '
    • ' - if 'userdefined-envs' in customer_report['addons']['vpc-cni']['env'].keys() : - htmlStart = htmlStart + '
  • userdefined-envs ' + str(customer_report['addons']['vpc-cni']['env']['userdefined-envs']) + '
    • ' - htmlStart = htmlStart + '
' - else : - htmlStart = htmlStart + '

VPC Cni

' - htmlStart = htmlStart + '

' + customer_report['addons']['vpc-cni']['version'] + '

' - if customer_report['nodegroup_details']: - node_groups = customer_report['nodegroup_details'] - htmlStart = htmlStart + '

Node groups

' - htmlStart = htmlStart + '' + htmlend = "" + if preflight: + htmlStart = ( + htmlStart + + "

Preflight check for cluster " + + cluster_name + + " in region " + + region + + " : " + + str(report["preflight_status"]) + + "

" + ) + else: + htmlStart = ( + htmlStart + + "

Postflight check for cluster " + + cluster_name + + " in region " + + region + + " : " + + str(report["preflight_status"]) + + "

" + ) + if report["preflight_status"]: + if preflight: + htmlStart = htmlStart + "

The preflight check for your cluster completed successfully

" + else: + htmlStart = htmlStart + "

The postflight check for your cluster completed successfully

" + htmlStart = htmlStart + "All things upgraded successfully" + + else: + if preflight: + htmlStart = ( + htmlStart + + "

The preflight check for your cluster failed, please check the below report for more details

" + ) + else: + htmlStart = ( + htmlStart + + "

The postflight check for your cluster failed, please check the below report for more details

" + ) + htmlStart = htmlStart + "Certain things couldn’t be upgraded during the upgrade" + + htmlStart = ( + htmlStart + + "

General Check details

Node group typeAutoscaling groupCustom launch templateAMI imageCustom AMINode typeVersionVersion Compatibility
" + ) + + htmlStart = htmlStart + "" + htmlStart = htmlStart + "" + htmlStart = ( + htmlStart + "" + ) + if "cluster upgradation" in customer_report.keys(): + htmlStart = ( + htmlStart + + "" + ) + htmlStart = htmlStart + "" + htmlStart = htmlStart + "" + if "depricated Api" in customer_report.keys(): + htmlStart = ( + htmlStart + "" + ) + htmlStart = ( + htmlStart + + "" + ) + htmlStart = ( + htmlStart + + "" + ) + htmlStart = ( + htmlStart + "" + ) + htmlStart = htmlStart + "" + htmlStart = htmlStart + "" + htmlStart = htmlStart + "
NameStatus
CMK Key" + customer_report["CMK Key"] + "
IAM role" + customer_report["IAM role"] + "
cluster autoscaler" + customer_report["cluster autoscaler"] + "
cluster upgradation" + + customer_report["cluster upgradation"] + + "
Cluster Roles
    " + for s in customer_report["cluster role"]: + htmlStart = htmlStart + "
  • " + str(s) + "
    • " + htmlStart = htmlStart + "
cluster version" + customer_report["cluster version"] + "
depricated Api" + str(customer_report["depricated Api"]) + "
horizontal auto scaler" + + customer_report["horizontal auto scaler"] + + "
pod disruption budget" + + str(customer_report["pod disruption budget"]) + + "
pod security policy" + customer_report["pod security policy"] + "
security group" + customer_report["security group"] + "
subnet
    " + for s in customer_report["subnet"]: + htmlStart = htmlStart + "
  • " + str(s) + "
    • " + htmlStart = htmlStart + "
" + + htmlStart = htmlStart + "

Addons details

" + if "coredns" in customer_report["addons"].keys() and len(customer_report["addons"]["coredns"].keys()) != 0: + if "image" in customer_report["addons"]["coredns"].keys(): + htmlStart = ( + htmlStart + + "

Core DNS

" + + "" + ) + htmlStart = htmlStart + "" + htmlStart = htmlStart + "" + htmlStart = htmlStart + "
CorefileImageMount PathVersion
" + customer_report["addons"]["coredns"]["corefile"]["message"] + "
    " + if "default-corefile-fields" in customer_report["addons"]["coredns"]["corefile"].keys(): + htmlStart = ( + htmlStart + + "
  • default-corefile-fields " + + str(customer_report["addons"]["coredns"]["corefile"]["default-corefile-fields"]) + + "
    • " + ) + if "userdefined-corefile-fields" in customer_report["addons"]["coredns"]["corefile"].keys(): + htmlStart = ( + htmlStart + + "
  • userdefined-corefile-fields " + + str(customer_report["addons"]["coredns"]["corefile"]["userdefined-corefile-fields"]) + + "
    • " + ) + htmlStart = htmlStart + "
" + customer_report["addons"]["coredns"]["image"] + "" + customer_report["addons"]["coredns"]["mount_paths"]["message"] + "
    " + if "default-mountpaths" in customer_report["addons"]["coredns"]["mount_paths"].keys(): + htmlStart = ( + htmlStart + + "
  • default-mountpaths " + + str(customer_report["addons"]["coredns"]["mount_paths"]["default-mountpaths"]) + + "
    • " + ) + if "userdefined-mountpaths" in customer_report["addons"]["coredns"]["mount_paths"].keys(): + htmlStart = ( + htmlStart + + "
  • userdefined-mountpaths " + + str(customer_report["addons"]["coredns"]["mount_paths"]["userdefined-mountpaths"]) + + "
    • " + ) + htmlStart = htmlStart + "
" + customer_report["addons"]["coredns"]["version"] + "
" + else: + htmlStart = htmlStart + "

Core DNS

" + htmlStart = htmlStart + "

" + customer_report["addons"]["coredns"]["version"] + "

" + + if ( + "kube-proxy" in customer_report["addons"].keys() + and len(customer_report["addons"]["kube-proxy"].keys()) != 0 + ): + if "image" in customer_report["addons"]["kube-proxy"].keys(): + htmlStart = ( + htmlStart + + "

Kube Proxy

" + + "" + ) + htmlStart = ( + htmlStart + + "" + ) + htmlStart = htmlStart + "" + htmlStart = ( + htmlStart + "" + htmlStart = ( + htmlStart + "" + ) + htmlStart = htmlStart + "
Certificate AuthorityImageMount PathVersionServer Endpoint
" + + customer_report["addons"]["kube-proxy"]["certificate-authority"]["message"] + + "" + customer_report["addons"]["kube-proxy"]["image"] + "" + customer_report["addons"]["kube-proxy"]["mount_paths"]["message"] + "
    " + ) + if "default-mountpaths" in customer_report["addons"]["kube-proxy"]["mount_paths"].keys(): + htmlStart = ( + htmlStart + + "
  • default-mountpaths " + + str(customer_report["addons"]["kube-proxy"]["mount_paths"]["default-mountpaths"]) + + "
    • " + ) + if "userdefined-mountpaths" in customer_report["addons"]["kube-proxy"]["mount_paths"].keys(): + htmlStart = ( + htmlStart + + "
  • userdefined-mountpaths " + + str(customer_report["addons"]["kube-proxy"]["mount_paths"]["userdefined-mountpaths"]) + + "
    • " + ) + htmlStart = htmlStart + "
" + customer_report["addons"]["kube-proxy"]["version"] + "" + customer_report["addons"]["kube-proxy"]["server-endpoint"]["message"] + "
" + else: + htmlStart = htmlStart + "

Kube Proxy

" + htmlStart = htmlStart + "

" + customer_report["addons"]["kube-proxy"]["version"] + "

" + + if "vpc-cni" in customer_report["addons"].keys() and len(customer_report["addons"]["vpc-cni"].keys()) != 0: + if "image" in customer_report["addons"]["vpc-cni"]: + htmlStart = ( + htmlStart + + "

VPC Cni

" + + "" + ) + htmlStart = htmlStart + "" + htmlStart = htmlStart + "" + htmlStart = htmlStart + "
ImageMount PathVersionEnv
" + customer_report["addons"]["vpc-cni"]["image"] + "" + customer_report["addons"]["vpc-cni"]["mount_paths"]["message"] + "
    " + if "default-mountpaths" in customer_report["addons"]["vpc-cni"]["mount_paths"].keys(): + htmlStart = ( + htmlStart + + "
  • default-mountpaths " + + str(customer_report["addons"]["vpc-cni"]["mount_paths"]["default-mountpaths"]) + + "
    • " + ) + if "userdefined-mountpaths" in customer_report["addons"]["vpc-cni"]["mount_paths"].keys(): + htmlStart = ( + htmlStart + + "
  • userdefined-mountpaths " + + str(customer_report["addons"]["vpc-cni"]["mount_paths"]["userdefined-mountpaths"]) + + "
    • " + ) + htmlStart = htmlStart + "
" + customer_report["addons"]["vpc-cni"]["version"] + "" + customer_report["addons"]["vpc-cni"]["env"]["message"] + "
    " + if "default-envs" in customer_report["addons"]["vpc-cni"]["env"].keys(): + htmlStart = ( + htmlStart + + "
  • default-envs " + + str(customer_report["addons"]["vpc-cni"]["env"]["default-envs"]) + + "
    • " + ) + if "userdefined-envs" in customer_report["addons"]["vpc-cni"]["env"].keys(): + htmlStart = ( + htmlStart + + "
  • userdefined-envs " + + str(customer_report["addons"]["vpc-cni"]["env"]["userdefined-envs"]) + + "
    • " + ) + htmlStart = htmlStart + "
" + else: + htmlStart = htmlStart + "

VPC Cni

" + htmlStart = htmlStart + "

" + customer_report["addons"]["vpc-cni"]["version"] + "

" + if customer_report["nodegroup_details"]: + node_groups = customer_report["nodegroup_details"] + htmlStart = htmlStart + "

Node groups

" + htmlStart = ( + htmlStart + + "" + ) for k in node_groups.keys(): - if k == 'fargate' and len(node_groups[k].keys()) > 0: + if k == "fargate" and len(node_groups[k].keys()) > 0: for id in node_groups[k].keys(): - htmlStart = htmlStart + '' + '' - if k == 'managed' and len(node_groups[k].keys()) > 0: + htmlStart = ( + htmlStart + + "" + + "" + ) + if k == "managed" and len(node_groups[k].keys()) > 0: + for autoscaling in node_groups[k].keys(): + for n in node_groups[k][autoscaling]: + if n == "instances": + for inst in node_groups[k][autoscaling][n]: + htmlStart = htmlStart + "" + htmlStart = htmlStart + "" + htmlStart = ( + htmlStart + + "" + ) + htmlStart = htmlStart + "" + htmlStart = htmlStart + "" + htmlStart = htmlStart + "" + htmlStart = htmlStart + "" + htmlStart = htmlStart + "" + htmlend = htmlend + "" + if k == "self-managed" and len(node_groups[k].keys()) > 0: for autoscaling in node_groups[k].keys(): for n in node_groups[k][autoscaling]: - if n == 'instances' : - for inst in node_groups[k][autoscaling][n] : - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - htmlend = htmlend + '' - if k == 'self-managed' and len(node_groups[k].keys()) > 0: - for autoscaling in node_groups[k].keys(): - for n in node_groups[k][autoscaling]: - if n == 'instances' : - for inst in node_groups[k][autoscaling][n] : - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - htmlStart = htmlStart + '' - htmlend = htmlend + '' - - htmlend = htmlend + '
Node group typeAutoscaling groupCustom launch templateAMI imageCustom AMINode typeVersionVersion Compatibility
Fargate--------' + id + '' + node_groups[k][id]['version'] + '' + str(node_groups[k][id]['version_compatibility']) + '
Fargate--------" + + id + + "" + + node_groups[k][id]["version"] + + "" + + str(node_groups[k][id]["version_compatibility"]) + + "
Managed" + autoscaling + "" + + str(node_groups[k][autoscaling]["custom_launch_template"]) + + "" + str(inst["ami"]) + "" + str(inst["custom_ami"]) + "" + str(inst["node_type"]) + "" + str(inst["version"]) + "" + str(inst["version_compatibility"]) + "
Managed' + autoscaling + '' + str(node_groups[k][autoscaling]["custom_launch_template"]) + '' + str(inst['ami']) + '' + str(inst['custom_ami']) + '' + str(inst['node_type']) + '' + str(inst['version']) + '' + str(inst['version_compatibility']) + '
Self Managed' + autoscaling + '' + '--' + '' + str(inst['ami']) + '' + str(inst['custom_ami']) + '' + str(inst['node_type']) + '' + str(inst['version']) + '' + str(inst['version_compatibility']) + '
' + if n == "instances": + for inst in node_groups[k][autoscaling][n]: + htmlStart = htmlStart + "Self Managed" + htmlStart = htmlStart + "" + autoscaling + "" + htmlStart = htmlStart + "" + "--" + "" + htmlStart = htmlStart + "" + str(inst["ami"]) + "" + htmlStart = htmlStart + "" + str(inst["custom_ami"]) + "" + htmlStart = htmlStart + "" + str(inst["node_type"]) + "" + htmlStart = htmlStart + "" + str(inst["version"]) + "" + htmlStart = htmlStart + "" + str(inst["version_compatibility"]) + "" + htmlend = htmlend + "" + + htmlend = htmlend + "" CHARSET = "UTF-8" HTML_EMAIL_CONTENT = htmlStart + htmlend - if preflight : - subject = "Preflight check report for cluster " + cluster_name + ' in region ' + region - else : - subject = "Postflight check report for cluster " + cluster_name + ' in region ' + region - try : + if preflight: + subject = "Preflight check report for cluster " + cluster_name + " in region " + region + else: + subject = "Postflight check report for cluster " + cluster_name + " in region " + region + try: response = ses_client.send_email( Destination={ "ToAddresses": [ @@ -1269,14 +1793,8 @@ def send_email(preflight,log_details,errors,cluster_name,region,report,customer_ }, Source=email, ) - print('It may take sometime for the user to get email delivered if verified') - except : - print('The email given is not verified by user to share the report over it') - except Exception as e : + print("It may take sometime for the user to get email delivered if verified") + except: + print("The email given is not verified by user to share the report over it") + except Exception as e: print("Error occurred while sharing email {err}".format(err=e)) - - - - - - diff --git a/eksupdate/src/self_managed.py b/eksupdate/src/self_managed.py index d40c7af..6180f69 100644 --- a/eksupdate/src/self_managed.py +++ b/eksupdate/src/self_managed.py @@ -1,179 +1,191 @@ -from platform import node -import boto3 import time -from .latest_ami import get_latestami -from botocore.utils import instance_cache + +import boto3 + from .ekslogs import logs_pusher +from .latest_ami import get_latestami + -def status_of_cluster(Clustname,regionName): - client = boto3.client('eks',region_name=regionName) - ''' Getting Self Managed Cluster Status''' - response = client.describe_cluster( - name=Clustname +def status_of_cluster(Clustname, regionName): + client = boto3.client("eks", region_name=regionName) + """ Getting Self Managed Cluster Status""" + response = client.describe_cluster(name=Clustname) + logs_pusher( + regionName=regionName, + cluster_name=Clustname, + msg="The Cluster Status = {stat} and Version = {ver}".format( + stat=response["cluster"]["status"], ver=response["cluster"]["version"] + ), ) - logs_pusher(regionName=regionName,cluster_name=Clustname,msg="The Cluster Status = {stat} and Version = {ver}".format(stat=response["cluster"]["status"],ver=response["cluster"]["version"])) return [response["cluster"]["status"], response["cluster"]["version"]] -def get_node_groups(Clustername,regionName): - client = boto3.client('eks',region_name=regionName) - ''' Getting Node Group list''' - response =client.list_nodegroups( - clusterName=Clustername, - maxResults=100 - ) - - return response['nodegroups'] - -def Desc_node_groups(Clustername,Nodegroup,regionName): - client = boto3.client('eks',region_name=regionName) - ''' Getting Descrption of Node Gorup ''' - response=client.describe_nodegroup( - clusterName=Clustername, - nodegroupName=Nodegroup) + +def get_node_groups(Clustername, regionName): + client = boto3.client("eks", region_name=regionName) + """ Getting Node Group list""" + response = client.list_nodegroups(clusterName=Clustername, maxResults=100) + + return response["nodegroups"] + + +def Desc_node_groups(Clustername, Nodegroup, regionName): + client = boto3.client("eks", region_name=regionName) + """ Getting Descrption of Node Gorup """ + response = client.describe_nodegroup(clusterName=Clustername, nodegroupName=Nodegroup) # print(response) # print(response.get('nodegroup')['version']) - logs_pusher(regionName=regionName,cluster_name=Clustername,msg="The NodeGroup = {ng} Status = {stat} and Version = {ver}".format(ng=Nodegroup,stat=response.get('nodegroup')['status'],ver=response.get('nodegroup')['version'])) - return [response.get('nodegroup')['status'],response.get('nodegroup')['version']] - -def get_asg_node_groups(Clustername,regionName): - client = boto3.client('eks',region_name=regionName) - ''' Getting asg of self managed node groups ''' - asg_groups=[] - node_groups=get_node_groups(Clustername,regionName) - if len(node_groups)==0: + logs_pusher( + regionName=regionName, + cluster_name=Clustername, + msg="The NodeGroup = {ng} Status = {stat} and Version = {ver}".format( + ng=Nodegroup, stat=response.get("nodegroup")["status"], ver=response.get("nodegroup")["version"] + ), + ) + return [response.get("nodegroup")["status"], response.get("nodegroup")["version"]] + + +def get_asg_node_groups(Clustername, regionName): + client = boto3.client("eks", region_name=regionName) + """ Getting asg of self managed node groups """ + asg_groups = [] + node_groups = get_node_groups(Clustername, regionName) + if len(node_groups) == 0: return [] for ng in node_groups: - response=client.describe_nodegroup(clusterName=Clustername,nodegroupName=ng).get('nodegroup')["resources"]["autoScalingGroups"] + response = client.describe_nodegroup(clusterName=Clustername, nodegroupName=ng).get("nodegroup")["resources"][ + "autoScalingGroups" + ] for asg_name in response: asg_groups.append(asg_name["name"]) - logs_pusher(regionName=regionName,cluster_name=Clustername,msg="The Asg's Of Node Groups ".format(inst=asg_groups)) + logs_pusher( + regionName=regionName, cluster_name=Clustername, msg="The Asg's Of Node Groups ".format(inst=asg_groups) + ) - return asg_groups -def filter_node_groups(cluster_name,node_list,latest_version,regionName): - client = boto3.client('eks',region_name=regionName) - ''' filtering Node groups ''' - old_ng=[] +def filter_node_groups(cluster_name, node_list, latest_version, regionName): + client = boto3.client("eks", region_name=regionName) + """ filtering Node groups """ + old_ng = [] for ng in node_list: - print("filter node group ",ng) - status,version=Desc_node_groups(Clustername=cluster_name,Nodegroup=ng,regionName=regionName) - if (status=="ACTIVE" or status=="UPDATING") and not version ==latest_version: + print("filter node group ", ng) + status, version = Desc_node_groups(Clustername=cluster_name, Nodegroup=ng, regionName=regionName) + if (status == "ACTIVE" or status == "UPDATING") and not version == latest_version: old_ng.append(ng) - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg="The Old Manged Node Groups Found Are {inst} ".format(inst=old_ng)) + logs_pusher( + regionName=regionName, + cluster_name=cluster_name, + msg="The Old Manged Node Groups Found Are {inst} ".format(inst=old_ng), + ) return old_ng -def lt_id_func(Clustername,Nodegroup,Version,regionName): +def lt_id_func(Clustername, Nodegroup, Version, regionName): - client = boto3.client('eks',region_name=regionName) - ec2=boto3.client('ec2',region_name=regionName) - res=client.describe_nodegroup( - clusterName=Clustername, - nodegroupName=Nodegroup - ) - Lt_id="" - version_no="" - AmiType=res['nodegroup']['amiType'] + client = boto3.client("eks", region_name=regionName) + ec2 = boto3.client("ec2", region_name=regionName) + res = client.describe_nodegroup(clusterName=Clustername, nodegroupName=Nodegroup) + Lt_id = "" + version_no = "" + AmiType = res["nodegroup"]["amiType"] if res["nodegroup"].get("launchTemplate"): - Lt_id,version_no=res['nodegroup']['launchTemplate']['id'],res['nodegroup']['launchTemplate']['version'] - os_lt=ec2.describe_launch_template_versions( - LaunchTemplateId=Lt_id, - Versions=[version_no]) - latest_ami="" - if AmiType=="CUSTOM": - current_ami=os_lt['LaunchTemplateVersions'][0]['LaunchTemplateData']['ImageId'] - - os_type=ec2.describe_images( - - ImageIds=[ - current_ami, - ], - )['Images'][0]['ImageLocation'] - - if isinstance(os_type,str) and "Windows_Server" in os_type: - os_type=os_type[:46] - - latest_ami=get_latestami(clustVersion=Version,instancetype=os_type,image_to_search=os_type,region_Name=regionName) - - return AmiType,Lt_id,version_no,latest_ami - -def update_current_launch_template_ami(lt_id,latest_ami,regionName): - - ec2 = boto3.client('ec2',region_name=regionName) - response = ec2.create_launch_template_version( - LaunchTemplateId=lt_id, - SourceVersion="$Latest", - VersionDescription="Latest-AMI", - LaunchTemplateData={ - "ImageId": latest_ami - } + Lt_id, version_no = res["nodegroup"]["launchTemplate"]["id"], res["nodegroup"]["launchTemplate"]["version"] + os_lt = ec2.describe_launch_template_versions(LaunchTemplateId=Lt_id, Versions=[version_no]) + latest_ami = "" + if AmiType == "CUSTOM": + current_ami = os_lt["LaunchTemplateVersions"][0]["LaunchTemplateData"]["ImageId"] + + os_type = ec2.describe_images(ImageIds=[current_ami,],)["Images"][ + 0 + ]["ImageLocation"] + + if isinstance(os_type, str) and "Windows_Server" in os_type: + os_type = os_type[:46] + + latest_ami = get_latestami( + clustVersion=Version, instancetype=os_type, image_to_search=os_type, region_Name=regionName ) - print(f"New launch template created with AMI {latest_ami}") + return AmiType, Lt_id, version_no, latest_ami -def Update_nodeGroup(Clustername,Nodegroup,Version,regionName): - client = boto3.client('eks',region_name=regionName) - start=time.time() - ''' updating Node group ''' - ami_type,lt_id,old_version,latest_ami=lt_id_func(Clustername,Nodegroup,Version,regionName) - if ami_type=="CUSTOM": - update_current_launch_template_ami(lt_id,latest_ami,regionName) +def update_current_launch_template_ami(lt_id, latest_ami, regionName): + + ec2 = boto3.client("ec2", region_name=regionName) + response = ec2.create_launch_template_version( + LaunchTemplateId=lt_id, + SourceVersion="$Latest", + VersionDescription="Latest-AMI", + LaunchTemplateData={"ImageId": latest_ami}, + ) + print(f"New launch template created with AMI {latest_ami}") +def Update_nodeGroup(Clustername, Nodegroup, Version, regionName): + client = boto3.client("eks", region_name=regionName) + start = time.time() + """ updating Node group """ + + ami_type, lt_id, old_version, latest_ami = lt_id_func(Clustername, Nodegroup, Version, regionName) + if ami_type == "CUSTOM": + update_current_launch_template_ami(lt_id, latest_ami, regionName) while True: try: - if status_of_cluster(Clustername,regionName)[0]=="ACTIVE" and Desc_node_groups(Clustername,Nodegroup,regionName)[0]=="ACTIVE" and Desc_node_groups(Clustername,Nodegroup,regionName)[1]!=Version: - if ami_type=="CUSTOM": + if ( + status_of_cluster(Clustername, regionName)[0] == "ACTIVE" + and Desc_node_groups(Clustername, Nodegroup, regionName)[0] == "ACTIVE" + and Desc_node_groups(Clustername, Nodegroup, regionName)[1] != Version + ): + if ami_type == "CUSTOM": client.update_nodegroup_version( clusterName=Clustername, - nodegroupName=Nodegroup, - launchTemplate={ - - 'version':"$Latest", - 'id': lt_id - }, - - + nodegroupName=Nodegroup, + launchTemplate={"version": "$Latest", "id": lt_id}, ) else: client.update_nodegroup_version( clusterName=Clustername, - nodegroupName=Nodegroup, - version=Version, + nodegroupName=Nodegroup, + version=Version, ) started = time.time() - print("Updating Node Group ",Nodegroup) + print("Updating Node Group ", Nodegroup) time.sleep(20) - if Desc_node_groups(Clustername,Nodegroup,regionName)[0]=="UPDATING": - end=time.time() - hours, rem = divmod(end-start, 3600) + if Desc_node_groups(Clustername, Nodegroup, regionName)[0] == "UPDATING": + end = time.time() + hours, rem = divmod(end - start, 3600) minutes, seconds = divmod(rem, 60) - print("The {Ng}".format(Ng=Nodegroup)+" NodeGroup is Still Updating ","{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) + print( + "The {Ng}".format(Ng=Nodegroup) + " NodeGroup is Still Updating ", + "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds), + ) time.sleep(20) - if Desc_node_groups(Clustername,Nodegroup,regionName)[0]=="DEGRADED": - raise Exception("NodeGroup has not started due to unavailability ") - if Desc_node_groups(Clustername,Nodegroup,regionName)[0]=="ACTIVE" and Desc_node_groups(Clustername,Nodegroup,regionName)[1]==Version: - end=time.time() - hours, rem = divmod(end-start, 3600) - minutes, seconds = divmod(rem, 60) - print("The Time Taken For the NodeGroup Upgrade "+str(Nodegroup),"{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) - - logs_pusher(regionName=regionName,cluster_name=Clustername,msg="The Taken For the NodeGroup Upgrade "+"{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) - return True - + if Desc_node_groups(Clustername, Nodegroup, regionName)[0] == "DEGRADED": + raise Exception("NodeGroup has not started due to unavailability ") + if ( + Desc_node_groups(Clustername, Nodegroup, regionName)[0] == "ACTIVE" + and Desc_node_groups(Clustername, Nodegroup, regionName)[1] == Version + ): + end = time.time() + hours, rem = divmod(end - start, 3600) + minutes, seconds = divmod(rem, 60) + print( + "The Time Taken For the NodeGroup Upgrade " + str(Nodegroup), + "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds), + ) + + logs_pusher( + regionName=regionName, + cluster_name=Clustername, + msg="The Taken For the NodeGroup Upgrade " + + "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds), + ) + return True + except Exception as e: print(e) raise Exception(e) - - - - - - - diff --git a/eksupdate/starter.py b/eksupdate/starter.py index 8f2f627..4586d5d 100644 --- a/eksupdate/starter.py +++ b/eksupdate/starter.py @@ -1,294 +1,408 @@ -from .src.boto_aws import update_cluster, is_cluster_exists, get_Asgs, outdated_lt, addAutoScaling, add_node, get_num_of_instances, get_latest_instance, wait_for_ready, get_outdated_Asg, worker_terminate, status_of_cluster,check_asg_autoscaler,enable_disable_autoscaler -import time import datetime -from .src.k8s_client import find_node, drain_nodes, unschedule_old_nodes, update_addons, delete_node,is_cluster_auto_scaler_present,clus_auto_enable_disable -from .src.self_managed import get_node_groups, Update_nodeGroup, get_asg_node_groups,filter_node_groups -from .src.latest_ami import get_latestami -from .src.eks_get_image_type import get_ami_name -from .src.preflight_module import pre_flight_checks -from .src.ekslogs import logs_pusher -from .src.eksctlfinal import eksctl_execute import queue import threading +import time + +from .src.boto_aws import ( + add_node, + addAutoScaling, + get_Asgs, + get_latest_instance, + get_num_of_instances, + get_outdated_Asg, + is_cluster_exists, + outdated_lt, + status_of_cluster, + update_cluster, + wait_for_ready, + worker_terminate, +) +from .src.eks_get_image_type import get_ami_name +from .src.eksctlfinal import eksctl_execute +from .src.ekslogs import logs_pusher +from .src.k8s_client import ( + clus_auto_enable_disable, + delete_node, + drain_nodes, + find_node, + is_cluster_auto_scaler_present, + unschedule_old_nodes, + update_addons, +) +from .src.latest_ami import get_latestami +from .src.preflight_module import pre_flight_checks +from .src.self_managed import Update_nodeGroup, filter_node_groups, get_asg_node_groups, get_node_groups + queue = queue.Queue() -class StatsWorker(threading.Thread): - def __init__(self, queue,id): + +class StatsWorker(threading.Thread): + def __init__(self, queue, id): threading.Thread.__init__(self) self.queue = queue - self.id=id + self.id = id def run(self): while self.queue.not_empty: - cluster_name,ng_name,to_update,regionName,max_retry,forced,typse=self.queue.get() - if typse=="managed": - start= time.time() - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg="Updating Node Group {ng} To version {versi}".format(ng=ng_name,versi=to_update)) - out=Update_nodeGroup(cluster_name, ng_name, to_update,regionName) + cluster_name, ng_name, to_update, regionName, max_retry, forced, typse = self.queue.get() + if typse == "managed": + start = time.time() + logs_pusher( + regionName=regionName, + cluster_name=cluster_name, + msg="Updating Node Group {ng} To version {versi}".format(ng=ng_name, versi=to_update), + ) + out = Update_nodeGroup(cluster_name, ng_name, to_update, regionName) # Update_nodeGroup(cluster_name, ng_name, to_update,regionName) end = time.time() - hours, rem = divmod(end-start, 3600) + hours, rem = divmod(end - start, 3600) minutes, seconds = divmod(rem, 60) - print("Updated Node Group {ng} To version {versi} ".format(ng=ng_name,versi=to_update),"{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg="Updated Node Group {ng} To version {versi}".format(ng=ng_name,versi=to_update)) - + print( + "Updated Node Group {ng} To version {versi} ".format(ng=ng_name, versi=to_update), + "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds), + ) + logs_pusher( + regionName=regionName, + cluster_name=cluster_name, + msg="Updated Node Group {ng} To version {versi}".format(ng=ng_name, versi=to_update), + ) + self.queue.task_done() - elif typse=="selfmanaged": - start= time.time() - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg="Updating Node Group {ng} To version {versi}".format(ng=ng_name,versi=to_update)) - out=actual_update(cluster_name=cluster_name,asg_iter=ng_name,to_update=to_update,regionName=regionName,max_retry=max_retry,forced=forced) + elif typse == "selfmanaged": + start = time.time() + logs_pusher( + regionName=regionName, + cluster_name=cluster_name, + msg="Updating Node Group {ng} To version {versi}".format(ng=ng_name, versi=to_update), + ) + out = actual_update( + cluster_name=cluster_name, + asg_iter=ng_name, + to_update=to_update, + regionName=regionName, + max_retry=max_retry, + forced=forced, + ) end = time.time() - hours, rem = divmod(end-start, 3600) + hours, rem = divmod(end - start, 3600) minutes, seconds = divmod(rem, 60) - print("Updated Node Group {ng} To version {versi} ".format(ng=ng_name,versi=to_update),"{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg="Updated Node Group {ng} To version {versi}".format(ng=ng_name,versi=to_update)) + print( + "Updated Node Group {ng} To version {versi} ".format(ng=ng_name, versi=to_update), + "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds), + ) + logs_pusher( + regionName=regionName, + cluster_name=cluster_name, + msg="Updated Node Group {ng} To version {versi}".format(ng=ng_name, versi=to_update), + ) self.queue.task_done() - - -def actual_update(cluster_name,asg_iter,to_update,regionName,max_retry,forced): - presentversion="1.1 eks update" - instance_type,image_to_search=get_ami_name(cluster_name,asg_iter,presentversion,regionName) - print("The Image Type Detected = ",instance_type) - if instance_type=="NAN": +def actual_update(cluster_name, asg_iter, to_update, regionName, max_retry, forced): + presentversion = "1.1 eks update" + instance_type, image_to_search = get_ami_name(cluster_name, asg_iter, presentversion, regionName) + print("The Image Type Detected = ", instance_type) + if instance_type == "NAN": return False - if isinstance(image_to_search,str) and "Windows_Server" in image_to_search: - image_to_search=image_to_search[:46] - latest_ami = get_latestami(to_update,instance_type,image_to_search,regionName) + if isinstance(image_to_search, str) and "Windows_Server" in image_to_search: + image_to_search = image_to_search[:46] + latest_ami = get_latestami(to_update, instance_type, image_to_search, regionName) print("The Latest AMI Recommended = {image}".format(image=latest_ami)) - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg="The Latest AMI Image = {image}".format(image=latest_ami)) - if get_outdated_Asg(asg_iter, latest_ami,regionName): - addAutoScaling(asg_iter, latest_ami,regionName) - print("New Launch Configuration Added to = {ast} With EKS AMI = {ami}".format(ast=asg_iter,ami=latest_ami)) - - outdated_instances = outdated_lt(asg_iter,regionName) + logs_pusher( + regionName=regionName, cluster_name=cluster_name, msg="The Latest AMI Image = {image}".format(image=latest_ami) + ) + if get_outdated_Asg(asg_iter, latest_ami, regionName): + addAutoScaling(asg_iter, latest_ami, regionName) + print("New Launch Configuration Added to = {ast} With EKS AMI = {ami}".format(ast=asg_iter, ami=latest_ami)) + + outdated_instances = outdated_lt(asg_iter, regionName) if len(outdated_instances) == 0: return True try: terminated_ids = [] - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg="The Outdate Instance Found Are = {instan}".format(instan=outdated_instances)) + logs_pusher( + regionName=regionName, + cluster_name=cluster_name, + msg="The Outdate Instance Found Are = {instan}".format(instan=outdated_instances), + ) for instance in outdated_instances: - befor_count = get_num_of_instances( - asg_iter, terminated_ids,regionName) - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg="Total Instance count = {count}".format(count=befor_count)) + befor_count = get_num_of_instances(asg_iter, terminated_ids, regionName) + logs_pusher( + regionName=regionName, + cluster_name=cluster_name, + msg="Total Instance count = {count}".format(count=befor_count), + ) add_time = datetime.datetime.now(datetime.timezone.utc) - if abs(befor_count-len(outdated_instances)) != len(outdated_instances): - add_node(asg_iter,regionName) + if abs(befor_count - len(outdated_instances)) != len(outdated_instances): + add_node(asg_iter, regionName) time.sleep(45) - latest_instance = get_latest_instance( - asg_name=asg_iter, add_time=add_time,regionName=regionName) - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg="The Instance Created = {instan}".format(instan=latest_instance)) - print(latest_instance, - "is Created and waiting for it to be ready") + latest_instance = get_latest_instance(asg_name=asg_iter, add_time=add_time, regionName=regionName) + logs_pusher( + regionName=regionName, + cluster_name=cluster_name, + msg="The Instance Created = {instan}".format(instan=latest_instance), + ) + print(latest_instance, "is Created and waiting for it to be ready") time.sleep(30) - wait_for_ready(latest_instance,regionName) - - old_pod_id = find_node( - cluster_name=cluster_name, instance_id=instance,op="find",regionName=regionName) - if old_pod_id!="NAN": - retry=0 - flag=0 - while retry<=max_retry: - if not find_node(cluster_name=cluster_name, instance_id=instance,op="find",regionName=regionName) =="NAN": - flag=1 - retry+=1 + wait_for_ready(latest_instance, regionName) + + old_pod_id = find_node(cluster_name=cluster_name, instance_id=instance, op="find", regionName=regionName) + if old_pod_id != "NAN": + retry = 0 + flag = 0 + while retry <= max_retry: + if ( + not find_node(cluster_name=cluster_name, instance_id=instance, op="find", regionName=regionName) + == "NAN" + ): + flag = 1 + retry += 1 time.sleep(10) - if flag==0: - worker_terminate(instance,regionName=regionName) + if flag == 0: + worker_terminate(instance, regionName=regionName) raise Exception("404 instance is not corresponded to particular node group") - + print("Unshceduling The worker Node ={wn} ".format(wn=old_pod_id)) - unschedule_old_nodes( - ClusterName=cluster_name, Nodename=old_pod_id,regionName=regionName) - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg="The Node is Unscheduled = {instan}".format(instan=old_pod_id)) + unschedule_old_nodes(ClusterName=cluster_name, Nodename=old_pod_id, regionName=regionName) + logs_pusher( + regionName=regionName, + cluster_name=cluster_name, + msg="The Node is Unscheduled = {instan}".format(instan=old_pod_id), + ) print("Worker Node Drained = {instan}".format(instan=old_pod_id)) - drain_nodes(cluster_name=cluster_name, Nodename=old_pod_id,forced=forced,regionName=regionName) - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg="The Worker Node is Drained = {instan}".format(instan=old_pod_id)) + drain_nodes(cluster_name=cluster_name, Nodename=old_pod_id, forced=forced, regionName=regionName) + logs_pusher( + regionName=regionName, + cluster_name=cluster_name, + msg="The Worker Node is Drained = {instan}".format(instan=old_pod_id), + ) print("Deleting worker Node Started ={op} ".format(op=old_pod_id)) - delete_node(cluster_name=cluster_name, NodeName=old_pod_id,regionName=regionName) - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg="The Worker Node is Deleted = {instan}".format(instan=old_pod_id)) + delete_node(cluster_name=cluster_name, NodeName=old_pod_id, regionName=regionName) + logs_pusher( + regionName=regionName, + cluster_name=cluster_name, + msg="The Worker Node is Deleted = {instan}".format(instan=old_pod_id), + ) print("Terminating Worker Node {wn}".format(wn=instance)) - worker_terminate(instance,regionName=regionName) + worker_terminate(instance, regionName=regionName) terminated_ids.append(instance) - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg="The Worker Node instance is Terminated = {instan}".format(instan=instance)) + logs_pusher( + regionName=regionName, + cluster_name=cluster_name, + msg="The Worker Node instance is Terminated = {instan}".format(instan=instance), + ) return True except Exception as e: raise (e) - - - - def main(args): try: - + cluster_name = args.name to_update = args.version - pass_vpc=args.pass_vpc - max_retry=args.max_retry - regionName=args.region - presentversion="NAN" - isPresent=False - forced= args.force - parlleld=args.parallel + pass_vpc = args.pass_vpc + max_retry = args.max_retry + regionName = args.region + presentversion = "NAN" + isPresent = False + forced = args.force + parlleld = args.parallel if args.eksctl: quit("updating using EKSCTL is still under testing will be launched soon") - ''' Preflight Logic ''' - if not (pre_flight_checks(True,cluster_name,regionName,args.pass_vpc,args.version,args.email,args.force)): - print('Pre flight check for cluster ' + cluster_name +' failed') + """ Preflight Logic """ + if not (pre_flight_checks(True, cluster_name, regionName, args.pass_vpc, args.version, args.email, args.force)): + print("Pre flight check for cluster " + cluster_name + " failed") quit() - else : - print('Pre flight check for the cluster ' + cluster_name + ' succeded') - if (args.preflight) : + else: + print("Pre flight check for the cluster " + cluster_name + " succeded") + if args.preflight: quit() # ''' upgrade Logic''' - st=time.time() - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg="The Cluster Upgrade Process has Started") - if is_cluster_exists(Clustname=cluster_name,regionName=regionName)=="ACTIVE" or is_cluster_exists(Clustname=cluster_name,regionName=regionName)=="UPDATING": - presentversion=status_of_cluster(cluster_name,regionName)[1] - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg="The Current Version of the Cluster is Detected = {version} ".format(version=presentversion)) + st = time.time() + logs_pusher(regionName=regionName, cluster_name=cluster_name, msg="The Cluster Upgrade Process has Started") + if ( + is_cluster_exists(Clustname=cluster_name, regionName=regionName) == "ACTIVE" + or is_cluster_exists(Clustname=cluster_name, regionName=regionName) == "UPDATING" + ): + presentversion = status_of_cluster(cluster_name, regionName)[1] + logs_pusher( + regionName=regionName, + cluster_name=cluster_name, + msg="The Current Version of the Cluster is Detected = {version} ".format(version=presentversion), + ) else: raise Exception("Cluster is Not Active") - # ''' # Checking Cluster is Active or Not Befor Making an Update # ''' - start=time.time() - if is_cluster_exists(Clustname=cluster_name,regionName=regionName)=="ACTIVE": - #if eksctl flag is enabled.s - if(args.eksctl!=False): + start = time.time() + if is_cluster_exists(Clustname=cluster_name, regionName=regionName) == "ACTIVE": + # if eksctl flag is enabled.s + if args.eksctl != False: print("updating using EKSCTL") eksctl_execute(args) - print('Pre flight check for the upgraded cluster') - if not (pre_flight_checks(cluster_name,regionName,topic = args.topic)): - print('Pre flight check for cluster ' + cluster_name +' failed after it upgraded') - else : - print('After update check for cluster completed successfully') + print("Pre flight check for the upgraded cluster") + if not (pre_flight_checks(cluster_name, regionName, topic=args.topic)): + print("Pre flight check for cluster " + cluster_name + " failed after it upgraded") + else: + print("After update check for cluster completed successfully") quit() - update_cluster(Clustname=cluster_name, Version=to_update,regionName=regionName) + update_cluster(Clustname=cluster_name, Version=to_update, regionName=regionName) time.sleep(5) - ''' Making Sure the Cluster is Updated''' - if status_of_cluster(cluster_name,regionName)[1] != to_update or status_of_cluster(cluster_name,regionName)[0] != "ACTIVE": - update_cluster(cluster_name, to_update,regionName) - - ''' finding the managed autoscaling groups ''' + """ Making Sure the Cluster is Updated""" + if ( + status_of_cluster(cluster_name, regionName)[1] != to_update + or status_of_cluster(cluster_name, regionName)[0] != "ACTIVE" + ): + update_cluster(cluster_name, to_update, regionName) + """ finding the managed autoscaling groups """ - end=time.time() - hours, rem = divmod(end-start, 3600) + end = time.time() + hours, rem = divmod(end - start, 3600) minutes, seconds = divmod(rem, 60) - print("The Time Taken For the Cluster to Upgrade ","{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg="Time Taken For the Cluster to Upgrade "+" {:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) - - - finding_manged = get_asg_node_groups(cluster_name,regionName) - - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg="The Manged Node Groups Found are "+",".join(finding_manged)) - - - asg_list = get_Asgs(cluster_name,regionName) + print( + "The Time Taken For the Cluster to Upgrade ", + "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds), + ) + logs_pusher( + regionName=regionName, + cluster_name=cluster_name, + msg="Time Taken For the Cluster to Upgrade " + + " {:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds), + ) + + finding_manged = get_asg_node_groups(cluster_name, regionName) + + logs_pusher( + regionName=regionName, + cluster_name=cluster_name, + msg="The Manged Node Groups Found are " + ",".join(finding_manged), + ) + + asg_list = get_Asgs(cluster_name, regionName) - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg="The Asg's Found Are "+",".join(asg_list)) - - ''' removing selfmanged from manged so that we dont update them again''' + logs_pusher(regionName=regionName, cluster_name=cluster_name, msg="The Asg's Found Are " + ",".join(asg_list)) - asg_list_self_managed = list(set(asg_list)-set(finding_manged)) + """ removing selfmanged from manged so that we dont update them again""" - ''' addons update ''' + asg_list_self_managed = list(set(asg_list) - set(finding_manged)) - finding_manged_nodes_names=get_node_groups(Clustername=cluster_name,regionName=regionName) - + """ addons update """ + + finding_manged_nodes_names = get_node_groups(Clustername=cluster_name, regionName=regionName) print(" The add-ons Update has been initiated.... ") - start_time=time.time() - start=time.time() - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg="The Addons Upgrade Started At "+str(start_time)) - - update_addons(cluster_name=cluster_name,version=to_update,vpcPass=pass_vpc,regionName=regionName) - end=time.time() - hours, rem = divmod(end-start, 3600) + start_time = time.time() + start = time.time() + logs_pusher( + regionName=regionName, cluster_name=cluster_name, msg="The Addons Upgrade Started At " + str(start_time) + ) + + update_addons(cluster_name=cluster_name, version=to_update, vpcPass=pass_vpc, regionName=regionName) + end = time.time() + hours, rem = divmod(end - start, 3600) minutes, seconds = divmod(rem, 60) - print("The Taken For the Addons Upgrade ","{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg="The Taken For the Addons Upgrade "+"{:0>2}:{:0>2}:{:05.2f}".format(int(hours),int(minutes),seconds)) - - ''' finding managed node groups with filter''' - finding_manged_nodes=filter_node_groups(cluster_name=cluster_name,node_list=finding_manged_nodes_names,latest_version=to_update,regionName=regionName) - if len(finding_manged)>0: - print("The OutDated Managed Node Groups = ",finding_manged) + print("The Taken For the Addons Upgrade ", "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds)) + logs_pusher( + regionName=regionName, + cluster_name=cluster_name, + msg="The Taken For the Addons Upgrade " + + "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds), + ) + + """ finding managed node groups with filter""" + finding_manged_nodes = filter_node_groups( + cluster_name=cluster_name, + node_list=finding_manged_nodes_names, + latest_version=to_update, + regionName=regionName, + ) + if len(finding_manged) > 0: + print("The OutDated Managed Node Groups = ", finding_manged) else: print("No OutDated Managed Node Groups Found ") - - replicas_value=0 - ''' checking auto scaler present and the value associated from it ''' + replicas_value = 0 + + """ checking auto scaler present and the value associated from it """ - isPresent,replicas_value=is_cluster_auto_scaler_present(ClusterName=cluster_name,regionName=regionName) + isPresent, replicas_value = is_cluster_auto_scaler_present(ClusterName=cluster_name, regionName=regionName) if isPresent: - clus_auto_enable_disable(ClusterName=cluster_name,type="pause",mx_val=replicas_value,regionName=regionName) - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg="Paused Cluster AutoScaler") - + clus_auto_enable_disable( + ClusterName=cluster_name, type="pause", mx_val=replicas_value, regionName=regionName + ) + logs_pusher(regionName=regionName, cluster_name=cluster_name, msg="Paused Cluster AutoScaler") + print("Paused the Cluster AutoScaler") else: print("No Cluster AutoScaler is Found") if parlleld: for x in range(20): - worker = StatsWorker(queue,x) + worker = StatsWorker(queue, x) worker.setDaemon(True) worker.start() - if len(finding_manged_nodes) != 0: for ng_name in finding_manged_nodes: - start=time.time() - print("Updating the Node Group = {ng} To version = {versi}".format(ng=ng_name,versi=to_update)) + start = time.time() + print("Updating the Node Group = {ng} To version = {versi}".format(ng=ng_name, versi=to_update)) if parlleld: - queue.put([cluster_name,ng_name,to_update,regionName,max_retry,forced,"managed"]) + queue.put([cluster_name, ng_name, to_update, regionName, max_retry, forced, "managed"]) else: - Update_nodeGroup(cluster_name, ng_name, to_update,regionName) + Update_nodeGroup(cluster_name, ng_name, to_update, regionName) if len(asg_list_self_managed) != 0: for asg_iter in asg_list_self_managed: if parlleld: - queue.put([cluster_name,asg_iter,to_update,regionName,max_retry,forced,"selfmanaged"]) + queue.put([cluster_name, asg_iter, to_update, regionName, max_retry, forced, "selfmanaged"]) else: - actual_update(cluster_name,asg_iter,to_update,regionName,max_retry,forced) + actual_update(cluster_name, asg_iter, to_update, regionName, max_retry, forced) if parlleld: queue.join() if isPresent: - clus_auto_enable_disable(ClusterName=cluster_name,type="start",mx_val=replicas_value,regionName=regionName) - print("Cluster Autoscaler is Enabled Again") - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg="Cluster Autoscaler is Enabled Again") - print(" EKS Cluster {Clustname} UPDATED TO {ver}".format(Clustname=cluster_name,ver=to_update)) - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg=" EKS Cluster {Clustname} UPDATED TO {ver}".format(Clustname=cluster_name,ver=to_update)) - print('Post flight check for the upgraded cluster') - if not (pre_flight_checks(False,cluster_name,regionName,args.pass_vpc,email = args.email)): - print('Post flight check for cluster ' + cluster_name +' failed after it upgraded') - else : - print('After update check for cluster completed successfully') - - + clus_auto_enable_disable( + ClusterName=cluster_name, type="start", mx_val=replicas_value, regionName=regionName + ) + print("Cluster Autoscaler is Enabled Again") + logs_pusher(regionName=regionName, cluster_name=cluster_name, msg="Cluster Autoscaler is Enabled Again") + print(" EKS Cluster {Clustname} UPDATED TO {ver}".format(Clustname=cluster_name, ver=to_update)) + logs_pusher( + regionName=regionName, + cluster_name=cluster_name, + msg=" EKS Cluster {Clustname} UPDATED TO {ver}".format(Clustname=cluster_name, ver=to_update), + ) + print("Post flight check for the upgraded cluster") + if not (pre_flight_checks(False, cluster_name, regionName, args.pass_vpc, email=args.email)): + print("Post flight check for cluster " + cluster_name + " failed after it upgraded") + else: + print("After update check for cluster completed successfully") except Exception as e: - if isPresent: - try: - clus_auto_enable_disable(ClusterName=cluster_name,type="start",mx_val=replicas_value,regionName=regionName) - print("Cluster Autoscaler is Enabled Again") - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg="Cluster Autoscaler is Enabled Again") - except Exception as e: + if isPresent: + try: + clus_auto_enable_disable( + ClusterName=cluster_name, type="start", mx_val=replicas_value, regionName=regionName + ) + print("Cluster Autoscaler is Enabled Again") + logs_pusher(regionName=regionName, cluster_name=cluster_name, msg="Cluster Autoscaler is Enabled Again") + except Exception as e: print("Enter AutoScaler Manullay") - logs_pusher(regionName=regionName,cluster_name=cluster_name,msg="The Cluster Upgrade Failed Due To = {err}".format(err=e)) - print(e) - + logs_pusher( + regionName=regionName, + cluster_name=cluster_name, + msg="The Cluster Upgrade Failed Due To = {err}".format(err=e), + ) + print(e) diff --git a/pyproject.toml b/pyproject.toml index 2868923..a8c3da7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,25 @@ include = [ [tool.poetry.scripts] eksupdate = "eksupdate.cli:entry" +# Styling and linting Configurations +[tool.isort] +profile = "black" +line_length = 120 + +[tool.black] +line-length = 120 +target-version = ["py310"] + +[tool.pylint] +max-line-length = 120 + +[tool.pylint.messages_control] +max-line-length = 120 + +[tool.pylint.format] +max-line-length = 120 +max-module-lines = 1000 + [tool.poetry.dependencies] python = "^3.8" boto3 = "^1.26.60"