diff --git a/cloud_governance/main/environment_variables.py b/cloud_governance/main/environment_variables.py index a5b5c6c4..6eaea4d2 100644 --- a/cloud_governance/main/environment_variables.py +++ b/cloud_governance/main/environment_variables.py @@ -28,7 +28,7 @@ def load_from_env(self): with open(file_path) as f: for line in f.readlines(): key, found, value = line.strip().partition("=") - setattr(self, key, value) + setattr(self, key, value.strip('"')) if not found: logger.error(f"ERROR: invalid line in {env}: {line.strip()}") continue @@ -63,24 +63,6 @@ def __init__(self): self.load_from_env() self.load_from_yaml() - # env files override true ENV. Not best order, but easier to write :/ - # .env.generated can be auto-generated (by an external tool) based on the local cluster's configuration. - for env in ".env", ".env.generated": - try: - file_path = os.path.join(os.path.dirname(__file__), env) - with open(file_path) as f: - for line in f.readlines(): - key, found, value = line.strip().partition("=") - if not found: - print("ERROR: invalid line in {env}: {line.strip()}") - continue - if key in os.environ: - continue # prefer env to env file - os.environ[key] = value - - except FileNotFoundError: - pass # ignore - ################################################################################################## # dynamic parameters - configure for local run # parameters for running policies diff --git a/cloud_governance/policy/aws/zombie_cluster_resource.py b/cloud_governance/policy/aws/zombie_cluster_resource.py index 203e9fac..05b6ea6e 100644 --- a/cloud_governance/policy/aws/zombie_cluster_resource.py +++ b/cloud_governance/policy/aws/zombie_cluster_resource.py @@ -1,6 +1,7 @@ import boto3 -from cloud_governance.policy.policy_operations.aws.zombie_cluster.zombie_cluster_common_methods import ZombieClusterCommonMethods +from cloud_governance.policy.policy_operations.aws.zombie_cluster.zombie_cluster_common_methods import \ + ZombieClusterCommonMethods from cloud_governance.common.clouds.aws.ec2.ec2_operations import EC2Operations from cloud_governance.common.logger.init_logger import logger from cloud_governance.common.clouds.aws.utils.utils import Utils @@ -57,7 +58,7 @@ def all_cluster_instance(self): instance_id = item['InstanceId'] if item.get('Tags'): for tag in item['Tags']: - if tag['Key'].startswith(self.cluster_prefix): + if tag['Key'].startswith(self.cluster_prefix) and tag.get('Value', '').lower() == 'owned': result_instance[instance_id] = tag['Key'] return result_instance @@ -78,7 +79,7 @@ def _cluster_instance(self): instance_id = item['InstanceId'] if item.get('Tags'): for tag in item['Tags']: - if tag['Key'].startswith(self.cluster_prefix): + if tag['Key'].startswith(self.cluster_prefix) and tag.get('Value', '').lower() == 'owned': result_instance[instance_id] = tag['Key'] return result_instance @@ -99,7 +100,7 @@ def __get_cluster_resources(self, resources_list: list, input_resource_id: str, continue if resource.get(tags): for tag in resource[tags]: - if tag['Key'].startswith(self.cluster_prefix): + if tag['Key'].startswith(self.cluster_prefix) and tag.get('Value', '').lower() == 'owned': # when input a specific cluster, return resource id of the input cluster for inner_tag in resource[tags]: if self.cluster_tag: @@ -192,11 +193,14 @@ def zombie_cluster_volume(self, vpc_id: str = '', cluster_tag_vpc: str = ''): available_volumes.append(volume) exist_volume = self.__get_cluster_resources(resources_list=available_volumes, input_resource_id='VolumeId') zombies = self.__get_zombie_resources(exist_volume) - resources = self._get_tags_of_zombie_resources(resources=volumes_data, resource_id_name='VolumeId', zombies=zombies, aws_service='ec2') + resources = self._get_tags_of_zombie_resources(resources=volumes_data, resource_id_name='VolumeId', + zombies=zombies, aws_service='ec2') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): - cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) + cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days( + resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, + cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: self.delete_ec2_resource.delete_zombie_resource(resource_id=zombie, resource='ec2_volume') else: @@ -239,11 +243,14 @@ def zombie_cluster_snapshot(self, vpc_id: str = '', cluster_tag_vpc: str = ''): snapshots_data = self.ec2_operations.get_snapshots() exist_snapshot = self.__get_cluster_resources(resources_list=snapshots_data, input_resource_id='SnapshotId') zombies = self.__get_zombie_resources(exist_snapshot) - resources = self._get_tags_of_zombie_resources(resources=snapshots_data, resource_id_name='SnapshotId', zombies=zombies, aws_service='ec2') + resources = self._get_tags_of_zombie_resources(resources=snapshots_data, resource_id_name='SnapshotId', + zombies=zombies, aws_service='ec2') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): - cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) + cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days( + resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, + cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: self.delete_ec2_resource.delete_zombie_resource(resource='ebs_snapshots', resource_id=zombie) else: @@ -325,22 +332,30 @@ def zombie_cluster_security_group(self, vpc_id: str = '', cluster_tag_vpc: str = exist_security_group = self.__get_cluster_resources(resources_list=security_groups, input_resource_id='GroupId') zombies = self.__get_zombie_resources(exist_security_group) if vpc_id and not zombies: - zombies = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=security_groups, output_tag='GroupId', cluster_tag=cluster_tag_vpc) - resources = self._get_tags_of_zombie_resources(resources=security_groups, resource_id_name='GroupId', zombies=zombies, aws_service='ec2') + zombies = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=security_groups, output_tag='GroupId', + cluster_tag=cluster_tag_vpc) + resources = self._get_tags_of_zombie_resources(resources=security_groups, resource_id_name='GroupId', + zombies=zombies, aws_service='ec2') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): security_groups = self.ec2_operations.get_security_groups() - vpc_id = self.__extract_vpc_id_from_resource_data(zombie_id=zombie, resource_data=security_groups, input_tag='GroupId') - zombie_ids = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=security_groups, output_tag='GroupId', cluster_tag=cluster_tag) - cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) + vpc_id = self.__extract_vpc_id_from_resource_data(zombie_id=zombie, resource_data=security_groups, + input_tag='GroupId') + zombie_ids = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=security_groups, + output_tag='GroupId', cluster_tag=cluster_tag) + cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days( + resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, + cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: for zombie_id in zombie_ids: - self.delete_ec2_resource.delete_zombie_resource('security_group', resource_id=zombie_id, vpc_id=vpc_id, cluster_tag=cluster_tag) + self.delete_ec2_resource.delete_zombie_resource('security_group', resource_id=zombie_id, + vpc_id=vpc_id, cluster_tag=cluster_tag) else: if self._force_delete and self.delete: for zombie_id in zombie_ids: - self.delete_ec2_resource.delete_zombie_resource('security_group', resource_id=zombie_id, vpc_id=vpc_id, cluster_tag=cluster_tag) + self.delete_ec2_resource.delete_zombie_resource('security_group', resource_id=zombie_id, + vpc_id=vpc_id, cluster_tag=cluster_tag) return zombies, cluster_left_out_days @@ -356,29 +371,41 @@ def zombie_cluster_elastic_ip(self, vpc_id: str = '', cluster_tag_vpc: str = '') exist_elastic_ip_association.append(elastic_ip) if elastic_ip.get('AllocationId'): exist_elastic_ip_allocation.append(elastic_ip) - exist_elastic_ip_ass = self.__get_cluster_resources(resources_list=exist_elastic_ip_association, input_resource_id='AssociationId') - exist_elastic_ip_all = self.__get_cluster_resources(resources_list=exist_elastic_ip_allocation, input_resource_id='AllocationId') + exist_elastic_ip_ass = self.__get_cluster_resources(resources_list=exist_elastic_ip_association, + input_resource_id='AssociationId') + exist_elastic_ip_all = self.__get_cluster_resources(resources_list=exist_elastic_ip_allocation, + input_resource_id='AllocationId') zombies_ass = self.__get_zombie_resources(exist_elastic_ip_ass) zombies_all = self.__get_zombie_resources(exist_elastic_ip_all) - resources_ass = self._get_tags_of_zombie_resources(resources=elastic_ips_data, resource_id_name='AllocationId', zombies=zombies_ass, aws_service='ec2') - resources_all = self._get_tags_of_zombie_resources(resources=elastic_ips_data, resource_id_name='AllocationId', zombies=zombies_all, aws_service='ec2') + resources_ass = self._get_tags_of_zombie_resources(resources=elastic_ips_data, resource_id_name='AllocationId', + zombies=zombies_ass, aws_service='ec2') + resources_all = self._get_tags_of_zombie_resources(resources=elastic_ips_data, resource_id_name='AllocationId', + zombies=zombies_all, aws_service='ec2') cluster_left_out_days = {} if zombies_ass: for zombie, cluster_tag in zombies_ass.items(): - cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources_ass, cluster_left_out_days=cluster_left_out_days, zombie=zombie,cluster_tag=cluster_tag) + cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days( + resources=resources_ass, cluster_left_out_days=cluster_left_out_days, zombie=zombie, + cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: - self.delete_ec2_resource.delete_zombie_resource(resource='elastic_ip', resource_id=zombie, deletion_type='disassociate') + self.delete_ec2_resource.delete_zombie_resource(resource='elastic_ip', resource_id=zombie, + deletion_type='disassociate') else: if self._force_delete: - self.delete_ec2_resource.delete_zombie_resource(resource='elastic_ip', resource_id=zombie, deletion_type='disassociate') + self.delete_ec2_resource.delete_zombie_resource(resource='elastic_ip', resource_id=zombie, + deletion_type='disassociate') if zombies_all: for zombie, cluster_tag in zombies_all.items(): - cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources_all, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) + cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days( + resources=resources_all, cluster_left_out_days=cluster_left_out_days, zombie=zombie, + cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: - self.delete_ec2_resource.delete_zombie_resource(resource='elastic_ip', resource_id=zombie, cluster_tag=cluster_tag) + self.delete_ec2_resource.delete_zombie_resource(resource='elastic_ip', resource_id=zombie, + cluster_tag=cluster_tag) else: if self._force_delete and self.delete: - self.delete_ec2_resource.delete_zombie_resource(resource='elastic_ip', resource_id=zombie, cluster_tag=cluster_tag) + self.delete_ec2_resource.delete_zombie_resource(resource='elastic_ip', resource_id=zombie, + cluster_tag=cluster_tag) zombies = {**zombies_all} return zombies, cluster_left_out_days @@ -387,28 +414,42 @@ def zombie_cluster_network_interface(self, vpc_id: str = '', cluster_tag_vpc: st This method returns list of zombie cluster's network interface according to existing instances and cluster name data """ network_interfaces_data = self.ec2_operations.get_network_interface() - exist_network_interface = self.__get_cluster_resources(resources_list=network_interfaces_data, input_resource_id='NetworkInterfaceId', tags='TagSet') + exist_network_interface = self.__get_cluster_resources(resources_list=network_interfaces_data, + input_resource_id='NetworkInterfaceId', tags='TagSet') zombies = self.__get_zombie_resources(exist_network_interface) if not zombies and vpc_id: - zombies = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=network_interfaces_data, output_tag='NetworkInterfaceId', tags='TagSet', cluster_tag=cluster_tag_vpc) - resources = self._get_tags_of_zombie_resources(resources=network_interfaces_data, resource_id_name='NetworkInterfaceId', zombies=zombies, aws_service='ec2', aws_tag='TagSet') + zombies = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=network_interfaces_data, + output_tag='NetworkInterfaceId', tags='TagSet', + cluster_tag=cluster_tag_vpc) + resources = self._get_tags_of_zombie_resources(resources=network_interfaces_data, + resource_id_name='NetworkInterfaceId', zombies=zombies, + aws_service='ec2', aws_tag='TagSet') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): network_interfaces_data = self.ec2_operations.get_network_interface() - vpc_id = self.__extract_vpc_id_from_resource_data(zombie_id=zombie, resource_data=network_interfaces_data, input_tag='NetworkInterfaceId') + vpc_id = self.__extract_vpc_id_from_resource_data(zombie_id=zombie, + resource_data=network_interfaces_data, + input_tag='NetworkInterfaceId') if vpc_id: - zombie_ids = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=network_interfaces_data, output_tag='NetworkInterfaceId', cluster_tag=cluster_tag, tags='TagSet') + zombie_ids = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=network_interfaces_data, + output_tag='NetworkInterfaceId', cluster_tag=cluster_tag, + tags='TagSet') else: zombie_ids = [zombie] - cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days( resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) + cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days( + resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, + cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: for zombie_id in zombie_ids: - self.delete_ec2_resource.delete_zombie_resource(resource='network_interface', resource_id=zombie_id, cluster_tag=cluster_tag) + self.delete_ec2_resource.delete_zombie_resource(resource='network_interface', + resource_id=zombie_id, cluster_tag=cluster_tag) else: if self._force_delete and self.delete: for zombie_id in zombie_ids: - self.delete_ec2_resource.delete_zombie_resource(resource='network_interface', resource_id=zombie_id, cluster_tag=cluster_tag) + self.delete_ec2_resource.delete_zombie_resource(resource='network_interface', + resource_id=zombie_id, + cluster_tag=cluster_tag) return zombies, cluster_left_out_days def zombie_cluster_load_balancer(self, vpc_id: str = '', cluster_tag_vpc: str = ''): @@ -433,16 +474,22 @@ def zombie_cluster_load_balancer(self, vpc_id: str = '', cluster_tag_vpc: str = exist_load_balancer[resource_id] = tag['Key'] break zombies = self.__get_zombie_resources(exist_load_balancer) - resources = self._get_tags_of_zombie_resources(resources=load_balancers_data, resource_id_name='LoadBalancerName', zombies=zombies, aws_service='elbv1') + resources = self._get_tags_of_zombie_resources(resources=load_balancers_data, + resource_id_name='LoadBalancerName', zombies=zombies, + aws_service='elbv1') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): - cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) + cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days( + resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, + cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: - self.delete_ec2_resource.delete_zombie_resource(resource='load_balancer', resource_id=zombie, cluster_tag=cluster_tag) + self.delete_ec2_resource.delete_zombie_resource(resource='load_balancer', resource_id=zombie, + cluster_tag=cluster_tag) else: if self._force_delete and self.delete: - self.delete_ec2_resource.delete_zombie_resource(resource='load_balancer', resource_id=zombie, cluster_tag=cluster_tag) + self.delete_ec2_resource.delete_zombie_resource(resource='load_balancer', resource_id=zombie, + cluster_tag=cluster_tag) return zombies, cluster_left_out_days def zombie_cluster_load_balancer_v2(self, vpc_id: str = '', cluster_tag_vpc: str = ''): @@ -466,16 +513,22 @@ def zombie_cluster_load_balancer_v2(self, vpc_id: str = '', cluster_tag_vpc: str exist_load_balancer[resource_id] = tag['Key'] break zombies = self.__get_zombie_resources(exist_load_balancer) - resources = self._get_tags_of_zombie_resources(resources=load_balancers_data, resource_id_name='LoadBalancerArn', zombies=zombies, aws_service='elbv2') + resources = self._get_tags_of_zombie_resources(resources=load_balancers_data, + resource_id_name='LoadBalancerArn', zombies=zombies, + aws_service='elbv2') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): - cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) + cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days( + resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, + cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: - self.delete_ec2_resource.delete_zombie_resource(resource='load_balancer_v2', resource_id=zombie, cluster_tag=cluster_tag) + self.delete_ec2_resource.delete_zombie_resource(resource='load_balancer_v2', resource_id=zombie, + cluster_tag=cluster_tag) else: if self._force_delete and self.delete: - self.delete_ec2_resource.delete_zombie_resource(resource='load_balancer_v2', resource_id=zombie, cluster_tag=cluster_tag) + self.delete_ec2_resource.delete_zombie_resource(resource='load_balancer_v2', resource_id=zombie, + cluster_tag=cluster_tag) return zombies, cluster_left_out_days def __get_all_exist_vpcs(self): @@ -502,16 +555,23 @@ def zombie_cluster_vpc(self): "NATG": self.zombie_cluster_nat_gateway, "NACL": self.zombie_cluster_network_acl, "ENI": self.zombie_cluster_network_interface, "IGW": self.zombie_cluster_internet_gateway, "SUB": self.zombie_cluster_subnet, "EIP": self.zombie_cluster_elastic_ip} - resources = self._get_tags_of_zombie_resources(resources=vpcs_data, resource_id_name='VpcId', zombies=zombies, aws_service='ec2') + resources = self._get_tags_of_zombie_resources(resources=vpcs_data, resource_id_name='VpcId', zombies=zombies, + aws_service='ec2') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): - cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) + cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days( + resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, + cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: - self.delete_ec2_resource.delete_zombie_resource(resource='vpc', resource_id=zombie, pending_resources=delete_dict, cluster_tag=cluster_tag) + self.delete_ec2_resource.delete_zombie_resource(resource='vpc', resource_id=zombie, + pending_resources=delete_dict, + cluster_tag=cluster_tag) else: if self._force_delete and self.delete: - self.delete_ec2_resource.delete_zombie_resource(resource='vpc', resource_id=zombie, pending_resources=delete_dict, cluster_tag=cluster_tag) + self.delete_ec2_resource.delete_zombie_resource(resource='vpc', resource_id=zombie, + pending_resources=delete_dict, + cluster_tag=cluster_tag) return zombies, cluster_left_out_days @@ -525,21 +585,28 @@ def zombie_cluster_subnet(self, vpc_id: str = '', cluster_tag_vpc: str = ''): if not zombies and vpc_id: zombies = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=subnets_data, output_tag='SubnetId', cluster_tag=cluster_tag_vpc) - resources = self._get_tags_of_zombie_resources(resources=subnets_data, resource_id_name='SubnetId', zombies=zombies, aws_service='ec2') + resources = self._get_tags_of_zombie_resources(resources=subnets_data, resource_id_name='SubnetId', + zombies=zombies, aws_service='ec2') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): subnets_data = self.ec2_operations.get_subnets() - vpc_id = self.__extract_vpc_id_from_resource_data(zombie_id=zombie, resource_data=subnets_data, input_tag='SubnetId') - zombie_ids = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=subnets_data, output_tag='SubnetId', cluster_tag=cluster_tag) - cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) + vpc_id = self.__extract_vpc_id_from_resource_data(zombie_id=zombie, resource_data=subnets_data, + input_tag='SubnetId') + zombie_ids = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=subnets_data, output_tag='SubnetId', + cluster_tag=cluster_tag) + cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days( + resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, + cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: for zombie_id in zombie_ids: - self.delete_ec2_resource.delete_zombie_resource(resource='subnet', resource_id=zombie_id, cluster_tag=cluster_tag) + self.delete_ec2_resource.delete_zombie_resource(resource='subnet', resource_id=zombie_id, + cluster_tag=cluster_tag) else: if self._force_delete and self.delete: for zombie_id in zombie_ids: - self.delete_ec2_resource.delete_zombie_resource(resource='subnet', resource_id=zombie_id, cluster_tag=cluster_tag) + self.delete_ec2_resource.delete_zombie_resource(resource='subnet', resource_id=zombie_id, + cluster_tag=cluster_tag) return zombies, cluster_left_out_days def zombie_cluster_route_table(self, vpc_id: str = '', cluster_tag_vpc: str = ''): @@ -547,26 +614,35 @@ def zombie_cluster_route_table(self, vpc_id: str = '', cluster_tag_vpc: str = '' This method returns list of cluster's route table according to cluster tag name and cluster name data """ route_tables_data = self.ec2_operations.get_route_tables() - exist_route_table = self.__get_cluster_resources(resources_list=route_tables_data, input_resource_id='RouteTableId') + exist_route_table = self.__get_cluster_resources(resources_list=route_tables_data, + input_resource_id='RouteTableId') zombies = self.__get_zombie_resources(exist_route_table) if not zombies and vpc_id: - zombies = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=route_tables_data, output_tag='RouteTableId', cluster_tag=cluster_tag_vpc) - resources = self._get_tags_of_zombie_resources(resources=route_tables_data, resource_id_name='RouteTableId', zombies=zombies, aws_service='ec2') + zombies = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=route_tables_data, + output_tag='RouteTableId', cluster_tag=cluster_tag_vpc) + resources = self._get_tags_of_zombie_resources(resources=route_tables_data, resource_id_name='RouteTableId', + zombies=zombies, aws_service='ec2') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): route_tables_data = self.ec2_operations.get_route_tables() - vpc_id = self.__extract_vpc_id_from_resource_data(zombie_id=zombie, resource_data=route_tables_data, input_tag='RouteTableId') + vpc_id = self.__extract_vpc_id_from_resource_data(zombie_id=zombie, resource_data=route_tables_data, + input_tag='RouteTableId') zombie_ids = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=route_tables_data, output_tag='RouteTableId', cluster_tag=cluster_tag) - cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) + cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days( + resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, + cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: for zombie_id in zombie_ids: - self.delete_ec2_resource.delete_zombie_resource(resource='route_table', resource_id=zombie_id, vpc_id=vpc_id, cluster_tag=cluster_tag) + self.delete_ec2_resource.delete_zombie_resource(resource='route_table', resource_id=zombie_id, + vpc_id=vpc_id, cluster_tag=cluster_tag) else: if self._force_delete and self.delete: for zombie_id in zombie_ids: - self.delete_ec2_resource.delete_zombie_resource(resource='route_table', resource_id=zombie_id, vpc_id=vpc_id, cluster_tag=cluster_tag) + self.delete_ec2_resource.delete_zombie_resource(resource='route_table', + resource_id=zombie_id, vpc_id=vpc_id, + cluster_tag=cluster_tag) return zombies, cluster_left_out_days def zombie_cluster_internet_gateway(self, vpc_id: str = '', cluster_tag_vpc: str = ''): @@ -574,28 +650,44 @@ def zombie_cluster_internet_gateway(self, vpc_id: str = '', cluster_tag_vpc: str This method returns list of cluster's route table internet gateway according to cluster tag name and cluster name data """ internet_gateways_data = self.ec2_operations.get_internet_gateways() - exist_internet_gateway = self.__get_cluster_resources(resources_list=internet_gateways_data, input_resource_id='InternetGatewayId') + exist_internet_gateway = self.__get_cluster_resources(resources_list=internet_gateways_data, + input_resource_id='InternetGatewayId') zombies = self.__get_zombie_resources(exist_internet_gateway) if not zombies and vpc_id: - zombies = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=internet_gateways_data, output_tag='InternetGatewayId', input_tag='Attachments', cluster_tag=cluster_tag_vpc) - resources = self._get_tags_of_zombie_resources(resources=internet_gateways_data, resource_id_name='InternetGatewayId', zombies=zombies, aws_service='ec2') + zombies = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=internet_gateways_data, + output_tag='InternetGatewayId', input_tag='Attachments', + cluster_tag=cluster_tag_vpc) + resources = self._get_tags_of_zombie_resources(resources=internet_gateways_data, + resource_id_name='InternetGatewayId', zombies=zombies, + aws_service='ec2') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): internet_gateways_data = self.ec2_operations.get_internet_gateways() - vpc_id = self.__extract_vpc_id_from_resource_data(zombie_id=zombie, resource_data=internet_gateways_data, input_tag='InternetGatewayId', output_tag='Attachments') + vpc_id = self.__extract_vpc_id_from_resource_data(zombie_id=zombie, + resource_data=internet_gateways_data, + input_tag='InternetGatewayId', + output_tag='Attachments') if vpc_id: - zombie_ids = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=internet_gateways_data, output_tag='InternetGatewayId', cluster_tag=cluster_tag, input_tag='Attachments') + zombie_ids = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=internet_gateways_data, + output_tag='InternetGatewayId', cluster_tag=cluster_tag, + input_tag='Attachments') else: zombie_ids = [zombie] - cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) + cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days( + resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, + cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: for zombie_id in zombie_ids: - self.delete_ec2_resource.delete_zombie_resource(resource='internet_gateway', resource_id=zombie_id, vpc_id=vpc_id, cluster_tag=cluster_tag) + self.delete_ec2_resource.delete_zombie_resource(resource='internet_gateway', + resource_id=zombie_id, vpc_id=vpc_id, + cluster_tag=cluster_tag) else: if self._force_delete and self.delete: for zombie_id in zombie_ids: - self.delete_ec2_resource.delete_zombie_resource(resource='internet_gateway', resource_id=zombie_id, vpc_id=vpc_id, cluster_tag=cluster_tag) + self.delete_ec2_resource.delete_zombie_resource(resource='internet_gateway', + resource_id=zombie_id, vpc_id=vpc_id, + cluster_tag=cluster_tag) return zombies, cluster_left_out_days @@ -604,24 +696,31 @@ def zombie_cluster_dhcp_option(self, vpc_id: str = '', cluster_tag_vpc: str = '' This method returns list of cluster's dhcp option according to cluster tag name and cluster name data """ dhcp_options_data = self.ec2_operations.get_dhcp_options() - exist_dhcp_option = self.__get_cluster_resources(resources_list=dhcp_options_data, input_resource_id='DhcpOptionsId') + exist_dhcp_option = self.__get_cluster_resources(resources_list=dhcp_options_data, + input_resource_id='DhcpOptionsId') zombies = self.__get_zombie_resources(exist_dhcp_option) - resources = self._get_tags_of_zombie_resources(resources=dhcp_options_data, resource_id_name='DhcpOptionsId', zombies=zombies, aws_service='ec2') + resources = self._get_tags_of_zombie_resources(resources=dhcp_options_data, resource_id_name='DhcpOptionsId', + zombies=zombies, aws_service='ec2') cluster_left_out_days = {} vpcs = self.ec2_client.describe_vpcs()['Vpcs'] if zombies: for zombie, cluster_tag in zombies.items(): - vpc_id = self.__extract_vpc_id_from_resource_data(zombie_id=zombie, resource_data=vpcs, input_tag='DhcpOptionsId') - cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) + vpc_id = self.__extract_vpc_id_from_resource_data(zombie_id=zombie, resource_data=vpcs, + input_tag='DhcpOptionsId') + cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days( + resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, + cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: if vpc_id: - self.delete_ec2_resource.delete_zombie_resource(resource='dhcp_options', resource_id=zombie, vpc_id=vpc_id) + self.delete_ec2_resource.delete_zombie_resource(resource='dhcp_options', resource_id=zombie, + vpc_id=vpc_id) else: self.delete_ec2_resource.delete_zombie_resource(resource='dhcp_options', resource_id=zombie) else: if self._force_delete and self.delete: if vpc_id: - self.delete_ec2_resource.delete_zombie_resource(resource='dhcp_options', resource_id=zombie, vpc_id=vpc_id) + self.delete_ec2_resource.delete_zombie_resource(resource='dhcp_options', resource_id=zombie, + vpc_id=vpc_id) else: self.delete_ec2_resource.delete_zombie_resource(resource='dhcp_options', resource_id=zombie) return zombies, cluster_left_out_days @@ -631,28 +730,38 @@ def zombie_cluster_vpc_endpoint(self, vpc_id: str = '', cluster_tag_vpc: str = ' This method returns list of cluster's vpc endpoint according to cluster tag name and cluster name data """ vpc_endpoints_data = self.ec2_operations.get_vpce() - exist_vpc_endpoint = self.__get_cluster_resources(resources_list=vpc_endpoints_data, input_resource_id='VpcEndpointId') + exist_vpc_endpoint = self.__get_cluster_resources(resources_list=vpc_endpoints_data, + input_resource_id='VpcEndpointId') zombies = self.__get_zombie_resources(exist_vpc_endpoint) if not zombies and vpc_id: - zombies = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=vpc_endpoints_data, output_tag='VpcEndpointId', cluster_tag=cluster_tag_vpc) - resources = self._get_tags_of_zombie_resources(resources=vpc_endpoints_data, resource_id_name='VpcEndpointId', zombies=zombies, aws_service='ec2') + zombies = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=vpc_endpoints_data, + output_tag='VpcEndpointId', cluster_tag=cluster_tag_vpc) + resources = self._get_tags_of_zombie_resources(resources=vpc_endpoints_data, resource_id_name='VpcEndpointId', + zombies=zombies, aws_service='ec2') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): vpc_endpoints_data = self.ec2_operations.get_vpce() - vpc_id = self.__extract_vpc_id_from_resource_data(zombie_id=zombie, resource_data=vpc_endpoints_data, input_tag='VpcEndpointId') + vpc_id = self.__extract_vpc_id_from_resource_data(zombie_id=zombie, resource_data=vpc_endpoints_data, + input_tag='VpcEndpointId') if vpc_id: - zombie_ids = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=vpc_endpoints_data, output_tag='VpcEndpointId', cluster_tag=cluster_tag) + zombie_ids = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=vpc_endpoints_data, + output_tag='VpcEndpointId', cluster_tag=cluster_tag) else: zombie_ids = [zombies] - cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) + cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days( + resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, + cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: for zombie_id in zombie_ids: - self.delete_ec2_resource.delete_zombie_resource(resource='vpc_endpoints', resource_id=zombie_id, cluster_tag=cluster_tag) + self.delete_ec2_resource.delete_zombie_resource(resource='vpc_endpoints', resource_id=zombie_id, + cluster_tag=cluster_tag) else: if self._force_delete and self.delete: for zombie_id in zombie_ids: - self.delete_ec2_resource.delete_zombie_resource(resource='vpc_endpoints', resource_id=zombie_id, cluster_tag=cluster_tag) + self.delete_ec2_resource.delete_zombie_resource(resource='vpc_endpoints', + resource_id=zombie_id, + cluster_tag=cluster_tag) return zombies, cluster_left_out_days def zombie_cluster_nat_gateway(self, vpc_id: str = '', cluster_tag_vpc: str = ''): @@ -660,25 +769,34 @@ def zombie_cluster_nat_gateway(self, vpc_id: str = '', cluster_tag_vpc: str = '' This method returns list of zombie cluster's nat gateway according to cluster tag name and cluster name data """ nat_gateways_data = self.ec2_operations.get_nat_gateways() - exist_nat_gateway = self.__get_cluster_resources(resources_list=nat_gateways_data, input_resource_id='NatGatewayId') + exist_nat_gateway = self.__get_cluster_resources(resources_list=nat_gateways_data, + input_resource_id='NatGatewayId') zombies = self.__get_zombie_resources(exist_nat_gateway) if not zombies and vpc_id: - zombies = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=nat_gateways_data, output_tag='NatGatewayId', cluster_tag=cluster_tag_vpc) - resources = self._get_tags_of_zombie_resources(resources=nat_gateways_data, resource_id_name='NatGatewayId', zombies=zombies, aws_service='ec2') + zombies = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=nat_gateways_data, + output_tag='NatGatewayId', cluster_tag=cluster_tag_vpc) + resources = self._get_tags_of_zombie_resources(resources=nat_gateways_data, resource_id_name='NatGatewayId', + zombies=zombies, aws_service='ec2') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): nat_gateways_data = self.ec2_operations.get_nat_gateways() vpc_id = self.__extract_vpc_id_from_resource_data(zombie, nat_gateways_data, input_tag='NatGatewayId') - zombie_ids = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=nat_gateways_data, output_tag='NatGatewayId', cluster_tag=cluster_tag) - cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) + zombie_ids = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=nat_gateways_data, + output_tag='NatGatewayId', cluster_tag=cluster_tag) + cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days( + resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, + cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: for zombie_id in zombie_ids: - self.delete_ec2_resource.delete_zombie_resource(resource='nat_gateways', resource_id=zombie_id, cluster_tag=cluster_tag) + self.delete_ec2_resource.delete_zombie_resource(resource='nat_gateways', resource_id=zombie_id, + cluster_tag=cluster_tag) else: if self._force_delete and self.delete: for zombie_id in zombie_ids: - self.delete_ec2_resource.delete_zombie_resource(resource='nat_gateways', resource_id=zombie_id, cluster_tag=cluster_tag) + self.delete_ec2_resource.delete_zombie_resource(resource='nat_gateways', + resource_id=zombie_id, + cluster_tag=cluster_tag) return zombies, cluster_left_out_days def zombie_cluster_network_acl(self, vpc_id: str = '', cluster_tag_vpc: str = ''): @@ -699,23 +817,32 @@ def zombie_cluster_network_acl(self, vpc_id: str = '', cluster_tag_vpc: str = '' zombie_resources[key] = value zombies = zombie_resources if not zombies and vpc_id: - zombies = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=network_acls_data, output_tag='NetworkAclId', cluster_tag=cluster_tag_vpc) - resources = self._get_tags_of_zombie_resources(resources=network_acls_data, resource_id_name='NetworkAclId', zombies=zombies, aws_service='ec2') + zombies = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=network_acls_data, + output_tag='NetworkAclId', cluster_tag=cluster_tag_vpc) + resources = self._get_tags_of_zombie_resources(resources=network_acls_data, resource_id_name='NetworkAclId', + zombies=zombies, aws_service='ec2') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): network_acls_data = self.ec2_operations.get_nacls() if not vpc_id: - vpc_id = self.__extract_vpc_id_from_resource_data(zombie_id=zombie, resource_data=network_acls_data, input_tag='NetworkAclId') - zombie_ids = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=network_acls_data, output_tag='NetworkAclId', cluster_tag=cluster_tag) - cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) + vpc_id = self.__extract_vpc_id_from_resource_data(zombie_id=zombie, resource_data=network_acls_data, + input_tag='NetworkAclId') + zombie_ids = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=network_acls_data, + output_tag='NetworkAclId', cluster_tag=cluster_tag) + cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days( + resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, + cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: for zombie_id in zombie_ids: - self.delete_ec2_resource.delete_zombie_resource(resource='network_acl', resource_id=zombie_id, vpc_id=vpc_id, cluster_tag=cluster_tag) + self.delete_ec2_resource.delete_zombie_resource(resource='network_acl', resource_id=zombie_id, + vpc_id=vpc_id, cluster_tag=cluster_tag) else: if self._force_delete and self.delete: for zombie_id in zombie_ids: - self.delete_ec2_resource.delete_zombie_resource(resource='network_acl', resource_id=zombie_id, vpc_id=vpc_id, cluster_tag=cluster_tag) + self.delete_ec2_resource.delete_zombie_resource(resource='network_acl', + resource_id=zombie_id, vpc_id=vpc_id, + cluster_tag=cluster_tag) return zombies, cluster_left_out_days def zombie_cluster_role(self): @@ -724,7 +851,8 @@ def zombie_cluster_role(self): * Role is a global resource, need to scan for live cluster in all regions """ exist_role_name_tag = {} - roles_data = self.__get_details_resource_list(func_name=self.iam_client.list_roles, input_tag='Roles', check_tag='Marker') + roles_data = self.__get_details_resource_list(func_name=self.iam_client.list_roles, input_tag='Roles', + check_tag='Marker') for role in roles_data: role_name = role['RoleName'] if 'worker-role' in role_name or 'master-role' in role_name: @@ -744,15 +872,20 @@ def zombie_cluster_role(self): cluster_left_out_days = {} if exist_role_name_tag: zombies = self.__get_all_zombie_resources(exist_role_name_tag) - resources = self._get_tags_of_zombie_resources(resources=roles_data, resource_id_name='RoleName', zombies=zombies, aws_service='role') + resources = self._get_tags_of_zombie_resources(resources=roles_data, resource_id_name='RoleName', + zombies=zombies, aws_service='role') if zombies: for zombie, cluster_tag in zombies.items(): - cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) + cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days( + resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, + cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: - self.delete_iam_resource.delete_iam_zombie_resource(resource_id=zombie, resource_type='iam_role') + self.delete_iam_resource.delete_iam_zombie_resource(resource_id=zombie, + resource_type='iam_role') else: if self._force_delete and self.delete: - self.delete_iam_resource.delete_iam_zombie_resource(resource_id=zombie, resource_type='iam_role') + self.delete_iam_resource.delete_iam_zombie_resource(resource_id=zombie, + resource_type='iam_role') return zombies, cluster_left_out_days def zombie_cluster_user(self): @@ -761,7 +894,8 @@ def zombie_cluster_user(self): * User is a global resource, need to scan for live cluster in all regions """ exist_user_name_tag = {} - users_data = self.__get_details_resource_list(func_name=self.iam_client.list_users, input_tag='Users', check_tag='Marker') + users_data = self.__get_details_resource_list(func_name=self.iam_client.list_users, input_tag='Users', + check_tag='Marker') for user in users_data: user_name = user['UserName'] user_data = self.iam_client.get_user(UserName=user_name) @@ -777,16 +911,20 @@ def zombie_cluster_user(self): exist_user_name_tag[user_name] = tag['Key'] break zombies = self.__get_all_zombie_resources(exist_user_name_tag) - resources = self._get_tags_of_zombie_resources(resources=users_data, resource_id_name='UserName', zombies=zombies, aws_service='user') + resources = self._get_tags_of_zombie_resources(resources=users_data, resource_id_name='UserName', + zombies=zombies, aws_service='user') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): - cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) + cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days( + resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, + cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: self.delete_iam_resource.delete_iam_zombie_resource(resource_id=zombie, resource_type='iam_user') else: if self._force_delete and self.delete: - self.delete_iam_resource.delete_iam_zombie_resource(resource_id=zombie, resource_type='iam_user') + self.delete_iam_resource.delete_iam_zombie_resource(resource_id=zombie, + resource_type='iam_user') return zombies, cluster_left_out_days @@ -816,11 +954,14 @@ def zombie_cluster_s3_bucket(self, cluster_stamp: str = 'image-registry'): exist_bucket_name_tag[bucket['Name']] = tag['Key'] break zombies = self.__get_all_zombie_resources(exist_bucket_name_tag) - resources = self._get_tags_of_zombie_resources(resources=response['Buckets'], resource_id_name='Name', zombies=zombies, aws_service='bucket', aws_tag='TagSet') + resources = self._get_tags_of_zombie_resources(resources=response['Buckets'], resource_id_name='Name', + zombies=zombies, aws_service='bucket', aws_tag='TagSet') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): - cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) + cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days( + resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, + cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: self.delete_s3_resource.delete_zombie_s3_resource(resource_type='s3_bucket', resource_id=zombie) else: