diff --git a/principalmapper/graphing/autoscaling_edges.py b/principalmapper/graphing/autoscaling_edges.py index 78e5b2e..e41e32f 100644 --- a/principalmapper/graphing/autoscaling_edges.py +++ b/principalmapper/graphing/autoscaling_edges.py @@ -19,7 +19,8 @@ import logging from typing import Dict, List, Optional -from botocore.exceptions import ClientError +from botocore.config import Config +from botocore.exceptions import ClientError, ConnectTimeoutError, EndpointConnectionError from principalmapper.common import Edge, Node from principalmapper.graphing.edge_checker import EdgeChecker @@ -50,7 +51,11 @@ def return_edges(self, nodes: List[Node], region_allow_list: Optional[List[str]] if self.session is not None: as_regions = botocore_tools.get_regions_to_search(self.session, 'autoscaling', region_allow_list, region_deny_list) for region in as_regions: - autoscaling_clients.append(self.session.create_client('autoscaling', region_name=region, **asargs)) + autoscaling_clients.append(self.session.create_client( + 'autoscaling', region_name=region, + config=Config(connect_timeout=10, retries={'max_attempts': 0}), + **asargs + )) launch_configs = [] for as_client in autoscaling_clients: @@ -66,7 +71,7 @@ def return_edges(self, nodes: List[Node], region_allow_list: Optional[List[str]] 'lc_iip': launch_config['IamInstanceProfile'] }) - except ClientError as ex: + except (ClientError, ConnectTimeoutError, EndpointConnectionError) as ex: logger.warning('Unable to search region {} for launch configs. The region may be disabled, or the error may ' 'be caused by an authorization issue. Continuing.'.format(as_client.meta.region_name)) logger.debug('Exception details: {}'.format(ex)) diff --git a/principalmapper/graphing/cloudformation_edges.py b/principalmapper/graphing/cloudformation_edges.py index b2f867f..0bbd148 100644 --- a/principalmapper/graphing/cloudformation_edges.py +++ b/principalmapper/graphing/cloudformation_edges.py @@ -20,7 +20,8 @@ import os from typing import List, Optional -from botocore.exceptions import ClientError +from botocore.config import Config +from botocore.exceptions import ClientError, ConnectTimeoutError, EndpointConnectionError from principalmapper.common import Edge, Node from principalmapper.graphing.edge_checker import EdgeChecker @@ -52,7 +53,11 @@ def return_edges(self, nodes: List[Node], region_allow_list: Optional[List[str]] if self.session is not None: cf_regions = botocore_tools.get_regions_to_search(self.session, 'cloudformation', region_allow_list, region_deny_list) for region in cf_regions: - cloudformation_clients.append(self.session.create_client('cloudformation', region_name=region, **cfargs)) + cloudformation_clients.append(self.session.create_client( + 'cloudformation', region_name=region, + config=Config(connect_timeout=10, retries={'max_attempts': 0}), + **cfargs + )) # grab existing cloudformation stacks stack_list = [] @@ -65,7 +70,7 @@ def return_edges(self, nodes: List[Node], region_allow_list: Optional[List[str]] if stack['StackStatus'] not in ['CREATE_FAILED', 'DELETE_COMPLETE', 'DELETE_FAILED', 'DELETE_IN_PROGRESS']: # ignore unusable stacks stack_list.append(stack) - except ClientError as ex: + except (ClientError, ConnectTimeoutError, EndpointConnectionError) as ex: logger.warning('Unable to search region {} for stacks. The region may be disabled, or the error may ' 'be caused by an authorization issue. Continuing.'.format(cf_client.meta.region_name)) logger.debug('Exception details: {}'.format(ex)) diff --git a/principalmapper/graphing/codebuild_edges.py b/principalmapper/graphing/codebuild_edges.py index a8b8cd2..d2466fd 100644 --- a/principalmapper/graphing/codebuild_edges.py +++ b/principalmapper/graphing/codebuild_edges.py @@ -19,7 +19,8 @@ import logging from typing import Dict, List, Optional -from botocore.exceptions import ClientError +from botocore.config import Config +from botocore.exceptions import ClientError, ConnectTimeoutError, EndpointConnectionError from principalmapper.common import Edge, Node from principalmapper.graphing.edge_checker import EdgeChecker @@ -51,7 +52,11 @@ def return_edges(self, nodes: List[Node], region_allow_list: Optional[List[str]] if self.session is not None: cf_regions = botocore_tools.get_regions_to_search(self.session, 'codebuild', region_allow_list, region_deny_list) for region in cf_regions: - codebuild_clients.append(self.session.create_client('codebuild', region_name=region, **cbargs)) + codebuild_clients.append(self.session.create_client( + 'codebuild', region_name=region, + config=Config(connect_timeout=10, retries={'max_attempts': 0}), + **cbargs + )) codebuild_projects = [] for cb_client in codebuild_clients: @@ -75,7 +80,7 @@ def return_edges(self, nodes: List[Node], region_allow_list: Optional[List[str]] 'project_tags': project_data['tags'] }) - except ClientError as ex: + except (ClientError, ConnectTimeoutError, EndpointConnectionError) as ex: logger.warning('Unable to search region {} for projects. The region may be disabled, or the error may ' 'be caused by an authorization issue. Continuing.'.format(cb_client.meta.region_name)) logger.debug('Exception details: {}'.format(ex)) diff --git a/principalmapper/graphing/gathering.py b/principalmapper/graphing/gathering.py index 9e64d95..106eebc 100644 --- a/principalmapper/graphing/gathering.py +++ b/principalmapper/graphing/gathering.py @@ -22,6 +22,8 @@ import botocore.session import botocore.exceptions +from botocore.config import Config +from botocore.exceptions import ConnectTimeoutError, EndpointConnectionError import principalmapper from principalmapper.common import Node, Group, Policy, Graph, OrganizationTree, OrganizationNode, OrganizationAccount from principalmapper.graphing import edge_identification @@ -337,7 +339,9 @@ def get_kms_key_policies(session: botocore.session.Session, region_allow_list: O try: # Grab the keys cmks = [] - kmsclient = session.create_client('kms', region_name=kms_region, **kmsargs) + kmsclient = session.create_client('kms', region_name=kms_region, + config=Config(connect_timeout=10, retries={'max_attempts': 0}), + **kmsargs) kms_paginator = kmsclient.get_paginator('list_keys') for page in kms_paginator.paginate(): cmks.extend([x['KeyArn'] for x in page['Keys']]) @@ -351,7 +355,7 @@ def get_kms_key_policies(session: botocore.session.Session, region_allow_list: O json.loads(policy_str) )) logger.info('Caching policy for {}'.format(cmk)) - except botocore.exceptions.ClientError as ex: + except (botocore.exceptions.ClientError, ConnectTimeoutError, EndpointConnectionError) as ex: logger.info('Unable to search KMS in region {} for key policies. The region may be disabled, or the current principal may not be authorized to access the service. Continuing.'.format(kms_region)) logger.debug('Exception was: {}'.format(ex)) continue @@ -376,7 +380,9 @@ def get_sns_topic_policies(session: botocore.session.Session, region_allow_list: try: # Grab the topics topics = [] - snsclient = session.create_client('sns', region_name=sns_region, **snsargs) + snsclient = session.create_client('sns', region_name=sns_region, + config=Config(connect_timeout=10, retries={'max_attempts': 0}), + **snsargs) sns_paginator = snsclient.get_paginator('list_topics') for page in sns_paginator.paginate(): topics.extend([x['TopicArn'] for x in page['Topics']]) @@ -390,7 +396,7 @@ def get_sns_topic_policies(session: botocore.session.Session, region_allow_list: json.loads(policy_str) )) logger.info('Caching policy for {}'.format(topic)) - except botocore.exceptions.ClientError as ex: + except (botocore.exceptions.ClientError, ConnectTimeoutError, EndpointConnectionError) as ex: logger.info('Unable to search SNS in region {} for topic policies. The region may be disabled, or the current principal may not be authorized to access the service. Continuing.'.format(sns_region)) logger.debug('Exception was: {}'.format(ex)) continue @@ -416,7 +422,9 @@ def get_sqs_queue_policies(session: botocore.session.Session, account_id: str, try: # Grab the queue names queue_urls = [] - sqsclient = session.create_client('sqs', region_name=sqs_region, **sqsargs) + sqsclient = session.create_client('sqs', region_name=sqs_region, + config=Config(connect_timeout=10, retries={'max_attempts': 0}), + **sqsargs) response = sqsclient.list_queues() if 'QueueUrls' in response: queue_urls.extend(response['QueueUrls']) @@ -445,7 +453,7 @@ def get_sqs_queue_policies(session: botocore.session.Session, account_id: str, } )) logger.info('Queue {} does not have a queue policy, adding a "stub" policy instead.'.format(queue_name)) - except botocore.exceptions.ClientError as ex: + except (botocore.exceptions.ClientError, ConnectTimeoutError, EndpointConnectionError) as ex: logger.info('Unable to search SQS in region {} for queues. The region may be disabled, or the current principal may not be authorized to access the service. Continuing.'.format(sqs_region)) logger.debug('Exception was: {}'.format(ex)) @@ -469,7 +477,9 @@ def get_secrets_manager_policies(session: botocore.session.Session, region_allow try: # Grab the ARNs of the secrets in this region secret_arns = [] - smclient = session.create_client('secretsmanager', region_name=sm_region, **smargs) + smclient = session.create_client('secretsmanager', region_name=sm_region, + config=Config(connect_timeout=10, retries={'max_attempts': 0}), + **smargs) list_secrets_paginator = smclient.get_paginator('list_secrets') for page in list_secrets_paginator.paginate(): if 'SecretList' in page: @@ -502,7 +512,7 @@ def get_secrets_manager_policies(session: botocore.session.Session, region_allow )) logger.info('Secret {} does not have a resource policy, inserting a "stub" policy instead'.format(secret_arn)) - except botocore.exceptions.ClientError as ex: + except (botocore.exceptions.ClientError, ConnectTimeoutError, EndpointConnectionError) as ex: logger.info('Unable to search Secrets Manager in region {} for secrets. The region may be disabled, or ' 'the current principal may not be authorized to access the service. ' 'Continuing.'.format(sm_region)) diff --git a/principalmapper/graphing/lambda_edges.py b/principalmapper/graphing/lambda_edges.py index 7f6c830..05f1651 100644 --- a/principalmapper/graphing/lambda_edges.py +++ b/principalmapper/graphing/lambda_edges.py @@ -20,7 +20,8 @@ import os from typing import List, Optional -from botocore.exceptions import ClientError +from botocore.config import Config +from botocore.exceptions import ClientError, ConnectTimeoutError, EndpointConnectionError from principalmapper.common import Edge, Node from principalmapper.graphing.edge_checker import EdgeChecker @@ -51,7 +52,11 @@ def return_edges(self, nodes: List[Node], region_allow_list: Optional[List[str]] if self.session is not None: lambda_regions = botocore_tools.get_regions_to_search(self.session, 'lambda', region_allow_list, region_deny_list) for region in lambda_regions: - lambda_clients.append(self.session.create_client('lambda', region_name=region, **lambdaargs)) + lambda_clients.append(self.session.create_client( + 'lambda', region_name=region, + config=Config(connect_timeout=10, retries={'max_attempts': 0}), + **lambdaargs + )) # grab existing lambda functions function_list = [] @@ -61,8 +66,8 @@ def return_edges(self, nodes: List[Node], region_allow_list: Optional[List[str]] for page in paginator.paginate(PaginationConfig={'PageSize': 25}): for func in page['Functions']: function_list.append(func) - except ClientError as ex: - logger.warning('Unable to search region {} for stacks. The region may be disabled, or the error may ' + except (ClientError, ConnectTimeoutError, EndpointConnectionError) as ex: + logger.warning('Unable to search region {} for functions. The region may be disabled, or the error may ' 'be caused by an authorization issue. Continuing.'.format(lambda_client.meta.region_name)) logger.debug('Exception details: {}'.format(ex)) diff --git a/principalmapper/util/case_insensitive_dict.py b/principalmapper/util/case_insensitive_dict.py index ff5ee90..0fd4a99 100644 --- a/principalmapper/util/case_insensitive_dict.py +++ b/principalmapper/util/case_insensitive_dict.py @@ -31,7 +31,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from collections import Mapping, MutableMapping, OrderedDict +from collections import OrderedDict +from collections.abc import Mapping, MutableMapping class CaseInsensitiveDict(MutableMapping):