From 7e9adfaf0b181f04d6bc8bf0255d9e0e4e255a64 Mon Sep 17 00:00:00 2001 From: David White <107427321+davidwhite-nd@users.noreply.github.com> Date: Wed, 19 Oct 2022 20:25:55 -0700 Subject: [PATCH] Update boto3 client with aws_session_token (#521) * adding logging to validate template * moving logging to validate template * adding error logging * streaming boto3 logs * streaming boto3 logs * try adding AWS_SESSION_TOKEN * Cleanup * add AWS_SESSION_TOKEN to boto clients and test suites * add newline at EOF * update verbiage for test/integration_*.py * add support for iam user, role and temporary access credentials sourced from environment * updating verbiage for comments * Simplify * format * version bump Co-authored-by: Meethil Yadav Co-authored-by: Alex Kennedy Co-authored-by: Alex Kennedy --- kingpin/actors/aws/base.py | 43 +++++++++---------- kingpin/actors/aws/iam/test/test_entities.py | 5 +++ kingpin/actors/aws/settings.py | 1 + kingpin/actors/aws/test/integration_base.py | 1 + .../aws/test/integration_cloudformation.py | 14 ++++-- kingpin/actors/aws/test/integration_s3.py | 7 +-- kingpin/actors/aws/test/test_base.py | 1 + .../actors/aws/test/test_cloudformation.py | 4 ++ kingpin/actors/aws/test/test_s3.py | 1 + kingpin/version.py | 2 +- 10 files changed, 48 insertions(+), 31 deletions(-) diff --git a/kingpin/actors/aws/base.py b/kingpin/actors/aws/base.py index bee99180..ac3a59d8 100644 --- a/kingpin/actors/aws/base.py +++ b/kingpin/actors/aws/base.py @@ -30,6 +30,10 @@ :AWS_SECRET_ACCESS_KEY: Your AWS secret + +:AWS_SESSION_TOKEN: + Your AWS session token + Only needed if you are using temporary access credentials """ import logging @@ -86,19 +90,23 @@ def __init__(self, *args, **kwargs): # By default, we will try to let Boto handle discovering its # credentials at instantiation time. This _can_ result in synchronous # API calls to the Metadata service, but those should be fast. - key = None - secret = None - + # # In the event though that someone has explicitly set the AWS access # keys in the environment (either for the purposes of a unit test, or # because they wanted to), we use those values. - if aws_settings.AWS_ACCESS_KEY_ID and aws_settings.AWS_SECRET_ACCESS_KEY: - key = aws_settings.AWS_ACCESS_KEY_ID - secret = aws_settings.AWS_SECRET_ACCESS_KEY + # + # Note: these get defualted to None in aws_settings if they are not + # found which will tell boto3 to fallback to default behavior. + boto3_client_kwargs = {} + boto3_client_kwargs["aws_access_key_id"] = aws_settings.AWS_ACCESS_KEY_ID + boto3_client_kwargs[ + "aws_secret_access_key" + ] = aws_settings.AWS_SECRET_ACCESS_KEY + boto3_client_kwargs["aws_session_token"] = aws_settings.AWS_SESSION_TOKEN # Establish connection objects that don't require a region self.iam_conn = boto3.client( - "iam", aws_access_key_id=key, aws_secret_access_key=secret + service_name="iam", config=None, **boto3_client_kwargs ) # Establish region-specific connection objects. @@ -114,29 +122,18 @@ def __init__(self, *args, **kwargs): "mode": "adaptive", }, ) + self.ecs_conn = boto3.client( - "ecs", - config=boto_config, - aws_access_key_id=key, - aws_secret_access_key=secret, + service_name="ecs", config=boto_config, **boto3_client_kwargs ) self.cf3_conn = boto3.client( - "cloudformation", - config=boto_config, - aws_access_key_id=key, - aws_secret_access_key=secret, + service_name="cloudformation", config=boto_config, **boto3_client_kwargs ) self.sqs_conn = boto3.client( - "sqs", - config=boto_config, - aws_access_key_id=key, - aws_secret_access_key=secret, + service_name="sqs", config=boto_config, **boto3_client_kwargs ) self.s3_conn = boto3.client( - "s3", - config=boto_config, - aws_access_key_id=key, - aws_secret_access_key=secret, + service_name="s3", config=boto_config, **boto3_client_kwargs ) @concurrent.run_on_executor diff --git a/kingpin/actors/aws/iam/test/test_entities.py b/kingpin/actors/aws/iam/test/test_entities.py index a2ed77be..da5bde90 100644 --- a/kingpin/actors/aws/iam/test/test_entities.py +++ b/kingpin/actors/aws/iam/test/test_entities.py @@ -26,6 +26,7 @@ def setUp(self): super(TestEntityBaseActor, self).setUp() settings.AWS_ACCESS_KEY_ID = "unit-test" settings.AWS_SECRET_ACCESS_KEY = "unit-test" + settings.AWS_SESSION_TOKEN = "unit-test" importlib.reload(entities) # Create our actor object with some basics... then mock out the IAM @@ -497,6 +498,7 @@ def setUp(self): super(TestUser, self).setUp() settings.AWS_ACCESS_KEY_ID = "unit-test" settings.AWS_SECRET_ACCESS_KEY = "unit-test" + settings.AWS_SESSION_TOKEN = "unit-test" importlib.reload(entities) # Create our actor object with some basics... then mock out the IAM @@ -686,6 +688,7 @@ def setUp(self): super(TestGroup, self).setUp() settings.AWS_ACCESS_KEY_ID = "unit-test" settings.AWS_SECRET_ACCESS_KEY = "unit-test" + settings.AWS_SESSION_TOKEN = "unit-test" importlib.reload(entities) # Create our actor object with some basics... then mock out the IAM @@ -845,6 +848,7 @@ def setUp(self): super(TestRole, self).setUp() settings.AWS_ACCESS_KEY_ID = "unit-test" settings.AWS_SECRET_ACCESS_KEY = "unit-test" + settings.AWS_SESSION_TOKEN = "unit-test" importlib.reload(entities) # Create our actor object with some basics... then mock out the IAM @@ -1055,6 +1059,7 @@ def setUp(self): super(TestInstanceProfile, self).setUp() settings.AWS_ACCESS_KEY_ID = "unit-test" settings.AWS_SECRET_ACCESS_KEY = "unit-test" + settings.AWS_SESSION_TOKEN = "unit-test" importlib.reload(entities) # Create our actor object with some basics... then mock out the IAM diff --git a/kingpin/actors/aws/settings.py b/kingpin/actors/aws/settings.py index 2abd30ca..e7582ff2 100644 --- a/kingpin/actors/aws/settings.py +++ b/kingpin/actors/aws/settings.py @@ -31,3 +31,4 @@ # During tests, we mock these out to blank strings to prevent these calls. AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID", None) AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY", None) +AWS_SESSION_TOKEN = os.getenv("AWS_SESSION_TOKEN", None) diff --git a/kingpin/actors/aws/test/integration_base.py b/kingpin/actors/aws/test/integration_base.py index eaf57372..6757c0c9 100644 --- a/kingpin/actors/aws/test/integration_base.py +++ b/kingpin/actors/aws/test/integration_base.py @@ -30,6 +30,7 @@ def integration_01a_check_credentials(self): settings.AWS_ACCESS_KEY_ID = "fake" settings.AWS_SECRET_ACCESS_KEY = "fake" + settings.AWS_SESSION_TOKEN = "fake" actor = base.AWSBaseActor("Test", {"region": self.region}) # Executing a random function call that is wrapped in _retry. diff --git a/kingpin/actors/aws/test/integration_cloudformation.py b/kingpin/actors/aws/test/integration_cloudformation.py index e566a5e3..90b474d2 100644 --- a/kingpin/actors/aws/test/integration_cloudformation.py +++ b/kingpin/actors/aws/test/integration_cloudformation.py @@ -25,8 +25,11 @@ class IntegrationCreate(testing.AsyncTestCase): * Delete that same stack Requirements: - Your AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY must have access to - create CF stacks. The stack we create is extremely simple, and should + You must have an AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY if using + an IAM user/role and also an AWS_SESSION_TOKEN if using temporary access credentials, + with permissions to create CF stacks. + + The stack we create is extremely simple, and should impact none of your AWS resources. The stack creates a simple S3 bucket, so your credentials must have access to create that buckets. @@ -103,8 +106,11 @@ class IntegrationStack(testing.AsyncTestCase): * Delete that same stack Requirements: - Your AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY must have access to - create CF stacks. The stack we create is extremely simple, and should + You must have an AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY if using + an IAM user/role and also an AWS_SESSION_TOKEN if using temporary access credentials, + with permissions to create CF stacks. + + The stack we create is extremely simple, and should impact none of your AWS resources. The stack creates a simple S3 bucket, so your credentials must have access to create that buckets. diff --git a/kingpin/actors/aws/test/integration_s3.py b/kingpin/actors/aws/test/integration_s3.py index 0615588d..5b43729a 100644 --- a/kingpin/actors/aws/test/integration_s3.py +++ b/kingpin/actors/aws/test/integration_s3.py @@ -27,9 +27,10 @@ class IntegrationS3(testing.AsyncTestCase): Requirements: You have to create an S3 Bucket named kingpin-integration-test and place it in the specified region (default us-east-1). - As with other tests, environment variables AWS_ACCESS_KEY_ID and - AWS_SECRET_ACCESS_KEY are expected, and the key should have - permissions to read S3 bucket information. + + You must have an AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY if using + an IAM user/role and also an AWS_SESSION_TOKEN if using temporary access credentials, + with permissions to read S3 bucket information. Note, these tests must be run in-order. The order is defined by their definition order in this file. Nose follows this order according diff --git a/kingpin/actors/aws/test/test_base.py b/kingpin/actors/aws/test/test_base.py index 64cf6250..7f6c74db 100644 --- a/kingpin/actors/aws/test/test_base.py +++ b/kingpin/actors/aws/test/test_base.py @@ -52,6 +52,7 @@ def setUp(self): super(TestBase, self).setUp() settings.AWS_ACCESS_KEY_ID = "unit-test" settings.AWS_SECRET_ACCESS_KEY = "unit-test" + settings.AWS_SESSION_TOKEN = "unit-test" importlib.reload(base) @testing.gen_test diff --git a/kingpin/actors/aws/test/test_cloudformation.py b/kingpin/actors/aws/test/test_cloudformation.py index b59f5aad..d02640db 100644 --- a/kingpin/actors/aws/test/test_cloudformation.py +++ b/kingpin/actors/aws/test/test_cloudformation.py @@ -54,6 +54,7 @@ def setUp(self): super(TestCloudFormationBaseActor, self).setUp() settings.AWS_ACCESS_KEY_ID = "unit-test" settings.AWS_SECRET_ACCESS_KEY = "unit-test" + settings.AWS_SESSION_TOKEN = "unit-test" importlib.reload(cloudformation) self.actor = cloudformation.CloudFormationBaseActor( @@ -401,6 +402,7 @@ def setUp(self): super(TestCreate, self).setUp() settings.AWS_ACCESS_KEY_ID = "unit-test" settings.AWS_SECRET_ACCESS_KEY = "unit-test" + settings.AWS_SESSION_TOKEN = "unit-test" importlib.reload(cloudformation) # Need to recreate the api call queues between tests # because nose creates a new ioloop per test run. @@ -649,6 +651,7 @@ def setUp(self): super(TestDelete, self).setUp() settings.AWS_ACCESS_KEY_ID = "unit-test" settings.AWS_SECRET_ACCESS_KEY = "unit-test" + settings.AWS_SESSION_TOKEN = "unit-test" importlib.reload(cloudformation) # Need to recreate the api call queues between tests # because nose creates a new ioloop per test run. @@ -694,6 +697,7 @@ def setUp(self): super(TestStack, self).setUp() settings.AWS_ACCESS_KEY_ID = "unit-test" settings.AWS_SECRET_ACCESS_KEY = "unit-test" + settings.AWS_SESSION_TOKEN = "unit-test" importlib.reload(cloudformation) # Need to recreate the api call queues between tests # because nose creates a new ioloop per test run. diff --git a/kingpin/actors/aws/test/test_s3.py b/kingpin/actors/aws/test/test_s3.py index deeff8f4..46c5632e 100644 --- a/kingpin/actors/aws/test/test_s3.py +++ b/kingpin/actors/aws/test/test_s3.py @@ -17,6 +17,7 @@ def setUp(self): super(TestBucket, self).setUp() settings.AWS_ACCESS_KEY_ID = "unit-test" settings.AWS_SECRET_ACCESS_KEY = "unit-test" + settings.AWS_SESSION_TOKEN = "unit-test" importlib.reload(s3_actor) self.actor = s3_actor.Bucket( diff --git a/kingpin/version.py b/kingpin/version.py index 72342752..7d02a160 100644 --- a/kingpin/version.py +++ b/kingpin/version.py @@ -13,4 +13,4 @@ # Copyright 2018 Nextdoor.com, Inc -__version__ = "2.0.3" +__version__ = "2.1.0"