Update boto3 client with aws_session_token (#521)

* adding logging to validate template * moving logging to validate template * adding error logging * streaming boto3 logs * streaming boto3 logs * try adding AWS_SESSION_TOKEN * Cleanup * add AWS_SESSION_TOKEN to boto clients and test suites * add newline at EOF * update verbiage for test/integration_*.py * add support for iam user, role and temporary access credentials sourced from environment * updating verbiage for comments * Simplify * format * version bump Co-authored-by: Meethil Yadav <[email protected]> Co-authored-by: Alex Kennedy <[email protected]> Co-authored-by: Alex Kennedy <[email protected]>
Nextdoor · Oct 20, 2022 · 7e9adfa · 7e9adfa
1 parent a312482
commit 7e9adfa
Show file tree

Hide file tree

Showing 10 changed files with 48 additions and 31 deletions.
diff --git a/kingpin/actors/aws/base.py b/kingpin/actors/aws/base.py
@@ -30,6 +30,10 @@
 
 :AWS_SECRET_ACCESS_KEY:
   Your AWS secret
+
+:AWS_SESSION_TOKEN:
+  Your AWS session token
+  Only needed if you are using temporary access credentials
 """
 
 import logging
@@ -86,19 +90,23 @@ def __init__(self, *args, **kwargs):
         # By default, we will try to let Boto handle discovering its
         # credentials at instantiation time. This _can_ result in synchronous
         # API calls to the Metadata service, but those should be fast.
-        key = None
-        secret = None
-
+        #
         # In the event though that someone has explicitly set the AWS access
         # keys in the environment (either for the purposes of a unit test, or
         # because they wanted to), we use those values.
-        if aws_settings.AWS_ACCESS_KEY_ID and aws_settings.AWS_SECRET_ACCESS_KEY:
-            key = aws_settings.AWS_ACCESS_KEY_ID
-            secret = aws_settings.AWS_SECRET_ACCESS_KEY
+        #
+        # Note: these get defualted to None in aws_settings if they are not
+        # found which will tell boto3 to fallback to default behavior.
+        boto3_client_kwargs = {}
+        boto3_client_kwargs["aws_access_key_id"] = aws_settings.AWS_ACCESS_KEY_ID
+        boto3_client_kwargs[
+            "aws_secret_access_key"
+        ] = aws_settings.AWS_SECRET_ACCESS_KEY
+        boto3_client_kwargs["aws_session_token"] = aws_settings.AWS_SESSION_TOKEN
 
         # Establish connection objects that don't require a region
         self.iam_conn = boto3.client(
-            "iam", aws_access_key_id=key, aws_secret_access_key=secret
+            service_name="iam", config=None, **boto3_client_kwargs
         )
 
         # Establish region-specific connection objects.
@@ -114,29 +122,18 @@ def __init__(self, *args, **kwargs):
                 "mode": "adaptive",
             },
         )
+
         self.ecs_conn = boto3.client(
-            "ecs",
-            config=boto_config,
-            aws_access_key_id=key,
-            aws_secret_access_key=secret,
+            service_name="ecs", config=boto_config, **boto3_client_kwargs
         )
         self.cf3_conn = boto3.client(
-            "cloudformation",
-            config=boto_config,
-            aws_access_key_id=key,
-            aws_secret_access_key=secret,
+            service_name="cloudformation", config=boto_config, **boto3_client_kwargs
         )
         self.sqs_conn = boto3.client(
-            "sqs",
-            config=boto_config,
-            aws_access_key_id=key,
-            aws_secret_access_key=secret,
+            service_name="sqs", config=boto_config, **boto3_client_kwargs
         )
         self.s3_conn = boto3.client(
-            "s3",
-            config=boto_config,
-            aws_access_key_id=key,
-            aws_secret_access_key=secret,
+            service_name="s3", config=boto_config, **boto3_client_kwargs
         )
 
     @concurrent.run_on_executor

diff --git a/kingpin/actors/aws/iam/test/test_entities.py b/kingpin/actors/aws/iam/test/test_entities.py
@@ -26,6 +26,7 @@ def setUp(self):
         super(TestEntityBaseActor, self).setUp()
         settings.AWS_ACCESS_KEY_ID = "unit-test"
         settings.AWS_SECRET_ACCESS_KEY = "unit-test"
+        settings.AWS_SESSION_TOKEN = "unit-test"
         importlib.reload(entities)
 
         # Create our actor object with some basics... then mock out the IAM
@@ -497,6 +498,7 @@ def setUp(self):
         super(TestUser, self).setUp()
         settings.AWS_ACCESS_KEY_ID = "unit-test"
         settings.AWS_SECRET_ACCESS_KEY = "unit-test"
+        settings.AWS_SESSION_TOKEN = "unit-test"
         importlib.reload(entities)
 
         # Create our actor object with some basics... then mock out the IAM
@@ -686,6 +688,7 @@ def setUp(self):
         super(TestGroup, self).setUp()
         settings.AWS_ACCESS_KEY_ID = "unit-test"
         settings.AWS_SECRET_ACCESS_KEY = "unit-test"
+        settings.AWS_SESSION_TOKEN = "unit-test"
         importlib.reload(entities)
 
         # Create our actor object with some basics... then mock out the IAM
@@ -845,6 +848,7 @@ def setUp(self):
         super(TestRole, self).setUp()
         settings.AWS_ACCESS_KEY_ID = "unit-test"
         settings.AWS_SECRET_ACCESS_KEY = "unit-test"
+        settings.AWS_SESSION_TOKEN = "unit-test"
         importlib.reload(entities)
 
         # Create our actor object with some basics... then mock out the IAM
@@ -1055,6 +1059,7 @@ def setUp(self):
         super(TestInstanceProfile, self).setUp()
         settings.AWS_ACCESS_KEY_ID = "unit-test"
         settings.AWS_SECRET_ACCESS_KEY = "unit-test"
+        settings.AWS_SESSION_TOKEN = "unit-test"
         importlib.reload(entities)
 
         # Create our actor object with some basics... then mock out the IAM

diff --git a/kingpin/actors/aws/settings.py b/kingpin/actors/aws/settings.py
@@ -31,3 +31,4 @@
 # During tests, we mock these out to blank strings to prevent these calls.
 AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID", None)
 AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY", None)
+AWS_SESSION_TOKEN = os.getenv("AWS_SESSION_TOKEN", None)
diff --git a/kingpin/actors/aws/test/integration_base.py b/kingpin/actors/aws/test/integration_base.py
@@ -30,6 +30,7 @@ def integration_01a_check_credentials(self):
 
         settings.AWS_ACCESS_KEY_ID = "fake"
         settings.AWS_SECRET_ACCESS_KEY = "fake"
+        settings.AWS_SESSION_TOKEN = "fake"
         actor = base.AWSBaseActor("Test", {"region": self.region})
 
         # Executing a random function call that is wrapped in _retry.

diff --git a/kingpin/actors/aws/test/integration_cloudformation.py b/kingpin/actors/aws/test/integration_cloudformation.py
@@ -25,8 +25,11 @@ class IntegrationCreate(testing.AsyncTestCase):
     * Delete that same stack
 
     Requirements:
-        Your AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY must have access to
-        create CF stacks. The stack we create is extremely simple, and should
+        You must have an AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY if using
+        an IAM user/role and also an AWS_SESSION_TOKEN if using temporary access credentials,
+        with permissions to create CF stacks.
+
+        The stack we create is extremely simple, and should
         impact none of your AWS resources. The stack creates a simple S3
         bucket, so your credentials must have access to create that buckets.
 
@@ -103,8 +106,11 @@ class IntegrationStack(testing.AsyncTestCase):
     * Delete that same stack
 
     Requirements:
-        Your AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY must have access to
-        create CF stacks. The stack we create is extremely simple, and should
+        You must have an AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY if using
+        an IAM user/role and also an AWS_SESSION_TOKEN if using temporary access credentials,
+        with permissions to create CF stacks.
+
+        The stack we create is extremely simple, and should
         impact none of your AWS resources. The stack creates a simple S3
         bucket, so your credentials must have access to create that buckets.
 

diff --git a/kingpin/actors/aws/test/integration_s3.py b/kingpin/actors/aws/test/integration_s3.py
@@ -27,9 +27,10 @@ class IntegrationS3(testing.AsyncTestCase):
     Requirements:
         You have to create an S3 Bucket named kingpin-integration-test and
         place it in the specified region (default us-east-1).
-        As with other tests, environment variables AWS_ACCESS_KEY_ID and
-        AWS_SECRET_ACCESS_KEY are expected, and the key should have
-        permissions to read S3 bucket information.
+
+        You must have an AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY if using
+        an IAM user/role and also an AWS_SESSION_TOKEN if using temporary access credentials,
+        with permissions to read S3 bucket information.
 
     Note, these tests must be run in-order. The order is defined by
     their definition order in this file. Nose follows this order according

diff --git a/kingpin/actors/aws/test/test_base.py b/kingpin/actors/aws/test/test_base.py
@@ -52,6 +52,7 @@ def setUp(self):
         super(TestBase, self).setUp()
         settings.AWS_ACCESS_KEY_ID = "unit-test"
         settings.AWS_SECRET_ACCESS_KEY = "unit-test"
+        settings.AWS_SESSION_TOKEN = "unit-test"
         importlib.reload(base)
 
     @testing.gen_test

diff --git a/kingpin/actors/aws/test/test_cloudformation.py b/kingpin/actors/aws/test/test_cloudformation.py
@@ -54,6 +54,7 @@ def setUp(self):
         super(TestCloudFormationBaseActor, self).setUp()
         settings.AWS_ACCESS_KEY_ID = "unit-test"
         settings.AWS_SECRET_ACCESS_KEY = "unit-test"
+        settings.AWS_SESSION_TOKEN = "unit-test"
         importlib.reload(cloudformation)
 
         self.actor = cloudformation.CloudFormationBaseActor(
@@ -401,6 +402,7 @@ def setUp(self):
         super(TestCreate, self).setUp()
         settings.AWS_ACCESS_KEY_ID = "unit-test"
         settings.AWS_SECRET_ACCESS_KEY = "unit-test"
+        settings.AWS_SESSION_TOKEN = "unit-test"
         importlib.reload(cloudformation)
         # Need to recreate the api call queues between tests
         # because nose creates a new ioloop per test run.
@@ -649,6 +651,7 @@ def setUp(self):
         super(TestDelete, self).setUp()
         settings.AWS_ACCESS_KEY_ID = "unit-test"
         settings.AWS_SECRET_ACCESS_KEY = "unit-test"
+        settings.AWS_SESSION_TOKEN = "unit-test"
         importlib.reload(cloudformation)
         # Need to recreate the api call queues between tests
         # because nose creates a new ioloop per test run.
@@ -694,6 +697,7 @@ def setUp(self):
         super(TestStack, self).setUp()
         settings.AWS_ACCESS_KEY_ID = "unit-test"
         settings.AWS_SECRET_ACCESS_KEY = "unit-test"
+        settings.AWS_SESSION_TOKEN = "unit-test"
         importlib.reload(cloudformation)
         # Need to recreate the api call queues between tests
         # because nose creates a new ioloop per test run.

diff --git a/kingpin/actors/aws/test/test_s3.py b/kingpin/actors/aws/test/test_s3.py
@@ -17,6 +17,7 @@ def setUp(self):
         super(TestBucket, self).setUp()
         settings.AWS_ACCESS_KEY_ID = "unit-test"
         settings.AWS_SECRET_ACCESS_KEY = "unit-test"
+        settings.AWS_SESSION_TOKEN = "unit-test"
         importlib.reload(s3_actor)
 
         self.actor = s3_actor.Bucket(

diff --git a/kingpin/version.py b/kingpin/version.py
@@ -13,4 +13,4 @@
 # Copyright 2018 Nextdoor.com, Inc
 
 
-__version__ = "2.0.3"
+__version__ = "2.1.0"
Original file line number	Diff line number	Diff line change
Expand Up		@@ -13,4 +13,4 @@
		# Copyright 2018 Nextdoor.com, Inc


		__version__ = "2.0.3"
		__version__ = "2.1.0"