Merge from aws/aws-sam-cli/develop

appveyor-ubuntu.yml

@@ -303,7 +303,7 @@ for:
 
     test_script:
       - "pip install -e \".[dev]\""
-      - sh: "pytest -vv tests/integration/sync --json-report --json-report-file=TEST_REPORT-integration-sync.json"
+      - sh: "pytest -vv tests/integration/sync -n 3 --reruns 3 --dist loadscope --json-report --json-report-file=TEST_REPORT-integration-sync.json"
 
   # Integ testing local
   -

appveyor-windows.yml

@@ -293,7 +293,7 @@ for:
       - "git --version"
       - "venv\\Scripts\\activate"
       - "docker system prune -a -f"
-      - ps: "pytest -vv tests/integration/sync --json-report --json-report-file=TEST_REPORT-integration-sync.json"
+      - ps: "pytest -vv tests/integration/sync -n 3 --reruns 3 --dist loadscope --json-report --json-report-file=TEST_REPORT-integration-sync.json"
 
   #Integ testing local
   - matrix:

requirements/base.txt

@@ -7,7 +7,7 @@ jmespath~=0.10.0
 ruamel_yaml==0.17.21
 PyYAML>=5.4.1,==5.*
 cookiecutter~=2.1.1
-aws-sam-translator==1.64.0
+aws-sam-translator==1.65.0
 #docker minor version updates can include breaking changes. Auto update micro version only.
 docker~=4.2.0
 dateparser~=1.1

requirements/dev.txt

@@ -12,7 +12,7 @@ boto3-stubs[apigateway,cloudformation,ecr,iam,lambda,s3,schemas,secretsmanager,s
 types-pywin32==306.0.0.0
 types-PyYAML==6.0.12
 types-chevron==0.14.2
-types-psutil==5.9.5.1
+types-psutil==5.9.5.12
 types-setuptools==65.4.0.0
 
 # Test requirements

requirements/reproducible-linux.txt

@@ -19,9 +19,9 @@ aws-lambda-builders==1.30.0 \
     --hash=sha256:861d3e6099590562fcb9f6fc5d014a26f5b5fc8aebb1d18529cd4179514f2a48 \
     --hash=sha256:ab22ffe47995bbbaff531af453746f3365fe01947c1ea8d98a7ae3ab52b46426
     # via aws-sam-cli (setup.py)
-aws-sam-translator==1.64.0 \
-    --hash=sha256:0cc5b07dd6ef1de3525d887a3b9557168e04cb44327706a43661653bad30687f \
-    --hash=sha256:c44725f12b05d4881e3bc077f70e23ebce56ea78c729acf0ca9f51302b27d304
+aws-sam-translator==1.65.0 \
+    --hash=sha256:34dcd1f86bfc3ae11f3defe6f67b74fe863dacf0a3b04156c41dc92600e8dc8d \
+    --hash=sha256:49cf0583a1bc17c686417c11132c5122b40ecbcb9980e0c9e58b65914dd76ebc
     # via
     #   aws-sam-cli (setup.py)
     #   cfn-lint

samcli/__init__.py

@@ -2,4 +2,4 @@
 SAM CLI version
 """
 
-__version__ = "1.81.0"
+__version__ = "1.82.0"

samcli/hook_packages/terraform/designs/resource_linking_generalized.md

@@ -0,0 +1,163 @@
+# Resource Linking
+
+After translating a resource from Terraform to CloudFormation, 
+that resource often needs to be linked to another based on a field.
+
+For example, AWS Lambda Functions are linked to AWS Lambda Layers through the 
+`Layers` property of a Lambda Function.
+
+E.g.
+```yaml
+Resources:
+  AwsLambdaFunction:
+    Type: AWS::Lambda::Function
+    Properties:
+      FunctionName: s3_lambda_function
+      Code: function.zip
+      Handler: app.lambda_handler
+      PackageType: Zip
+      Runtime: python3.8
+      Layers:
+      - Ref: AwsLambdaLayerVersion
+  AwsLambdaLayerVersion:
+    Type: AWS::Lambda::LayerVersion
+    Properties:
+      LayerName: lambda_layer1
+      CompatibleRuntimes:
+      - python3.8
+      Content: layer.zip
+```
+
+In this case, the function will be defined as the `source` resource and the layer
+as the `destination` resource. The `Layers` field links to the destination resource 
+using a CloudFormation intrinsic function `!Ref`, and points to the Lambda Layer Arn.
+
+Today, the linking logic is specific to Functions and Layers. The section below proposes
+a method of generalization to make the linking of two resources more extensible and easier to implement.
+
+## Proposed Solution
+Currently, the linking design is functional and passes certain required parameters from 
+the translation function to a specific Function to Layer linking function. Since the fields required to link any two resources primarily the same, they can be abstracted away to a data class
+that will house them.
+
+```python
+@dataclass
+class ResourceLinkingPair:
+    source_resource_cfn_resource: Dict[str, List]
+    source_resource_tf_config: Dict[str, TFResource]
+    destination_resource_tf: Dict[str, Dict]
+    intrinsic_type: LinkerIntrinsics
+    cfn_intrinsic_attribute: Optional[str]
+    source_link_field_name: str
+    terraform_resource_type_prefix: str
+    linking_exceptions: ResourcePairLinkingExceptions
+```
+
+- `source_resource_cfn_resource` this the CFN representation of the resource to which the linked resource will be added
+- `source_resource_tf_config` the Terraform configuration object for the source resource
+- `destination_resource_tf` the destination resources' Terraform planned values (not the configuration object)
+- `intrinsic_type` the CFN intrinsic on which to link (`Ref` or `GetAtt`)
+- `cfn_intrinsic_attribute` in the case of `GetAtt`, the resource attribute to link to (should be None for `Ref`)
+- `source_link_field_name` the name in the source CFN resource to add the linked resource to
+- `terraform_resource_type_prefix` the Terraform resource type prefix used for finding all resource of that type
+- `linking_exceptions` these are exceptions that need to be created with messages specific to the resource linking pair to be used
+by the `ResourceLinker`. With specific exception types, metric data can be collected specific to the resources being linked.
+
+```python
+class ResourcePairLinkingExceptions:
+    multiple_resource_linking_exception: Type[UserException]
+    local_variable_linking_exception: Type[UserException]
+```
+
+
+Only the first three fields are computed, and need to be collected the same way they are today, when parsing the Terraform modules.
+E.g. collecting the Lambda function `source_resource_cfn_resource` and `source_resource_tf_config` looks like this:
+```python
+if resource_type == TF_AWS_LAMBDA_FUNCTION:
+    resolved_config_address = _get_configuration_address(resource_full_address)
+    matched_lambdas = source_resource_cfn_resource.get(resolved_config_address, [])
+    matched_lambdas.append(translated_resource)
+    source_resource_cfn_resource[resolved_config_address] = matched_lambdas
+    source_resource_tf_config[resolved_config_address] = config_resource
+```
+
+and on the destination Layer resource, collecting the `destination_resource_tf`
+```python
+if resource_type == TF_AWS_LAMBDA_LAYER_VERSION:
+    destination_resource_tf[logical_id] = resource
+```
+
+After collecting those three fields, a `ResourceLinkingPair` object can be instantiated.
+
+Once all `ResourceLinkingPair` objects have been created, a list of these objects can be passed to the `ResourceLinker`.
+```python
+class ResourceLinker:
+    _resource_pairs: List[ResourceLinkingPair]
+
+    def link_resources(self):
+        """
+        Iterate through all of the ResourceLinkingPair items and link the
+        corresponding source resource to destination resource
+        """
+
+    def _update_mapped_parent_resource_with_resolved_child_resources(self, destination_resources: List):
+        """
+        Set the resolved destination resource list to the mapped source resources.
+
+        Parameters
+        ----------
+        destination_resources: List
+            The resolved destination resource values that will be used as a value for the mapped CFN resource attribute.
+        """
+
+    def _process_reference_resource_value(self, resolved_destination_resource: ResolvedReference):
+        """
+        Process the a reference destination resource value of type ResolvedReference.
+
+        Parameters
+        ----------
+        resolved_destination_resource: ResolvedReference
+            The resolved destination resource reference.
+
+        Returns
+        -------
+        List[Dict[str, str]]
+            The resolved values that will be used as a value for the mapped CFN resource attribute.
+        """
+
+    def _process_resolved_resources(self, resolved_destination_resource: List[Union[ConstantValue, ResolvedReference]]):
+        """
+        Process the resolved destination resources.
+
+        Parameters
+        ----------
+        resolved_destination_resource: List[Union[ConstantValue, ResolvedReference]]
+            The resolved destination resources to be processed for the input source resource.
+
+        Returns
+        --------
+        List[Dict[str, str]]:
+            The list of destination resources after processing
+        """
+```
+
+The `ResourceLinker` contains a public `link_resources` method to begin execution of the resource linking. The methods 
+shown in the class are all generalized versions of the existing Function to Layer linking functions that exist today.
+
+`link_resources` ← `_link_lambda_function_to_layer`, `_link_lambda_functions_to_layers`
+`_update_mapped_parent_resource_with_resolved_child_resources` ← `_update_mapped_lambda_function_with_resolved_layers`
+`_process_reference_resource_value` ← `_process_reference_layer_value`
+`_process_resolved_resources` ← `_process_resolved_layers` 
+
+To generalize these functions: 
+- All instances of hard-coded properties need to be replaced with the corresponding values
+defined in the `ResourceLinkingPair` instance
+- Exception messages should be updated to be more generic
+- The `_update_mapped_parent_resource_with_resolved_child_resources` method should be updated to support writing other intrinsic types.
+
+
+### Adding New Resource Links
+With the proposed change, creating a new link between two resources will include:
+1. Collecting the three required fields (`source_resource_cfn_resource`, `source_resource_tf_config` and `destination_resource_tf`)
+2. Creating a `ResourceLinkingPair` instance
+3. Appending the `ResourceLinkingPair` to the list of pairs

samcli/hook_packages/terraform/hooks/prepare/property_builder.py

@@ -12,6 +12,7 @@
 )
 from samcli.lib.hook.exceptions import PrepareHookException
 from samcli.lib.utils.packagetype import IMAGE, ZIP
+from samcli.lib.utils.resources import AWS_APIGATEWAY_METHOD as CFN_AWS_APIGATEWAY_METHOD
 from samcli.lib.utils.resources import AWS_APIGATEWAY_RESOURCE as CFN_AWS_APIGATEWAY_RESOURCE
 from samcli.lib.utils.resources import AWS_APIGATEWAY_RESTAPI as CFN_AWS_APIGATEWAY_RESTAPI
 from samcli.lib.utils.resources import AWS_APIGATEWAY_STAGE as CFN_AWS_APIGATEWAY_STAGE
@@ -25,6 +26,7 @@
 TF_AWS_API_GATEWAY_RESOURCE = "aws_api_gateway_resource"
 TF_AWS_API_GATEWAY_REST_API = "aws_api_gateway_rest_api"
 TF_AWS_API_GATEWAY_STAGE = "aws_api_gateway_stage"
+TF_AWS_API_GATEWAY_METHOD = "aws_api_gateway_method"
 
 
 def _build_code_property(tf_properties: dict, resource: TFResource) -> Any:
@@ -241,6 +243,13 @@ def _check_image_config_value(image_config: Any) -> bool:
     "PathPart": _get_property_extractor("path_part"),
 }
 
+AWS_API_GATEWAY_METHOD_PROPERTY_BUILDER_MAPPING: PropertyBuilderMapping = {
+    "RestApiId": _get_property_extractor("rest_api_id"),
+    "ResourceId": _get_property_extractor("resource_id"),
+    "HttpMethod": _get_property_extractor("http_method"),
+    "OperationName": _get_property_extractor("operation_name"),
+}
+
 RESOURCE_TRANSLATOR_MAPPING: Dict[str, ResourceTranslator] = {
     TF_AWS_LAMBDA_FUNCTION: ResourceTranslator(CFN_AWS_LAMBDA_FUNCTION, AWS_LAMBDA_FUNCTION_PROPERTY_BUILDER_MAPPING),
     TF_AWS_LAMBDA_LAYER_VERSION: ResourceTranslator(
@@ -255,4 +264,7 @@ def _check_image_config_value(image_config: Any) -> bool:
     TF_AWS_API_GATEWAY_RESOURCE: ResourceTranslator(
         CFN_AWS_APIGATEWAY_RESOURCE, AWS_API_GATEWAY_RESOURCE_PROPERTY_BUILDER_MAPPING
     ),
+    TF_AWS_API_GATEWAY_METHOD: ResourceTranslator(
+        CFN_AWS_APIGATEWAY_METHOD, AWS_API_GATEWAY_METHOD_PROPERTY_BUILDER_MAPPING
+    ),
 }

samcli/hook_packages/terraform/hooks/prepare/resource_linking.py

@@ -4,8 +4,11 @@
 """
 import logging
 import re
-from typing import Dict, List, Optional, Union
+from dataclasses import dataclass
+from enum import Enum
+from typing import Dict, List, Optional, Type, Union
 
+from samcli.commands.exceptions import UserException
 from samcli.hook_packages.terraform.hooks.prepare.exceptions import (
     InvalidResourceLinkingException,
     LocalVariablesLinkingLimitationException,
@@ -29,6 +32,78 @@
 COMPILED_REGULAR_EXPRESSION = re.compile(r"\[[^\[\]]*\]")
 
 
+class LinkerIntrinsics(Enum):
+    Ref = "Ref"
+    GetAtt = "GetAtt"
+
+
+class ResourcePairLinkingExceptions:
+    multiple_resource_linking_exception: Type[UserException]
+    local_variable_linking_exception: Type[UserException]
+
+
+@dataclass
+class ResourceLinkingPair:
+    source_resource_cfn_resource: Dict[str, List]
+    source_resource_tf_config: Dict[str, TFResource]
+    destination_resource_tf: Dict[str, Dict]
+    intrinsic_type: LinkerIntrinsics
+    cfn_intrinsic_attribute: Optional[str]
+    source_link_field_name: str
+    terraform_resource_type_prefix: str
+    linking_exceptions: ResourcePairLinkingExceptions
+
+
+class ResourceLinker:
+    _resource_pairs: List[ResourceLinkingPair]
+
+    def link_resources(self):
+        """
+        Iterate through all of the ResourceLinkingPair items and link the
+        corresponding source resource to destination resource
+        """
+
+    def _update_mapped_parent_resource_with_resolved_child_resources(self, destination_resources: List):
+        """
+        Set the resolved destination resource list to the mapped source resources.
+
+        Parameters
+        ----------
+        destination_resources: List
+            The resolved destination resource values that will be used as a value for the mapped CFN resource attribute.
+        """
+
+    def _process_reference_resource_value(self, resolved_destination_resource: ResolvedReference):
+        """
+        Process the a reference destination resource value of type ResolvedReference.
+
+        Parameters
+        ----------
+        resolved_destination_resource: ResolvedReference
+            The resolved destination resource reference.
+
+        Returns
+        -------
+        List[Dict[str, str]]
+            The resolved values that will be used as a value for the mapped CFN resource attribute.
+        """
+
+    def _process_resolved_resources(self, resolved_destination_resource: List[Union[ConstantValue, ResolvedReference]]):
+        """
+        Process the resolved destination resources.
+
+        Parameters
+        ----------
+        resolved_destination_resource: List[Union[ConstantValue, ResolvedReference]]
+            The resolved destination resources to be processed for the input source resource.
+
+        Returns
+        --------
+        List[Dict[str, str]]:
+            The list of destination resources after processing
+        """
+
+
 def _build_module(
     module_name: Optional[str],
     module_configuration: Dict,

samcli/lib/package/utils.py

@@ -225,7 +225,7 @@ def zip_folder(folder_path, zip_method):
         The md5 hash of the directory
     """
     md5hash = dir_checksum(folder_path, followlinks=True)
-    filename = os.path.join(tempfile.gettempdir(), "data-" + md5hash)
+    filename = os.path.join(tempfile.mkdtemp(), "data-" + md5hash)
 
     zipfile_name = zip_method(filename, folder_path)
     try:

samcli/lib/providers/cfn_api_provider.py

@@ -145,7 +145,7 @@ def _extract_cloud_formation_authorizer(logical_id: str, resource: dict, collect
             validation_string=validation_expression,
         )
 
-        collector.add_authorizers(rest_api_id, {name: lambda_authorizer})
+        collector.add_authorizers(rest_api_id, {logical_id: lambda_authorizer})
 
     @staticmethod
     def _extract_cfn_gateway_v2_authorizer(logical_id: str, resource: dict, collector: ApiCollector) -> None:
@@ -185,7 +185,7 @@ def _extract_cfn_gateway_v2_authorizer(logical_id: str, resource: dict, collecto
             use_simple_response=simple_responses,
         )
 
-        collector.add_authorizers(api_id, {name: lambda_authorizer})
+        collector.add_authorizers(api_id, {logical_id: lambda_authorizer})
 
     @staticmethod
     def _extract_cloud_formation_route(

samcli/lib/utils/preview_runtimes.py

@@ -4,4 +4,4 @@
 """
 from typing import Set
 
-PREVIEW_RUNTIMES: Set[str] = {"java17"}
+PREVIEW_RUNTIMES: Set[str] = set()