Merge pull request #371 from aws/develop

chore: merge develop to master

aws_lambda_builders/__init__.py

@@ -1,5 +1,5 @@
 """
 AWS Lambda Builder Library
 """
-__version__ = "1.17.0"
+__version__ = "1.18.0"
 RPC_PROTOCOL_VERSION = "0.3"

aws_lambda_builders/actions.py

@@ -5,7 +5,7 @@
 import logging
 import os
 import shutil
-import six
+from typing import Set, Iterator, Tuple
 
 from aws_lambda_builders.utils import copytree
 
@@ -58,7 +58,7 @@ def __new__(mcs, name, bases, class_dict):
 
         # Validate class variables
         # All classes must provide a name
-        if not isinstance(cls.NAME, six.string_types):
+        if not isinstance(cls.NAME, str):
             raise ValueError("Action must provide a valid name")
 
         if not Purpose.has_value(cls.PURPOSE):
@@ -67,7 +67,7 @@ def __new__(mcs, name, bases, class_dict):
         return cls
 
 
-class BaseAction(six.with_metaclass(_ActionMetaClass, object)):
+class BaseAction(object, metaclass=_ActionMetaClass):
     """
     Base class for all actions. It does not provide any implementation.
     """
@@ -125,14 +125,9 @@ def __init__(self, source_dir, artifact_dir, destination_dir):
         self.dest_dir = destination_dir
 
     def execute(self):
-        source = set(os.listdir(self.source_dir))
-        artifact = set(os.listdir(self.artifact_dir))
-        dependencies = artifact - source
-
-        for name in dependencies:
-            dependencies_source = os.path.join(self.artifact_dir, name)
-            new_destination = os.path.join(self.dest_dir, name)
+        deps_manager = DependencyManager(self.source_dir, self.artifact_dir, self.dest_dir)
 
+        for dependencies_source, new_destination in deps_manager.yield_source_dest():
             if os.path.isdir(dependencies_source):
                 copytree(dependencies_source, new_destination)
             else:
@@ -154,14 +149,9 @@ def __init__(self, source_dir, artifact_dir, destination_dir):
         self.dest_dir = destination_dir
 
     def execute(self):
-        source = set(os.listdir(self.source_dir))
-        artifact = set(os.listdir(self.artifact_dir))
-        dependencies = artifact - source
-
-        for name in dependencies:
-            dependencies_source = os.path.join(self.artifact_dir, name)
-            new_destination = os.path.join(self.dest_dir, name)
+        deps_manager = DependencyManager(self.source_dir, self.artifact_dir, self.dest_dir)
 
+        for dependencies_source, new_destination in deps_manager.yield_source_dest():
             # shutil.move can't create subfolders if this is the first file in that folder
             if os.path.isfile(dependencies_source):
                 os.makedirs(os.path.dirname(new_destination), exist_ok=True)
@@ -198,3 +188,36 @@ def execute(self):
                 shutil.rmtree(target_path)
             else:
                 os.remove(target_path)
+
+
+class DependencyManager:
+    """
+    Class for handling the management of dependencies between directories
+    """
+
+    # Ignore these files when comparing against which dependencies to move
+    # This allows for the installation of dependencies in the source directory
+    IGNORE_LIST = ["node_modules"]
+
+    def __init__(self, source_dir, artifact_dir, destination_dir) -> None:
+        self._source_dir: str = source_dir
+        self._artifact_dir: str = artifact_dir
+        self._dest_dir: str = destination_dir
+        self._dependencies: Set[str] = set()
+
+    def yield_source_dest(self) -> Iterator[Tuple[str, str]]:
+        self._set_dependencies()
+        for dep in self._dependencies:
+            yield os.path.join(self._artifact_dir, dep), os.path.join(self._dest_dir, dep)
+
+    def _set_dependencies(self) -> None:
+        source = self._get_source_files_exclude_deps()
+        artifact = set(os.listdir(self._artifact_dir))
+        self._dependencies = artifact - source
+
+    def _get_source_files_exclude_deps(self) -> Set[str]:
+        source_files = set(os.listdir(self._source_dir))
+        for item in self.IGNORE_LIST:
+            if item in source_files:
+                source_files.remove(item)
+        return source_files

aws_lambda_builders/workflow.py

@@ -6,7 +6,6 @@
 import logging
 
 from collections import namedtuple
-import six
 
 from aws_lambda_builders.binary_path import BinaryPath
 from aws_lambda_builders.path_resolver import PathResolver
@@ -118,7 +117,7 @@ def __new__(mcs, name, bases, class_dict):
         # Validate class variables
 
         # All classes must provide a name
-        if not isinstance(cls.NAME, six.string_types):
+        if not isinstance(cls.NAME, str):
             raise ValueError("Workflow must provide a valid name")
 
         # All workflows must express their capabilities
@@ -131,7 +130,7 @@ def __new__(mcs, name, bases, class_dict):
         return cls
 
 
-class BaseWorkflow(six.with_metaclass(_WorkflowMetaClass, object)):
+class BaseWorkflow(object, metaclass=_WorkflowMetaClass):
     """
     Default implementation of the builder workflow. It provides several useful capabilities out-of-box that help
     minimize the scope of build actions.

requirements/base.txt

@@ -1 +1 @@
-six~=1.11
+

tests/integration/workflows/python_pip/test_python_pip.py

@@ -1,6 +1,5 @@
 import os
 import shutil
-import six
 import sys
 import platform
 import tempfile
@@ -146,7 +145,7 @@ def test_mismatch_runtime_python_project(self):
                 "Binary validation failed" not in ex_s
                 and "pip executable not found in your python environment" not in ex_s
             ):
-                six.raise_from(AssertionError("Unexpected exception"), ex)
+                raise AssertionError("Unexpected exception") from ex
 
     def test_runtime_validate_python_project_fail_open_unsupported_runtime(self):
         with self.assertRaises(WorkflowFailedError):

tests/unit/test_actions.py

@@ -1,5 +1,8 @@
+from pathlib import Path
+from typing import List, Tuple
 from unittest import TestCase
-from mock import patch, ANY
+from mock import patch, ANY, Mock
+from parameterized import parameterized
 
 from aws_lambda_builders.actions import (
     BaseAction,
@@ -8,6 +11,7 @@
     CopyDependenciesAction,
     MoveDependenciesAction,
     CleanUpAction,
+    DependencyManager,
 )
 
 
@@ -128,3 +132,47 @@ def test_must_copy(self, path_mock, listdir_mock, isdir_mock, rmtree_mock, rm_mo
         listdir_mock.assert_any_call(target_dir)
         rmtree_mock.assert_any_call("dir")
         rm_mock.assert_any_call("file")
+
+
+class TestDependencyManager(TestCase):
+    @parameterized.expand(
+        [
+            (
+                ["app.js", "package.js", "libs", "node_modules"],
+                ["app.js", "package.js", "libs", "node_modules"],
+                [("artifacts/node_modules", "dest/node_modules")],
+                None,
+            ),
+            (
+                ["file1, file2", "dep1", "dep2"],
+                ["file1, file2", "dep1", "dep2"],
+                [("artifacts/dep1", "dest/dep1"), ("artifacts/dep2", "dest/dep2")],
+                ["dep1", "dep2"],
+            ),
+            (
+                ["file1, file2"],
+                ["file1, file2", "dep1", "dep2"],
+                [("artifacts/dep1", "dest/dep1"), ("artifacts/dep2", "dest/dep2")],
+                ["dep1", "dep2"],
+            ),
+        ]
+    )
+    @patch("aws_lambda_builders.actions.os.listdir")
+    def test_excludes_dependencies_from_source(
+        self, source_files, artifact_files, expected, mock_dependencies, patched_list_dir
+    ):
+        dependency_manager = DependencyManager("source", "artifacts", "dest")
+        dependency_manager.IGNORE_LIST = (
+            dependency_manager.IGNORE_LIST if mock_dependencies is None else mock_dependencies
+        )
+        patched_list_dir.side_effect = [source_files, artifact_files]
+        source_destinations = list(
+            TestDependencyManager._convert_strings_to_paths(list(dependency_manager.yield_source_dest()))
+        )
+        expected_paths = TestDependencyManager._convert_strings_to_paths(expected)
+        for expected_source_dest in expected_paths:
+            self.assertIn(expected_source_dest, source_destinations)
+
+    @staticmethod
+    def _convert_strings_to_paths(source_dest_list):
+        return map(lambda item: (Path(item[0]), Path(item[1])), source_dest_list)