diff --git a/aws_lambda_builders/__init__.py b/aws_lambda_builders/__init__.py index 3a56481e9..8f6235d2f 100644 --- a/aws_lambda_builders/__init__.py +++ b/aws_lambda_builders/__init__.py @@ -1,5 +1,5 @@ """ AWS Lambda Builder Library """ -__version__ = "1.17.0" +__version__ = "1.18.0" RPC_PROTOCOL_VERSION = "0.3" diff --git a/aws_lambda_builders/actions.py b/aws_lambda_builders/actions.py index 489e0b722..a7154f66a 100644 --- a/aws_lambda_builders/actions.py +++ b/aws_lambda_builders/actions.py @@ -5,7 +5,7 @@ import logging import os import shutil -import six +from typing import Set, Iterator, Tuple from aws_lambda_builders.utils import copytree @@ -58,7 +58,7 @@ def __new__(mcs, name, bases, class_dict): # Validate class variables # All classes must provide a name - if not isinstance(cls.NAME, six.string_types): + if not isinstance(cls.NAME, str): raise ValueError("Action must provide a valid name") if not Purpose.has_value(cls.PURPOSE): @@ -67,7 +67,7 @@ def __new__(mcs, name, bases, class_dict): return cls -class BaseAction(six.with_metaclass(_ActionMetaClass, object)): +class BaseAction(object, metaclass=_ActionMetaClass): """ Base class for all actions. It does not provide any implementation. """ @@ -125,14 +125,9 @@ def __init__(self, source_dir, artifact_dir, destination_dir): self.dest_dir = destination_dir def execute(self): - source = set(os.listdir(self.source_dir)) - artifact = set(os.listdir(self.artifact_dir)) - dependencies = artifact - source - - for name in dependencies: - dependencies_source = os.path.join(self.artifact_dir, name) - new_destination = os.path.join(self.dest_dir, name) + deps_manager = DependencyManager(self.source_dir, self.artifact_dir, self.dest_dir) + for dependencies_source, new_destination in deps_manager.yield_source_dest(): if os.path.isdir(dependencies_source): copytree(dependencies_source, new_destination) else: @@ -154,14 +149,9 @@ def __init__(self, source_dir, artifact_dir, destination_dir): self.dest_dir = destination_dir def execute(self): - source = set(os.listdir(self.source_dir)) - artifact = set(os.listdir(self.artifact_dir)) - dependencies = artifact - source - - for name in dependencies: - dependencies_source = os.path.join(self.artifact_dir, name) - new_destination = os.path.join(self.dest_dir, name) + deps_manager = DependencyManager(self.source_dir, self.artifact_dir, self.dest_dir) + for dependencies_source, new_destination in deps_manager.yield_source_dest(): # shutil.move can't create subfolders if this is the first file in that folder if os.path.isfile(dependencies_source): os.makedirs(os.path.dirname(new_destination), exist_ok=True) @@ -198,3 +188,36 @@ def execute(self): shutil.rmtree(target_path) else: os.remove(target_path) + + +class DependencyManager: + """ + Class for handling the management of dependencies between directories + """ + + # Ignore these files when comparing against which dependencies to move + # This allows for the installation of dependencies in the source directory + IGNORE_LIST = ["node_modules"] + + def __init__(self, source_dir, artifact_dir, destination_dir) -> None: + self._source_dir: str = source_dir + self._artifact_dir: str = artifact_dir + self._dest_dir: str = destination_dir + self._dependencies: Set[str] = set() + + def yield_source_dest(self) -> Iterator[Tuple[str, str]]: + self._set_dependencies() + for dep in self._dependencies: + yield os.path.join(self._artifact_dir, dep), os.path.join(self._dest_dir, dep) + + def _set_dependencies(self) -> None: + source = self._get_source_files_exclude_deps() + artifact = set(os.listdir(self._artifact_dir)) + self._dependencies = artifact - source + + def _get_source_files_exclude_deps(self) -> Set[str]: + source_files = set(os.listdir(self._source_dir)) + for item in self.IGNORE_LIST: + if item in source_files: + source_files.remove(item) + return source_files diff --git a/aws_lambda_builders/workflow.py b/aws_lambda_builders/workflow.py index d58f0c430..7aef1bd84 100644 --- a/aws_lambda_builders/workflow.py +++ b/aws_lambda_builders/workflow.py @@ -6,7 +6,6 @@ import logging from collections import namedtuple -import six from aws_lambda_builders.binary_path import BinaryPath from aws_lambda_builders.path_resolver import PathResolver @@ -118,7 +117,7 @@ def __new__(mcs, name, bases, class_dict): # Validate class variables # All classes must provide a name - if not isinstance(cls.NAME, six.string_types): + if not isinstance(cls.NAME, str): raise ValueError("Workflow must provide a valid name") # All workflows must express their capabilities @@ -131,7 +130,7 @@ def __new__(mcs, name, bases, class_dict): return cls -class BaseWorkflow(six.with_metaclass(_WorkflowMetaClass, object)): +class BaseWorkflow(object, metaclass=_WorkflowMetaClass): """ Default implementation of the builder workflow. It provides several useful capabilities out-of-box that help minimize the scope of build actions. diff --git a/requirements/base.txt b/requirements/base.txt index 0150babf3..8b1378917 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1 +1 @@ -six~=1.11 + diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index 3a0aff97e..ce6bc4aba 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -1,6 +1,5 @@ import os import shutil -import six import sys import platform import tempfile @@ -146,7 +145,7 @@ def test_mismatch_runtime_python_project(self): "Binary validation failed" not in ex_s and "pip executable not found in your python environment" not in ex_s ): - six.raise_from(AssertionError("Unexpected exception"), ex) + raise AssertionError("Unexpected exception") from ex def test_runtime_validate_python_project_fail_open_unsupported_runtime(self): with self.assertRaises(WorkflowFailedError): diff --git a/tests/unit/test_actions.py b/tests/unit/test_actions.py index 0178c1a38..a2e475344 100644 --- a/tests/unit/test_actions.py +++ b/tests/unit/test_actions.py @@ -1,5 +1,8 @@ +from pathlib import Path +from typing import List, Tuple from unittest import TestCase -from mock import patch, ANY +from mock import patch, ANY, Mock +from parameterized import parameterized from aws_lambda_builders.actions import ( BaseAction, @@ -8,6 +11,7 @@ CopyDependenciesAction, MoveDependenciesAction, CleanUpAction, + DependencyManager, ) @@ -128,3 +132,47 @@ def test_must_copy(self, path_mock, listdir_mock, isdir_mock, rmtree_mock, rm_mo listdir_mock.assert_any_call(target_dir) rmtree_mock.assert_any_call("dir") rm_mock.assert_any_call("file") + + +class TestDependencyManager(TestCase): + @parameterized.expand( + [ + ( + ["app.js", "package.js", "libs", "node_modules"], + ["app.js", "package.js", "libs", "node_modules"], + [("artifacts/node_modules", "dest/node_modules")], + None, + ), + ( + ["file1, file2", "dep1", "dep2"], + ["file1, file2", "dep1", "dep2"], + [("artifacts/dep1", "dest/dep1"), ("artifacts/dep2", "dest/dep2")], + ["dep1", "dep2"], + ), + ( + ["file1, file2"], + ["file1, file2", "dep1", "dep2"], + [("artifacts/dep1", "dest/dep1"), ("artifacts/dep2", "dest/dep2")], + ["dep1", "dep2"], + ), + ] + ) + @patch("aws_lambda_builders.actions.os.listdir") + def test_excludes_dependencies_from_source( + self, source_files, artifact_files, expected, mock_dependencies, patched_list_dir + ): + dependency_manager = DependencyManager("source", "artifacts", "dest") + dependency_manager.IGNORE_LIST = ( + dependency_manager.IGNORE_LIST if mock_dependencies is None else mock_dependencies + ) + patched_list_dir.side_effect = [source_files, artifact_files] + source_destinations = list( + TestDependencyManager._convert_strings_to_paths(list(dependency_manager.yield_source_dest())) + ) + expected_paths = TestDependencyManager._convert_strings_to_paths(expected) + for expected_source_dest in expected_paths: + self.assertIn(expected_source_dest, source_destinations) + + @staticmethod + def _convert_strings_to_paths(source_dest_list): + return map(lambda item: (Path(item[0]), Path(item[1])), source_dest_list)