From f63bce9e9e6f0b58f264c894b5a2e15703bd9aac Mon Sep 17 00:00:00 2001 From: Fawad Rafi Date: Thu, 7 Nov 2024 05:43:14 +0000 Subject: [PATCH 01/24] Added unit tests to workflow --- .github/workflows/ci.yaml | 32 ++++++++++++++++++++++++++++++++ tests/clients_tests.py | 2 +- tests/helpers_tests.py | 7 ++++--- 3 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/ci.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..ec24ea4 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,32 @@ +name: Python Unit Tests with Coverage + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.9' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install coverage pytest + + - name: Run tests with coverage + run: coverage run -m pytest + + - name: Generate coverage report + run: coverage report -m diff --git a/tests/clients_tests.py b/tests/clients_tests.py index 60ca83e..733b312 100644 --- a/tests/clients_tests.py +++ b/tests/clients_tests.py @@ -7,7 +7,7 @@ class TestClients(unittest.TestCase): @patch("cfa_azure.clients.logger") - @patch("cfa_azure.helpers.read_config", MagicMock(return_value=FAKE_CONFIG)) + @patch("cfa_azure.helpers.read_config", MagicMock(return_value=FAKE_CONFIG_MINIMAL)) @patch("cfa_azure.helpers.check_config_req", MagicMock(return_value=True)) @patch("cfa_azure.helpers.get_sp_secret", MagicMock(return_value=True)) @patch("cfa_azure.helpers.get_sp_credential", MagicMock(return_value=True)) diff --git a/tests/helpers_tests.py b/tests/helpers_tests.py index 8572cc3..e26af4f 100644 --- a/tests/helpers_tests.py +++ b/tests/helpers_tests.py @@ -517,7 +517,7 @@ def test_get_pool_parameters(self): container_image_name=FAKE_CONTAINER_IMAGE, container_registry_url=FAKE_CONTAINER_REGISTRY, container_registry_server=FAKE_CONTAINER_REGISTRY, - config=FAKE_CONFIG, + config=FAKE_CONFIG_MINIMAL, mount_config=[], autoscale_formula_path="some_autoscale_formula", timeout=60, @@ -528,13 +528,14 @@ def test_get_pool_parameters(self): ) self.assertIsNotNone(response) + @patch("cfa_azure.helpers.get_deployment_config", MagicMock(return_value={"virtualMachineConfiguration": {}})) def test_get_pool_parameters_use_default(self): response = cfa_azure.helpers.get_pool_parameters( mode="autoscale", container_image_name=FAKE_CONTAINER_IMAGE, container_registry_url=FAKE_CONTAINER_REGISTRY, container_registry_server=FAKE_CONTAINER_REGISTRY, - config=FAKE_CONFIG, + config=FAKE_CONFIG_MINIMAL, mount_config=[], autoscale_formula_path="some_autoscale_formula", timeout=60, @@ -552,7 +553,7 @@ def test_get_pool_parameters_bad_mode(self): container_image_name=FAKE_CONTAINER_IMAGE, container_registry_url=FAKE_CONTAINER_REGISTRY, container_registry_server=FAKE_CONTAINER_REGISTRY, - config=FAKE_CONFIG, + config=FAKE_CONFIG_MINIMAL, mount_config=[], autoscale_formula_path="some_autoscale_formula", timeout=60, From 6bfb2372f07f42bdd23647dcff3a7acbe1e0acfb Mon Sep 17 00:00:00 2001 From: Fawad Rafi Date: Thu, 7 Nov 2024 05:45:51 +0000 Subject: [PATCH 02/24] Added CI unit tests --- .github/workflows/ci.yaml | 4 ++-- cfa_azure/clients.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ec24ea4..91cc19e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2,9 +2,9 @@ name: Python Unit Tests with Coverage on: push: - branches: [ main ] + branches: [ unit_tests ] pull_request: - branches: [ main ] + branches: [ unit_tests ] jobs: build: diff --git a/cfa_azure/clients.py b/cfa_azure/clients.py index 0a42156..053ec0a 100644 --- a/cfa_azure/clients.py +++ b/cfa_azure/clients.py @@ -381,6 +381,7 @@ def update_containers( else: logger.info(f"Pool {pool_name} does not exist. New pool will be created.") + container_image_name=self.container_image_name if not 'pool_id' in self.config['Batch']: self.config['Batch']['pool_id'] = pool_name @@ -474,6 +475,7 @@ def update_container_set( else: logger.info(f"Pool {pool_name} does not exist. New pool will be created.") + container_image_name=self.container_image_name if not 'pool_id' in self.config['Batch']: self.config['Batch']['pool_id'] = pool_name From d8e39984ecbaf497ff11990a01f30126b98679b8 Mon Sep 17 00:00:00 2001 From: Fawad Rafi Date: Thu, 7 Nov 2024 06:35:13 +0000 Subject: [PATCH 03/24] Updated unit tests --- tests/clients_tests.py | 2 ++ tests/fake_client.py | 20 +++++++++++++++++++- tests/helpers_tests.py | 1 + 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/clients_tests.py b/tests/clients_tests.py index 733b312..0ec1142 100644 --- a/tests/clients_tests.py +++ b/tests/clients_tests.py @@ -22,6 +22,7 @@ def setUp(self, mock_logger): mock_logger.info.assert_called_with("Client initialized! Happy coding!") @patch("cfa_azure.clients.logger") + @patch("cfa_azure.helpers.get_deployment_config", MagicMock(return_value={"virtualMachineConfiguration": {}})) def test_set_pool_info(self, mock_logger): self.azure_client.set_pool_info( mode="fixed", @@ -116,6 +117,7 @@ def test_set_debugging_disable(self, mock_logger): self.assertFalse(self.azure_client.debug) mock_logger.debug.assert_called_with("Debugging turned off.") + @patch("cfa_azure.helpers.get_deployment_config", MagicMock(return_value={"virtualMachineConfiguration": {}})) def test_create_pool(self): self.azure_client.set_pool_info(mode="autoscale") pool_details = self.azure_client.create_pool(FAKE_BATCH_POOL) diff --git a/tests/fake_client.py b/tests/fake_client.py index b91b2fd..e29b745 100644 --- a/tests/fake_client.py +++ b/tests/fake_client.py @@ -136,7 +136,7 @@ class FakeTask: def state(self): return batchmodels.TaskState.completed - def add(self, job_id, task, exit_conditions:dict): + def add(self, job_id, task): return True def as_dict(self): @@ -196,12 +196,26 @@ def get_secret(self, vault_sp_secret_id=None): class FakePool: class FakePoolInfo: + class FakeScaleSettings: + @property + def auto_scale(self): + return "fixed" + class FakeDeploymentConfig: class VMConfiguration: class ContainerConfig: + class FakeContainerRegistry: + @property + def registry_server(self): + return "registry_server" + @property def container_image_names(self): return [FAKE_CONTAINER_IMAGE] + + @property + def container_registries(self): + return [self.FakeContainerRegistry()] @property def container_configuration(self): @@ -233,6 +247,10 @@ def last_modified(self): def vm_size(self): return FAKE_POOL_SIZE + @property + def scale_settings(self): + return self.FakeScaleSettings() + def get(self): return True diff --git a/tests/helpers_tests.py b/tests/helpers_tests.py index e26af4f..baf48fd 100644 --- a/tests/helpers_tests.py +++ b/tests/helpers_tests.py @@ -511,6 +511,7 @@ def test_check_azure_container_exists_missing_tag(self): self.assertIsNone(response) @patch("cfa_azure.helpers.get_autoscale_formula", MagicMock(return_value=FAKE_AUTOSCALE_FORMULA)) + @patch("cfa_azure.helpers.get_deployment_config", MagicMock(return_value={"virtualMachineConfiguration": {}})) def test_get_pool_parameters(self): response = cfa_azure.helpers.get_pool_parameters( mode="autoscale", From 4efbc13e29fd1437fb8d2f82385c92c4d1f30b54 Mon Sep 17 00:00:00 2001 From: xop5 <153699395+xop5@users.noreply.github.com> Date: Tue, 12 Nov 2024 10:32:12 -0500 Subject: [PATCH 04/24] Update .github/workflows/ci.yaml Co-authored-by: Nate McIntosh --- .github/workflows/ci.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 91cc19e..3260de7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,11 +19,11 @@ jobs: with: python-version: '3.9' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install coverage pytest + - name: install poetry + run: pip install poetry + + - name: install package + run: poetry install --with test - name: Run tests with coverage run: coverage run -m pytest From 3044ad1fe0b1853dcdad5e3c812625f44c2eb2dc Mon Sep 17 00:00:00 2001 From: xop5 <153699395+xop5@users.noreply.github.com> Date: Tue, 12 Nov 2024 10:32:23 -0500 Subject: [PATCH 05/24] Update .github/workflows/ci.yaml Co-authored-by: Nate McIntosh --- .github/workflows/ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3260de7..dd1f400 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -14,10 +14,10 @@ jobs: - name: Checkout code uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: ${{ matrix.python-version }} - name: install poetry run: pip install poetry From b5ab49b44d015a32194bc0a51b107e98ffd10c69 Mon Sep 17 00:00:00 2001 From: xop5 <153699395+xop5@users.noreply.github.com> Date: Tue, 12 Nov 2024 10:32:34 -0500 Subject: [PATCH 06/24] Update .github/workflows/ci.yaml Co-authored-by: Nate McIntosh --- .github/workflows/ci.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index dd1f400..e55e60b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,6 +9,9 @@ on: jobs: build: runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - name: Checkout code From c3754ee5dcb8de3a42e41129bf6f46f58cc13cf4 Mon Sep 17 00:00:00 2001 From: xop5 <153699395+xop5@users.noreply.github.com> Date: Tue, 12 Nov 2024 10:32:41 -0500 Subject: [PATCH 07/24] Update .github/workflows/ci.yaml Co-authored-by: Nate McIntosh --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e55e60b..1929f8e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -15,7 +15,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 From a86cf9e0e537f4535b600239dc1639b1c1c02510 Mon Sep 17 00:00:00 2001 From: xop5 <153699395+xop5@users.noreply.github.com> Date: Tue, 12 Nov 2024 11:32:08 -0500 Subject: [PATCH 08/24] Update .github/workflows/ci.yaml Co-authored-by: Nate McIntosh --- .github/workflows/ci.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1929f8e..18c760a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2,9 +2,8 @@ name: Python Unit Tests with Coverage on: push: - branches: [ unit_tests ] + branches: [ master ] pull_request: - branches: [ unit_tests ] jobs: build: From ffe73e18693e76d857bdbd1f2850b8a08628fac4 Mon Sep 17 00:00:00 2001 From: Fawad Rafi Date: Tue, 12 Nov 2024 18:59:35 +0000 Subject: [PATCH 09/24] Updated CI tests --- .github/workflows/ci.yaml | 8 ++++---- pyproject.toml | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 18c760a..e5ffc92 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -21,11 +21,11 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: install poetry - run: pip install poetry + - name: Install test libraries + run: pip install -r tests/requirements.test.txt - - name: install package - run: poetry install --with test + - name: Install poetry package packages + run: poetry install - name: Run tests with coverage run: coverage run -m pytest diff --git a/pyproject.toml b/pyproject.toml index daa6253..8b76031 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,3 +24,7 @@ pyyaml = "*" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["*_tests.py"] \ No newline at end of file From 4c9504aa5be66556eaa81dbb7666b63557b4469d Mon Sep 17 00:00:00 2001 From: Fawad Rafi Date: Tue, 12 Nov 2024 19:12:24 +0000 Subject: [PATCH 10/24] Added requirements file --- tests/requirements.test.txt | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 tests/requirements.test.txt diff --git a/tests/requirements.test.txt b/tests/requirements.test.txt new file mode 100644 index 0000000..6ba0591 --- /dev/null +++ b/tests/requirements.test.txt @@ -0,0 +1,6 @@ +coverage +numpy +pandas +poetry +pytest +pytest-xdist \ No newline at end of file From 0785c2b8e1d35fedc36b27f052e5f54b1e1a9539 Mon Sep 17 00:00:00 2001 From: Fawad Rafi Date: Tue, 12 Nov 2024 19:18:54 +0000 Subject: [PATCH 11/24] Updated requirements --- tests/requirements.test.txt | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/requirements.test.txt b/tests/requirements.test.txt index 6ba0591..f09b052 100644 --- a/tests/requirements.test.txt +++ b/tests/requirements.test.txt @@ -1,6 +1,17 @@ +azure-identity==1.16.1 +azure-keyvault==4.2.0 +azure-batch==14.0.0 +azure-mgmt-batch==17.1.0 +azure-storage-blob==12.17.0 +azure-containerregistry==1.2.0 coverage +cryptography>=43.0.1 +docker numpy pandas +pathlib poetry pytest -pytest-xdist \ No newline at end of file +pytest-xdist +pyyaml +toml==0.10.2 \ No newline at end of file From 34d6ca796545ed903ccfb7beeeb307434b253518 Mon Sep 17 00:00:00 2001 From: Fawad Rafi Date: Tue, 12 Nov 2024 19:22:03 +0000 Subject: [PATCH 12/24] Updated callee requirements --- tests/requirements.test.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/requirements.test.txt b/tests/requirements.test.txt index f09b052..e1552b3 100644 --- a/tests/requirements.test.txt +++ b/tests/requirements.test.txt @@ -4,6 +4,7 @@ azure-batch==14.0.0 azure-mgmt-batch==17.1.0 azure-storage-blob==12.17.0 azure-containerregistry==1.2.0 +callee coverage cryptography>=43.0.1 docker From 60e1c870af996703523d65732925716600b4653e Mon Sep 17 00:00:00 2001 From: Fawad Rafi Date: Tue, 12 Nov 2024 19:38:46 +0000 Subject: [PATCH 13/24] Remove pre commit --- .github/workflows/pre-commit.yaml | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 .github/workflows/pre-commit.yaml diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml deleted file mode 100644 index ea064a2..0000000 --- a/.github/workflows/pre-commit.yaml +++ /dev/null @@ -1,14 +0,0 @@ -name: pre-commit - -on: - pull_request: - push: - branches: [main] - -jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 - - uses: ./.github/actions/pre-commit From e8b7ecc4c98783e9a501963973594f2330b9325c Mon Sep 17 00:00:00 2001 From: Fawad Rafi Date: Mon, 18 Nov 2024 01:06:11 +0000 Subject: [PATCH 14/24] Updated unit tests --- .gitignore | 2 +- tests/clients_tests.py | 16 ++++++++-------- tests/fake_client.py | 8 +++++++- tests/requirements.test.txt | 2 +- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 5966da2..85d7c57 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,4 @@ acr* venv* req* yaml* -.coverage +.coverage \ No newline at end of file diff --git a/tests/clients_tests.py b/tests/clients_tests.py index 0ec1142..63420f6 100644 --- a/tests/clients_tests.py +++ b/tests/clients_tests.py @@ -72,7 +72,7 @@ def test_add_task_inputfiles(self): self.assertIsNotNone(task_list) @patch("cfa_azure.helpers.check_azure_container_exists", MagicMock(return_value=FAKE_CONTAINER_IMAGE)) - @patch("cfa_azure.helpers.get_pool_full_info", MagicMock(return_value=dict2obj(FAKE_POOL_INFO))) + @patch("cfa_azure.helpers.get_pool_full_info", MagicMock(return_value=FakeClient.FakePool.FakePoolInfo())) def test_add_task_dependencies(self): self.azure_client.pool_name = FAKE_BATCH_POOL task_1 = self.azure_client.add_task( @@ -289,7 +289,7 @@ def test_check_job_status_noexist(self, mock_logger): @patch("cfa_azure.clients.logger") @patch("cfa_azure.helpers.check_pool_exists", MagicMock(return_value=True)) @patch("cfa_azure.helpers.get_batch_service_client", MagicMock(return_value=FakeClient())) - @patch("cfa_azure.helpers.delete_pool", MagicMock(return_value=True)) + @patch("cfa_azure.helpers.delete_pool", MagicMock(return_value=FakeClient())) @patch("cfa_azure.helpers.create_batch_pool", MagicMock(return_value=FAKE_BATCH_POOL)) def test_update_container_set(self, mock_logger): containers = [ @@ -306,9 +306,10 @@ def test_update_container_set(self, mock_logger): @patch("cfa_azure.helpers.check_pool_exists", MagicMock(return_value=True)) @patch("cfa_azure.helpers.get_batch_service_client", MagicMock(return_value=FakeClient())) - @patch("cfa_azure.helpers.delete_pool", MagicMock(return_value=True)) + @patch("cfa_azure.helpers.delete_pool", MagicMock(return_value=FakeClient())) @patch("cfa_azure.helpers.format_rel_path", MagicMock(return_value="/some_path")) @patch("cfa_azure.helpers.create_batch_pool", MagicMock(return_value=FAKE_BATCH_POOL)) + @patch("cfa_azure.helpers.get_sp_secret", MagicMock(return_value=True)) def test_update_container_set_forced(self): self.azure_client.blob_service_client = FakeClient() containers = [ @@ -325,7 +326,7 @@ def test_update_container_set_forced(self): @patch("cfa_azure.helpers.check_pool_exists", MagicMock(return_value=False)) @patch("cfa_azure.helpers.get_batch_service_client", MagicMock(return_value=FakeClient())) - @patch("cfa_azure.helpers.delete_pool", MagicMock(return_value=True)) + @patch("cfa_azure.helpers.delete_pool", MagicMock(return_value=FakeClient())) @patch("cfa_azure.helpers.create_batch_pool", MagicMock(return_value=FAKE_BATCH_POOL)) def test_update_containers_new_pool(self): containers = [ @@ -343,7 +344,7 @@ def test_update_containers_new_pool(self): @patch("cfa_azure.clients.logger") @patch("cfa_azure.helpers.check_pool_exists", MagicMock(return_value=True)) @patch("cfa_azure.helpers.get_batch_service_client", MagicMock(return_value=FakeClient())) - @patch("cfa_azure.helpers.delete_pool", MagicMock(return_value=True)) + @patch("cfa_azure.helpers.delete_pool", MagicMock(return_value=FakeClient())) @patch("cfa_azure.helpers.create_batch_pool", MagicMock(return_value=FAKE_BATCH_POOL)) def test_update_containers(self, mock_logger): pool_name = self.azure_client.update_containers( @@ -354,12 +355,11 @@ def test_update_containers(self, mock_logger): mock_logger.error.assert_called_with(f"There are 2 compute nodes actively running tasks in pool {FAKE_BATCH_POOL}. Please wait for jobs to complete or retry withy force_update=True.") self.assertIsNone(pool_name) - @patch("cfa_azure.clients.logger") - @patch("cfa_azure.clients.AzureClient.get_container_image_name", MagicMock(return_value=FAKE_CONTAINER_IMAGE)) @patch("cfa_azure.helpers.check_pool_exists", MagicMock(return_value=True)) @patch("cfa_azure.helpers.get_batch_service_client", MagicMock(return_value=FakeClient())) @patch("cfa_azure.helpers.delete_pool", MagicMock(return_value=FakeClient())) @patch("cfa_azure.helpers.create_batch_pool", MagicMock(return_value=FAKE_BATCH_POOL)) + @patch("cfa_azure.helpers.get_sp_secret", MagicMock(return_value=True)) def test_update_containers_forced(self): pool_name = self.azure_client.update_containers( pool_name=FAKE_BATCH_POOL, @@ -371,7 +371,7 @@ def test_update_containers_forced(self): @patch("cfa_azure.helpers.check_pool_exists", MagicMock(return_value=False)) @patch("cfa_azure.helpers.get_batch_service_client", MagicMock(return_value=FakeClient())) - @patch("cfa_azure.helpers.delete_pool", MagicMock(return_value=True)) + @patch("cfa_azure.helpers.delete_pool", MagicMock(return_value=FakeClient())) @patch("cfa_azure.helpers.create_batch_pool", MagicMock(return_value=FAKE_BATCH_POOL)) def test_update_containers_new_pool(self): pool_name = self.azure_client.update_containers( diff --git a/tests/fake_client.py b/tests/fake_client.py index e29b745..c6c0d75 100644 --- a/tests/fake_client.py +++ b/tests/fake_client.py @@ -103,7 +103,6 @@ } class FakeClient: - class FakeBatchJob: def delete(self, *args): return True @@ -200,6 +199,9 @@ class FakeScaleSettings: @property def auto_scale(self): return "fixed" + + def as_dict(self): + return FAKE_POOL_INFO class FakeDeploymentConfig: class VMConfiguration: @@ -209,6 +211,10 @@ class FakeContainerRegistry: def registry_server(self): return "registry_server" + @property + def user_name(self): + return "user_name" + @property def container_image_names(self): return [FAKE_CONTAINER_IMAGE] diff --git a/tests/requirements.test.txt b/tests/requirements.test.txt index e1552b3..8ca2299 100644 --- a/tests/requirements.test.txt +++ b/tests/requirements.test.txt @@ -8,7 +8,7 @@ callee coverage cryptography>=43.0.1 docker -numpy +numpy>=2.1.1 pandas pathlib poetry From 45cea7034ebdcacfa007a0224a595817de4723c6 Mon Sep 17 00:00:00 2001 From: Fawad Rafi Date: Mon, 18 Nov 2024 01:11:42 +0000 Subject: [PATCH 15/24] Run tests for unit_tests branch --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e5ffc92..6ae60a4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2,7 +2,7 @@ name: Python Unit Tests with Coverage on: push: - branches: [ master ] + branches: [ master, unit_tests ] pull_request: jobs: From c6a8693426007249e31b0700b30fe6b8aa5f8184 Mon Sep 17 00:00:00 2001 From: Fawad Rafi Date: Mon, 18 Nov 2024 01:29:12 +0000 Subject: [PATCH 16/24] Fixed failing unit tests --- cfa_azure/helpers.py | 2 +- tests/helpers_tests.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cfa_azure/helpers.py b/cfa_azure/helpers.py index 79aa70b..d20c46c 100644 --- a/cfa_azure/helpers.py +++ b/cfa_azure/helpers.py @@ -1889,8 +1889,8 @@ def check_config_req(config: str): return True else: logger.warning( + "%s missing from the config file and will be required by client.", str(list(req - loaded)), - "missing from the config file and will be required by client.", ) return False diff --git a/tests/helpers_tests.py b/tests/helpers_tests.py index baf48fd..42235c5 100644 --- a/tests/helpers_tests.py +++ b/tests/helpers_tests.py @@ -318,7 +318,7 @@ def test_get_completed_tasks(self): self.assertEqual(task_summary['completed tasks'], 1) def test_check_config_req(self): - status = cfa_azure.helpers.check_config_req(FAKE_CONFIG) + status = cfa_azure.helpers.check_config_req(FAKE_CONFIG_MINIMAL) self.assertIsNotNone(status) def test_check_config_req_badconfig(self): From 6e6e7e03e735ffc6cf721ccd4a9bd10fb1947133 Mon Sep 17 00:00:00 2001 From: Fawad Rafi Date: Mon, 18 Nov 2024 01:35:11 +0000 Subject: [PATCH 17/24] Avoid download tests --- tests/helpers_tests.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tests/helpers_tests.py b/tests/helpers_tests.py index 42235c5..a08e7fe 100644 --- a/tests/helpers_tests.py +++ b/tests/helpers_tests.py @@ -416,19 +416,6 @@ def test_download_directory_extensions(self, mock_logger): ) mock_logger.debug.assert_called_with("Download complete.") - @patch("cfa_azure.helpers.logger") - @patch("cfa_azure.helpers.check_blob_existence", MagicMock(return_value=True)) - def test_download_file(self, mock_logger): - blob_service_client = FakeClient() - cfa_azure.helpers.download_file( - c_client=blob_service_client, - src_path="some_path/", - dest_path="/another_path", - do_check= False, - verbose = False - ) - mock_logger.debug.assert_called_with("File downloaded.") - @patch("cfa_azure.helpers.logger") @patch("cfa_azure.helpers.download_file", MagicMock(return_value=True)) def test_download_directory_extensions_inclusions(self, mock_logger): From fe7b38a458633e2a250dd01aaba8a6295433c739 Mon Sep 17 00:00:00 2001 From: Fawad Rafi Date: Mon, 18 Nov 2024 01:49:03 +0000 Subject: [PATCH 18/24] Update requirements txt --- tests/requirements.test.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/requirements.test.txt b/tests/requirements.test.txt index 8ca2299..6249f62 100644 --- a/tests/requirements.test.txt +++ b/tests/requirements.test.txt @@ -9,6 +9,7 @@ coverage cryptography>=43.0.1 docker numpy>=2.1.1 +setuptools pandas pathlib poetry From 1426c084460cd12d9efb62d9bef23882e8de2e8d Mon Sep 17 00:00:00 2001 From: Fawad Rafi Date: Mon, 18 Nov 2024 01:53:58 +0000 Subject: [PATCH 19/24] Removed Python 3.13 due to Pandas build issue --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6ae60a4..b2addbc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12"] steps: - name: Checkout code From dcb34f5e67114a1cc4ade9833c1ff6a538ff0236 Mon Sep 17 00:00:00 2001 From: Fawad Rafi Date: Wed, 20 Nov 2024 18:09:33 +0000 Subject: [PATCH 20/24] Removed unit tests branch --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b2addbc..bf2e862 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2,7 +2,7 @@ name: Python Unit Tests with Coverage on: push: - branches: [ master, unit_tests ] + branches: [ master ] pull_request: jobs: From e5fe6f19a06d0a94ab4318af88a4f46573543d11 Mon Sep 17 00:00:00 2001 From: Fawad Rafi Date: Wed, 20 Nov 2024 18:12:10 +0000 Subject: [PATCH 21/24] Run unit tests on all branches --- .github/workflows/ci.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bf2e862..2fde378 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,10 +1,5 @@ name: Python Unit Tests with Coverage -on: - push: - branches: [ master ] - pull_request: - jobs: build: runs-on: ubuntu-latest From 9a241c3c9ef3fe5c3d66293ca9b8ff78fb7bf02c Mon Sep 17 00:00:00 2001 From: Fawad Rafi Date: Wed, 20 Nov 2024 18:17:55 +0000 Subject: [PATCH 22/24] Removed obsolete CI --- .github/workflows/ci.yml | 35 ----------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 1929f8e..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Python Unit Tests with Coverage - -on: - push: - branches: [ unit_tests ] - pull_request: - branches: [ unit_tests ] - -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: install poetry - run: pip install poetry - - - name: install package - run: poetry install --with test - - - name: Run tests with coverage - run: coverage run -m pytest - - - name: Generate coverage report - run: coverage report -m From 2f44e1244cf2536011de586a3a9bc5ff96bdfd6f Mon Sep 17 00:00:00 2001 From: Fawad Rafi Date: Wed, 20 Nov 2024 19:33:19 +0000 Subject: [PATCH 23/24] Added trigger --- .github/workflows/ci.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2fde378..a1aef90 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,5 +1,9 @@ name: Python Unit Tests with Coverage +on: + push: + branches: '**' + jobs: build: runs-on: ubuntu-latest From 77536c9634039e42a212e33bb14ce8db434cda3d Mon Sep 17 00:00:00 2001 From: Fawad Rafi Date: Wed, 20 Nov 2024 19:42:54 +0000 Subject: [PATCH 24/24] Default logs folder --- cfa_azure/clients.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cfa_azure/clients.py b/cfa_azure/clients.py index 4e7aa7f..8726708 100644 --- a/cfa_azure/clients.py +++ b/cfa_azure/clients.py @@ -38,6 +38,7 @@ def __init__(self, config_path: str = None, use_env_vars: bool = False): self.pool_parameters = None self.timeout = None self.save_logs_to_blob = None + self.logs_folder = "stdout_stderr" logger.debug("Attributes initialized in client.")