From c1831d5b00f9658d780d4936f268b3c5d118e711 Mon Sep 17 00:00:00 2001 From: Haimasree Date: Fri, 12 Nov 2021 16:12:50 +0100 Subject: [PATCH 1/4] Adding singularity tests --- .circleci/config.yml | 71 ++++++++++++++++++++++++++++++++++++++++++++ .circleci/setup.sh | 2 +- 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5bc02bb7a..f52deeab2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -33,6 +33,11 @@ variables: run: name: Install hdf5 and pkgconfig command: conda install --yes -c conda-forge hdf5=1.10.6 pkgconfig + install_singularity: &install_singularity + run: + name: Install Singularity + command: | + conda install --yes -c conda-forge singularity restore_cache: &restore_cache restore_cache: keys: @@ -71,6 +76,24 @@ jobs: command: kipoi test-source kipoi --git-range master HEAD --common_env no_output_timeout: 60m - *store_artifacts + + # test only the newly added model with singularity + test_new_models_singularity: + <<: *defaults + steps: + - checkout + - *install_conda + # - *install_sys_deps + - *install_gitlfs + - *install_additional_pkgs + - *install_singularity + # - *restore_cache + - *setup + - run: + name: run tests + command: kipoi test-source kipoi --git-range master HEAD --verbose --singularity + no_output_timeout: 60m + - *store_artifacts # test all models in the repo test_all_models: @@ -95,6 +118,31 @@ jobs: command: kipoi test-source kipoi --all --num_of_shards << parameters.num_of_shards >> --shard_id << parameters.shard_id >> - *store_artifacts + # test all models in the repo + test_all_models_singularity: + parameters: + num_of_shards: + type: integer + shard_id: + type: integer + <<: *defaults + steps: + - checkout + - *install_conda + # - *install_sys_deps + - *install_gitlfs + - *install_additional_pkgs + - *install_singularity + # - *restore_cache + - *setup + - run: + name: run tests + # Use --clean_env to remove the environment of each model + no_output_timeout: 60m + command: kipoi test-source kipoi --all --num_of_shards << parameters.num_of_shards >> --shard_id << parameters.shard_id >> --singularity + - *store_artifacts + + test_all_models_common_env: <<: *defaults steps: @@ -134,6 +182,13 @@ workflows: ignore: - master - test_all + - test_new_models_singularity: + filters: + branches: + ignore: + - master + - test_all + - add-singularity-tests test-all-branch: jobs: - test_all_models: @@ -146,6 +201,17 @@ workflows: only: - master - test_all + - test_all_models_singularity: + matrix: + parameters: + num_of_shards: [4] + shard_id: [0, 1, 2, 3] + filters: + branches: + only: + - master + - test_all + - add-singularity-tests - test_all_models_common_env: filters: branches: @@ -164,6 +230,11 @@ workflows: only: - master jobs: + - test_all_models: + matrix: + parameters: + num_of_shards: [4] + shard_id: [0, 1, 2, 3] - test_all_models: matrix: parameters: diff --git a/.circleci/setup.sh b/.circleci/setup.sh index 37b80b50f..9bba86009 100755 --- a/.circleci/setup.sh +++ b/.circleci/setup.sh @@ -47,7 +47,7 @@ echo "Installing Kipoi." # install kipoi and others from the master branch for now pip install git+https://github.com/kipoi/kipoi-utils.git pip install git+https://github.com/kipoi/kipoi-conda.git -pip install git+https://github.com/kipoi/kipoi.git +pip install git+https://github.com/kipoi/kipoi.git@add-singularity # pip install kipoi # fi From 28748e13bf2201820e4957c7bc558b168193395f Mon Sep 17 00:00:00 2001 From: Haimasree Date: Fri, 12 Nov 2021 16:13:47 +0100 Subject: [PATCH 2/4] minor fix --- .circleci/config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f52deeab2..1786c203b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -188,7 +188,6 @@ workflows: ignore: - master - test_all - - add-singularity-tests test-all-branch: jobs: - test_all_models: @@ -235,7 +234,7 @@ workflows: parameters: num_of_shards: [4] shard_id: [0, 1, 2, 3] - - test_all_models: + - test_all_models_singularity: matrix: parameters: num_of_shards: [4] From d253033836e024c2b76d76a8f9f2c68d3e091900 Mon Sep 17 00:00:00 2001 From: Haimasree Date: Fri, 12 Nov 2021 18:02:22 +0100 Subject: [PATCH 3/4] Fixing deepcpgdna singularity container info --- .circleci/config.yml | 4 ++-- .circleci/setup.sh | 2 +- shared/containers/model-to-singularity.json | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1786c203b..e1dc522eb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -203,8 +203,8 @@ workflows: - test_all_models_singularity: matrix: parameters: - num_of_shards: [4] - shard_id: [0, 1, 2, 3] + num_of_shards: [12] + shard_id: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] #TODO: Switch back to 4 filters: branches: only: diff --git a/.circleci/setup.sh b/.circleci/setup.sh index 9bba86009..37b80b50f 100755 --- a/.circleci/setup.sh +++ b/.circleci/setup.sh @@ -47,7 +47,7 @@ echo "Installing Kipoi." # install kipoi and others from the master branch for now pip install git+https://github.com/kipoi/kipoi-utils.git pip install git+https://github.com/kipoi/kipoi-conda.git -pip install git+https://github.com/kipoi/kipoi.git@add-singularity +pip install git+https://github.com/kipoi/kipoi.git # pip install kipoi # fi diff --git a/shared/containers/model-to-singularity.json b/shared/containers/model-to-singularity.json index 93f662a1c..616ecfa02 100644 --- a/shared/containers/model-to-singularity.json +++ b/shared/containers/model-to-singularity.json @@ -1,8 +1,8 @@ { "DeepCpG_DNA": { - "url": "https://zenodo.org/record/5644007/files/kipoi-docker_sharedpy3keras2.sif?download=1", - "name": "kipoi-docker_sharedpy3keras2", - "md5": "0e5354145f54c2c8c7efc3bb265aebac" + "url": "https://zenodo.org/record/5644005/files/kipoi-docker_sharedpy3keras1.2.sif?download=1", + "name": "kipoi-docker_sharedpy3keras1.2", + "md5": "cd748bae471b6af0b2cdaecca1e1c6ac" }, "CpGenie": { "url": "https://zenodo.org/record/5644005/files/kipoi-docker_sharedpy3keras1.2.sif?download=1", From 0f8a548bba505804f83b198bcf00c501d8f75f9f Mon Sep 17 00:00:00 2001 From: Haimasree Date: Fri, 12 Nov 2021 19:08:17 +0100 Subject: [PATCH 4/4] Update config.yml --- .circleci/config.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e1dc522eb..58db62db3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -203,14 +203,13 @@ workflows: - test_all_models_singularity: matrix: parameters: - num_of_shards: [12] - shard_id: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] #TODO: Switch back to 4 + num_of_shards: [4] + shard_id: [0, 1, 2, 3] filters: branches: only: - master - test_all - - add-singularity-tests - test_all_models_common_env: filters: branches: