From 0a4491d7da2c0c56ca939aec79dacd6cc2c33396 Mon Sep 17 00:00:00 2001 From: Kabilar Date: Sat, 22 Jul 2023 23:19:38 -0500 Subject: [PATCH 01/12] Update Dockerfiles --- docker/cpu.Dockerfile | 4 ++-- docker/gpu.Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/cpu.Dockerfile b/docker/cpu.Dockerfile index 61d5e03c..6b8e02fc 100644 --- a/docker/cpu.Dockerfile +++ b/docker/cpu.Dockerfile @@ -14,8 +14,8 @@ RUN python3 -m pip install --no-cache-dir /opt/nobrainer datalad datalad-osf RUN git config --global user.email "neuronets@example.com" \ && git config --global user.name "Neuronets maintainers" RUN datalad clone https://github.com/neuronets/trained-models /models \ - && cd /models && git-annex enableremote osf-storage \ - && datalad get -r . + && cd /models && git-annex enableremote osf-storage \ + && datalad get -s osf-storage . ENV LC_ALL=C.UTF-8 \ LANG=C.UTF-8 WORKDIR "/work" diff --git a/docker/gpu.Dockerfile b/docker/gpu.Dockerfile index d2af4b36..5c0b0b35 100644 --- a/docker/gpu.Dockerfile +++ b/docker/gpu.Dockerfile @@ -15,7 +15,7 @@ RUN git config --global user.email "neuronets@example.com" \ && git config --global user.name "Neuronets maintainers" RUN datalad clone https://github.com/neuronets/trained-models /models \ && cd /models && git-annex enableremote osf-storage \ - && datalad get -r . + && datalad get -s osf-storage . ENV LC_ALL=C.UTF-8 \ LANG=C.UTF-8 WORKDIR "/work" From 4566cb36ba6ffd65e54abc3ebde7927243125d96 Mon Sep 17 00:00:00 2001 From: Kabilar Date: Sat, 22 Jul 2023 23:50:45 -0500 Subject: [PATCH 02/12] Update changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1e0e2a4..123a0a49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# Upcoming + +#### 🐛 Bug Fix + +- Fix: Update Dockerfiles to download models from the `osf-storage` datalad data source [#252](https://github.com/neuronets/nobrainer/pull/252) ([@kabilar](https://github.com/kabilar)) + # 0.5.0 (Wed Jul 19 2023) #### 🚀 Enhancement From 523df95cbcc5049f1fb1c2a525eed9dee66ab4a8 Mon Sep 17 00:00:00 2001 From: Kabilar Date: Sat, 22 Jul 2023 23:54:34 -0500 Subject: [PATCH 03/12] Remove spacing --- docker/cpu.Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/cpu.Dockerfile b/docker/cpu.Dockerfile index 6b8e02fc..45fb3b15 100644 --- a/docker/cpu.Dockerfile +++ b/docker/cpu.Dockerfile @@ -14,8 +14,8 @@ RUN python3 -m pip install --no-cache-dir /opt/nobrainer datalad datalad-osf RUN git config --global user.email "neuronets@example.com" \ && git config --global user.name "Neuronets maintainers" RUN datalad clone https://github.com/neuronets/trained-models /models \ - && cd /models && git-annex enableremote osf-storage \ - && datalad get -s osf-storage . + && cd /models && git-annex enableremote osf-storage \ + && datalad get -s osf-storage . ENV LC_ALL=C.UTF-8 \ LANG=C.UTF-8 WORKDIR "/work" From aedb6672ee05eb5398c95ab9618cf2eb94141101 Mon Sep 17 00:00:00 2001 From: Kabilar Date: Sun, 23 Jul 2023 17:38:12 -0500 Subject: [PATCH 04/12] Add GitHub Actions workflow --- .github/workflows/ci.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c5397ca..1fafd200 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,3 +29,19 @@ jobs: - name: Test with pytest run: | pytest + image-build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-20.04] + python-version: ["3.10", "3.9", "3.8"] + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - name: Test CPU Docker image build + run: | + docker build -t neuronets/nobrainer:master-cpu -f docker/cpu.Dockerfile . + - name: Test GPU Docker image build + run: | + docker build -t neuronets/nobrainer:master-gpu -f docker/gpu.Dockerfile . From 8787a216d402c6ec51cabdcb432b8c69f6a9d49b Mon Sep 17 00:00:00 2001 From: Kabilar Date: Sun, 23 Jul 2023 17:41:12 -0500 Subject: [PATCH 05/12] Update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 123a0a49..c0750304 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Upcoming +#### 🚀 Enhancement + +- Update: Add Actions for Docker image build [#252](https://github.com/neuronets/nobrainer/pull/252) ([@kabilar](https://github.com/kabilar)) + #### 🐛 Bug Fix - Fix: Update Dockerfiles to download models from the `osf-storage` datalad data source [#252](https://github.com/neuronets/nobrainer/pull/252) ([@kabilar](https://github.com/kabilar)) From 09837988bf6a7fbd40d36b6ac9158450ec84f1d5 Mon Sep 17 00:00:00 2001 From: Kabilar Date: Sun, 23 Jul 2023 18:25:27 -0500 Subject: [PATCH 06/12] Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0750304..8b068c4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ #### 🚀 Enhancement -- Update: Add Actions for Docker image build [#252](https://github.com/neuronets/nobrainer/pull/252) ([@kabilar](https://github.com/kabilar)) +- Update: Add GitHub Actions to test the build of the Docker images [#252](https://github.com/neuronets/nobrainer/pull/252) ([@kabilar](https://github.com/kabilar)) #### 🐛 Bug Fix From 585ad2ac733cb43e8783513c6cc66a6d4afc7f54 Mon Sep 17 00:00:00 2001 From: Kabilar Date: Sun, 23 Jul 2023 18:27:45 -0500 Subject: [PATCH 07/12] Fix syntax --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b068c4c..283dd607 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ #### 🐛 Bug Fix -- Fix: Update Dockerfiles to download models from the `osf-storage` datalad data source [#252](https://github.com/neuronets/nobrainer/pull/252) ([@kabilar](https://github.com/kabilar)) +- Fix: Update Dockerfiles to download models from the `osf-storage` DataLad data source [#252](https://github.com/neuronets/nobrainer/pull/252) ([@kabilar](https://github.com/kabilar)) # 0.5.0 (Wed Jul 19 2023) From d3fda6f7d5cc6cfb9421631d95ef0925360df920 Mon Sep 17 00:00:00 2001 From: Kabilar Date: Thu, 27 Jul 2023 21:55:33 -0500 Subject: [PATCH 08/12] Fix tensorflow installation --- CHANGELOG.md | 1 + docker/cpu.Dockerfile | 2 +- docker/gpu.Dockerfile | 5 ++--- setup.cfg | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 283dd607..7d4551bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ #### 🐛 Bug Fix - Fix: Update Dockerfiles to download models from the `osf-storage` DataLad data source [#252](https://github.com/neuronets/nobrainer/pull/252) ([@kabilar](https://github.com/kabilar)) +- Fix: Update Dockerfiles to fix `tensorflow`` installation [#252](https://github.com/neuronets/nobrainer/pull/252) ([@kabilar](https://github.com/kabilar)) # 0.5.0 (Wed Jul 19 2023) diff --git a/docker/cpu.Dockerfile b/docker/cpu.Dockerfile index 45fb3b15..3a691c9a 100644 --- a/docker/cpu.Dockerfile +++ b/docker/cpu.Dockerfile @@ -1,4 +1,4 @@ -FROM tensorflow/tensorflow:2.10.0-jupyter +FROM tensorflow/tensorflow:2.13.0-jupyter RUN curl -sSL http://neuro.debian.net/lists/focal.us-nh.full | tee /etc/apt/sources.list.d/neurodebian.sources.list \ && export GNUPGHOME="$(mktemp -d)" \ && echo "disable-ipv6" >> ${GNUPGHOME}/dirmngr.conf \ diff --git a/docker/gpu.Dockerfile b/docker/gpu.Dockerfile index 5c0b0b35..0d4e2708 100644 --- a/docker/gpu.Dockerfile +++ b/docker/gpu.Dockerfile @@ -1,4 +1,4 @@ -FROM tensorflow/tensorflow:2.10.0-gpu-jupyter +FROM tensorflow/tensorflow:2.13.0-gpu-jupyter RUN curl -sSL http://neuro.debian.net/lists/focal.us-nh.full | tee /etc/apt/sources.list.d/neurodebian.sources.list \ && export GNUPGHOME="$(mktemp -d)" \ && echo "disable-ipv6" >> ${GNUPGHOME}/dirmngr.conf \ @@ -8,8 +8,7 @@ RUN curl -sSL http://neuro.debian.net/lists/focal.us-nh.full | tee /etc/apt/sour && apt-get install -y git-annex-standalone git \ && rm -rf /tmp/* COPY [".", "/opt/nobrainer"] -RUN cd /opt/nobrainer \ - && sed -i 's/tensorflow >=/tensorflow-gpu >=/g' setup.cfg +RUN cd /opt/nobrainer RUN python3 -m pip install --no-cache-dir /opt/nobrainer datalad datalad-osf RUN git config --global user.email "neuronets@example.com" \ && git config --global user.name "Neuronets maintainers" diff --git a/setup.cfg b/setup.cfg index c1e002d8..2059f017 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ install_requires = numpy scikit-image tensorflow-probability >= 0.11.0 - tensorflow >= 2.10.0 + tensorflow >= 2.12.0 tensorflow-addons >= 0.12.0 psutil zip_safe = False From 5be8163cfb5f00f39755a6be25927815e9287540 Mon Sep 17 00:00:00 2001 From: Kabilar Date: Thu, 27 Jul 2023 22:22:49 -0500 Subject: [PATCH 09/12] Fix syntax --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d4551bb..bffb2d84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ #### 🐛 Bug Fix - Fix: Update Dockerfiles to download models from the `osf-storage` DataLad data source [#252](https://github.com/neuronets/nobrainer/pull/252) ([@kabilar](https://github.com/kabilar)) -- Fix: Update Dockerfiles to fix `tensorflow`` installation [#252](https://github.com/neuronets/nobrainer/pull/252) ([@kabilar](https://github.com/kabilar)) +- Fix: Update Dockerfiles to fix `tensorflow` installation [#252](https://github.com/neuronets/nobrainer/pull/252) ([@kabilar](https://github.com/kabilar)) # 0.5.0 (Wed Jul 19 2023) From dae831870a4ba77f1d542c59259e7818b44e984e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 31 Jul 2023 23:46:10 +0000 Subject: [PATCH 10/12] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/PyCQA/flake8: 6.0.0 → 6.1.0](https://github.com/PyCQA/flake8/compare/6.0.0...6.1.0) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 97d9244d..4982ebff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ repos: hooks: - id: black - repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 + rev: 6.1.0 hooks: - id: flake8 - repo: https://github.com/PyCQA/isort From f79506c8f591d0771bcfb8040c8ef44a133f2dfe Mon Sep 17 00:00:00 2001 From: Kabilar Date: Sun, 20 Aug 2023 20:15:24 -0500 Subject: [PATCH 11/12] Remove changelog update --- CHANGELOG.md | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bffb2d84..a1e0e2a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,3 @@ -# Upcoming - -#### 🚀 Enhancement - -- Update: Add GitHub Actions to test the build of the Docker images [#252](https://github.com/neuronets/nobrainer/pull/252) ([@kabilar](https://github.com/kabilar)) - -#### 🐛 Bug Fix - -- Fix: Update Dockerfiles to download models from the `osf-storage` DataLad data source [#252](https://github.com/neuronets/nobrainer/pull/252) ([@kabilar](https://github.com/kabilar)) -- Fix: Update Dockerfiles to fix `tensorflow` installation [#252](https://github.com/neuronets/nobrainer/pull/252) ([@kabilar](https://github.com/kabilar)) - # 0.5.0 (Wed Jul 19 2023) #### 🚀 Enhancement From adf7125f0041d1a56dbb1d699ae8952e46e6337b Mon Sep 17 00:00:00 2001 From: ohinds Date: Tue, 29 Aug 2023 19:04:34 -0700 Subject: [PATCH 12/12] Self-hosted runner weekly keepalive (#260) * Add an action to keep our self-hosted runner alive. This action runs once a week on a cron schedule and does basically nothing, just waits for the runner to come up and then shuts it down. --- .github/workflows/keepalive.yaml | 75 ++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 .github/workflows/keepalive.yaml diff --git a/.github/workflows/keepalive.yaml b/.github/workflows/keepalive.yaml new file mode 100644 index 00000000..35119f57 --- /dev/null +++ b/.github/workflows/keepalive.yaml @@ -0,0 +1,75 @@ +name: Weekly register keepalive +run-name: Weekly register keepalive +on: + schedule: + - cron: '15 0 * * 0' +jobs: + + start_ec2_runner: + runs-on: ubuntu-latest + env: + AWS_KEY_ID: ${{ secrets.AWS_KEY_ID }} + AWS_KEY_SECRET: ${{ secrets.AWS_KEY_SECRET }} + AWS_INSTANCE_TYPE: ${{ vars.AWS_INSTANCE_TYPE }} + AWS_IMAGE_ID: ${{ vars.AWS_IMAGE_ID }} + AWS_SECURITY_GROUP: ${{ vars.AWS_SECURITY_GROUP }} + outputs: + EC2_INSTANCE_ID: ${{ steps.start_runner.outputs.EC2_INSTANCE_ID }} + steps: + - name: configure_aws + run: | + set -xe + sudo apt update + sudo apt install -y awscli jq + mkdir ${HOME}/.aws + echo "[default]" > ${HOME}/.aws/credentials + echo "aws_access_key_id = ${AWS_KEY_ID}" >> ${HOME}/.aws/credentials + echo "aws_secret_access_key = ${AWS_KEY_SECRET}" >> ${HOME}/.aws/credentials + echo "region = us-east-1" >> ${HOME}/.aws/credentials + cat ${HOME}/.aws/credentials + - name: start_runner + id: start_runner + run: | + set -xe + output=$(aws ec2 run-instances \ + --instance-type ${AWS_INSTANCE_TYPE} \ + --image-id ${AWS_IMAGE_ID} \ + --security-group-ids ${AWS_SECURITY_GROUP} \ + --count 1) + EC2_INSTANCE_ID=$(echo ${output} | jq -r ".Instances[0].InstanceId") + echo "EC2_INSTANCE_ID=${EC2_INSTANCE_ID}" >> "$GITHUB_OUTPUT" + + guide_notebooks_regression_ec2: + runs-on: [self-hosted, nobrainer-ci-ec2-gpu] + needs: start_ec2_runner + steps: + - name: keepalive_task + run: | + set -xe + ps aux | grep runner + + stop_ec2_runner: + if: always() + runs-on: ubuntu-latest + needs: [start_ec2_runner, guide_notebooks_regression_ec2] + env: + AWS_KEY_ID: ${{ secrets.AWS_KEY_ID }} + AWS_KEY_SECRET: ${{ secrets.AWS_KEY_SECRET }} + steps: + - name: configure_aws + run: | + set -xe + sudo apt update + sudo apt install -y awscli + mkdir ${HOME}/.aws + echo "[default]" > ${HOME}/.aws/credentials + echo "aws_access_key_id = ${AWS_KEY_ID}" >> ${HOME}/.aws/credentials + echo "aws_secret_access_key = ${AWS_KEY_SECRET}" >> ${HOME}/.aws/credentials + echo "region = us-east-1" >> ${HOME}/.aws/credentials + cat ${HOME}/.aws/credentials + - name: stop_runner + env: + EC2_INSTANCE_ID: ${{ needs.start_ec2_runner.outputs.EC2_INSTANCE_ID }} + run: | + set -xe + aws ec2 terminate-instances --instance-ids ${EC2_INSTANCE_ID}