Merge branch 'master' into ohinds-nb-dataset

.github/workflows/ci.yml

@@ -29,3 +29,19 @@ jobs:
       - name: Test with pytest
         run: |
           pytest
+  image-build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-20.04]
+        python-version: ["3.10", "3.9", "3.8"]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+      - name: Test CPU Docker image build
+        run: |
+          docker build -t neuronets/nobrainer:master-cpu -f docker/cpu.Dockerfile .
+      - name: Test GPU Docker image build
+        run: |
+          docker build -t neuronets/nobrainer:master-gpu -f docker/gpu.Dockerfile .

.github/workflows/keepalive.yaml

@@ -0,0 +1,75 @@
+name: Weekly register keepalive
+run-name: Weekly register keepalive
+on:
+  schedule:
+    - cron: '15 0 * * 0'
+jobs:
+
+  start_ec2_runner:
+    runs-on: ubuntu-latest
+    env:
+      AWS_KEY_ID: ${{ secrets.AWS_KEY_ID }}
+      AWS_KEY_SECRET: ${{ secrets.AWS_KEY_SECRET }}
+      AWS_INSTANCE_TYPE: ${{ vars.AWS_INSTANCE_TYPE }}
+      AWS_IMAGE_ID: ${{ vars.AWS_IMAGE_ID }}
+      AWS_SECURITY_GROUP: ${{ vars.AWS_SECURITY_GROUP }}
+    outputs:
+      EC2_INSTANCE_ID: ${{ steps.start_runner.outputs.EC2_INSTANCE_ID }}
+    steps:
+      - name: configure_aws
+        run: |
+          set -xe
+          sudo apt update
+          sudo apt install -y awscli jq
+          mkdir ${HOME}/.aws
+          echo "[default]" > ${HOME}/.aws/credentials
+          echo "aws_access_key_id = ${AWS_KEY_ID}" >> ${HOME}/.aws/credentials
+          echo "aws_secret_access_key = ${AWS_KEY_SECRET}" >> ${HOME}/.aws/credentials
+          echo "region = us-east-1" >> ${HOME}/.aws/credentials
+          cat ${HOME}/.aws/credentials
+      - name: start_runner
+        id: start_runner
+        run: |
+          set -xe
+          output=$(aws ec2 run-instances \
+            --instance-type ${AWS_INSTANCE_TYPE} \
+            --image-id ${AWS_IMAGE_ID} \
+            --security-group-ids ${AWS_SECURITY_GROUP} \
+            --count 1)
+          EC2_INSTANCE_ID=$(echo ${output} | jq -r ".Instances[0].InstanceId")
+          echo "EC2_INSTANCE_ID=${EC2_INSTANCE_ID}" >> "$GITHUB_OUTPUT"
+
+  guide_notebooks_regression_ec2:
+    runs-on: [self-hosted, nobrainer-ci-ec2-gpu]
+    needs: start_ec2_runner
+    steps:
+      - name: keepalive_task
+        run: |
+          set -xe
+          ps aux | grep runner
+
+  stop_ec2_runner:
+    if: always()
+    runs-on: ubuntu-latest
+    needs: [start_ec2_runner, guide_notebooks_regression_ec2]
+    env:
+      AWS_KEY_ID: ${{ secrets.AWS_KEY_ID }}
+      AWS_KEY_SECRET: ${{ secrets.AWS_KEY_SECRET }}
+    steps:
+      - name: configure_aws
+        run: |
+          set -xe
+          sudo apt update
+          sudo apt install -y awscli
+          mkdir ${HOME}/.aws
+          echo "[default]" > ${HOME}/.aws/credentials
+          echo "aws_access_key_id = ${AWS_KEY_ID}" >> ${HOME}/.aws/credentials
+          echo "aws_secret_access_key = ${AWS_KEY_SECRET}" >> ${HOME}/.aws/credentials
+          echo "region = us-east-1" >> ${HOME}/.aws/credentials
+          cat ${HOME}/.aws/credentials
+      - name: stop_runner
+        env:
+          EC2_INSTANCE_ID: ${{ needs.start_ec2_runner.outputs.EC2_INSTANCE_ID }}
+        run: |
+          set -xe
+          aws ec2 terminate-instances --instance-ids ${EC2_INSTANCE_ID}

.pre-commit-config.yaml

@@ -16,7 +16,7 @@ repos:
     hooks:
     -   id: black
 -   repo: https://github.com/PyCQA/flake8
-    rev: 6.0.0
+    rev: 6.1.0
     hooks:
     - id: flake8
 -   repo: https://github.com/PyCQA/isort

docker/cpu.Dockerfile

@@ -1,4 +1,4 @@
-FROM tensorflow/tensorflow:2.10.0-jupyter
+FROM tensorflow/tensorflow:2.13.0-jupyter
 RUN curl -sSL http://neuro.debian.net/lists/focal.us-nh.full | tee /etc/apt/sources.list.d/neurodebian.sources.list \
   && export GNUPGHOME="$(mktemp -d)" \
   && echo "disable-ipv6" >> ${GNUPGHOME}/dirmngr.conf \
@@ -15,7 +15,7 @@ RUN git config --global user.email "[email protected]" \
     && git config --global user.name "Neuronets maintainers"
 RUN datalad clone https://github.com/neuronets/trained-models /models \
   && cd /models && git-annex enableremote osf-storage \
-  && datalad get -r .
+  && datalad get -s osf-storage .
 ENV LC_ALL=C.UTF-8 \
     LANG=C.UTF-8
 WORKDIR "/work"

docker/gpu.Dockerfile

@@ -1,4 +1,4 @@
-FROM tensorflow/tensorflow:2.10.0-gpu-jupyter
+FROM tensorflow/tensorflow:2.13.0-gpu-jupyter
 RUN curl -sSL http://neuro.debian.net/lists/focal.us-nh.full | tee /etc/apt/sources.list.d/neurodebian.sources.list \
   && export GNUPGHOME="$(mktemp -d)" \
   && echo "disable-ipv6" >> ${GNUPGHOME}/dirmngr.conf \
@@ -8,14 +8,13 @@ RUN curl -sSL http://neuro.debian.net/lists/focal.us-nh.full | tee /etc/apt/sour
   && apt-get install -y git-annex-standalone git \
   && rm -rf /tmp/*
 COPY [".", "/opt/nobrainer"]
-RUN cd /opt/nobrainer \
-    && sed -i 's/tensorflow >=/tensorflow-gpu >=/g' setup.cfg
+RUN cd /opt/nobrainer
 RUN python3 -m pip install --no-cache-dir /opt/nobrainer datalad datalad-osf
 RUN git config --global user.email "[email protected]" \
     && git config --global user.name "Neuronets maintainers"
 RUN datalad clone https://github.com/neuronets/trained-models /models \
   && cd /models && git-annex enableremote osf-storage \
-  && datalad get -r .
+  && datalad get -s osf-storage .
 ENV LC_ALL=C.UTF-8 \
     LANG=C.UTF-8
 WORKDIR "/work"

setup.cfg

@@ -38,7 +38,7 @@ install_requires =
     numpy
     scikit-image
     tensorflow-probability >= 0.11.0
-    tensorflow >= 2.10.0
+    tensorflow >= 2.12.0
     tensorflow-addons >= 0.12.0
     psutil
 zip_safe = False