diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c5397ca..1fafd200 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,3 +29,19 @@ jobs: - name: Test with pytest run: | pytest + image-build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-20.04] + python-version: ["3.10", "3.9", "3.8"] + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 1 + - name: Test CPU Docker image build + run: | + docker build -t neuronets/nobrainer:master-cpu -f docker/cpu.Dockerfile . + - name: Test GPU Docker image build + run: | + docker build -t neuronets/nobrainer:master-gpu -f docker/gpu.Dockerfile . diff --git a/.github/workflows/keepalive.yaml b/.github/workflows/keepalive.yaml new file mode 100644 index 00000000..35119f57 --- /dev/null +++ b/.github/workflows/keepalive.yaml @@ -0,0 +1,75 @@ +name: Weekly register keepalive +run-name: Weekly register keepalive +on: + schedule: + - cron: '15 0 * * 0' +jobs: + + start_ec2_runner: + runs-on: ubuntu-latest + env: + AWS_KEY_ID: ${{ secrets.AWS_KEY_ID }} + AWS_KEY_SECRET: ${{ secrets.AWS_KEY_SECRET }} + AWS_INSTANCE_TYPE: ${{ vars.AWS_INSTANCE_TYPE }} + AWS_IMAGE_ID: ${{ vars.AWS_IMAGE_ID }} + AWS_SECURITY_GROUP: ${{ vars.AWS_SECURITY_GROUP }} + outputs: + EC2_INSTANCE_ID: ${{ steps.start_runner.outputs.EC2_INSTANCE_ID }} + steps: + - name: configure_aws + run: | + set -xe + sudo apt update + sudo apt install -y awscli jq + mkdir ${HOME}/.aws + echo "[default]" > ${HOME}/.aws/credentials + echo "aws_access_key_id = ${AWS_KEY_ID}" >> ${HOME}/.aws/credentials + echo "aws_secret_access_key = ${AWS_KEY_SECRET}" >> ${HOME}/.aws/credentials + echo "region = us-east-1" >> ${HOME}/.aws/credentials + cat ${HOME}/.aws/credentials + - name: start_runner + id: start_runner + run: | + set -xe + output=$(aws ec2 run-instances \ + --instance-type ${AWS_INSTANCE_TYPE} \ + --image-id ${AWS_IMAGE_ID} \ + --security-group-ids ${AWS_SECURITY_GROUP} \ + --count 1) + EC2_INSTANCE_ID=$(echo ${output} | jq -r ".Instances[0].InstanceId") + echo "EC2_INSTANCE_ID=${EC2_INSTANCE_ID}" >> "$GITHUB_OUTPUT" + + guide_notebooks_regression_ec2: + runs-on: [self-hosted, nobrainer-ci-ec2-gpu] + needs: start_ec2_runner + steps: + - name: keepalive_task + run: | + set -xe + ps aux | grep runner + + stop_ec2_runner: + if: always() + runs-on: ubuntu-latest + needs: [start_ec2_runner, guide_notebooks_regression_ec2] + env: + AWS_KEY_ID: ${{ secrets.AWS_KEY_ID }} + AWS_KEY_SECRET: ${{ secrets.AWS_KEY_SECRET }} + steps: + - name: configure_aws + run: | + set -xe + sudo apt update + sudo apt install -y awscli + mkdir ${HOME}/.aws + echo "[default]" > ${HOME}/.aws/credentials + echo "aws_access_key_id = ${AWS_KEY_ID}" >> ${HOME}/.aws/credentials + echo "aws_secret_access_key = ${AWS_KEY_SECRET}" >> ${HOME}/.aws/credentials + echo "region = us-east-1" >> ${HOME}/.aws/credentials + cat ${HOME}/.aws/credentials + - name: stop_runner + env: + EC2_INSTANCE_ID: ${{ needs.start_ec2_runner.outputs.EC2_INSTANCE_ID }} + run: | + set -xe + aws ec2 terminate-instances --instance-ids ${EC2_INSTANCE_ID} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 97d9244d..4982ebff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ repos: hooks: - id: black - repo: https://github.com/PyCQA/flake8 - rev: 6.0.0 + rev: 6.1.0 hooks: - id: flake8 - repo: https://github.com/PyCQA/isort diff --git a/docker/cpu.Dockerfile b/docker/cpu.Dockerfile index 61d5e03c..3a691c9a 100644 --- a/docker/cpu.Dockerfile +++ b/docker/cpu.Dockerfile @@ -1,4 +1,4 @@ -FROM tensorflow/tensorflow:2.10.0-jupyter +FROM tensorflow/tensorflow:2.13.0-jupyter RUN curl -sSL http://neuro.debian.net/lists/focal.us-nh.full | tee /etc/apt/sources.list.d/neurodebian.sources.list \ && export GNUPGHOME="$(mktemp -d)" \ && echo "disable-ipv6" >> ${GNUPGHOME}/dirmngr.conf \ @@ -15,7 +15,7 @@ RUN git config --global user.email "neuronets@example.com" \ && git config --global user.name "Neuronets maintainers" RUN datalad clone https://github.com/neuronets/trained-models /models \ && cd /models && git-annex enableremote osf-storage \ - && datalad get -r . + && datalad get -s osf-storage . ENV LC_ALL=C.UTF-8 \ LANG=C.UTF-8 WORKDIR "/work" diff --git a/docker/gpu.Dockerfile b/docker/gpu.Dockerfile index d2af4b36..0d4e2708 100644 --- a/docker/gpu.Dockerfile +++ b/docker/gpu.Dockerfile @@ -1,4 +1,4 @@ -FROM tensorflow/tensorflow:2.10.0-gpu-jupyter +FROM tensorflow/tensorflow:2.13.0-gpu-jupyter RUN curl -sSL http://neuro.debian.net/lists/focal.us-nh.full | tee /etc/apt/sources.list.d/neurodebian.sources.list \ && export GNUPGHOME="$(mktemp -d)" \ && echo "disable-ipv6" >> ${GNUPGHOME}/dirmngr.conf \ @@ -8,14 +8,13 @@ RUN curl -sSL http://neuro.debian.net/lists/focal.us-nh.full | tee /etc/apt/sour && apt-get install -y git-annex-standalone git \ && rm -rf /tmp/* COPY [".", "/opt/nobrainer"] -RUN cd /opt/nobrainer \ - && sed -i 's/tensorflow >=/tensorflow-gpu >=/g' setup.cfg +RUN cd /opt/nobrainer RUN python3 -m pip install --no-cache-dir /opt/nobrainer datalad datalad-osf RUN git config --global user.email "neuronets@example.com" \ && git config --global user.name "Neuronets maintainers" RUN datalad clone https://github.com/neuronets/trained-models /models \ && cd /models && git-annex enableremote osf-storage \ - && datalad get -r . + && datalad get -s osf-storage . ENV LC_ALL=C.UTF-8 \ LANG=C.UTF-8 WORKDIR "/work" diff --git a/setup.cfg b/setup.cfg index c1e002d8..2059f017 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ install_requires = numpy scikit-image tensorflow-probability >= 0.11.0 - tensorflow >= 2.10.0 + tensorflow >= 2.12.0 tensorflow-addons >= 0.12.0 psutil zip_safe = False