Skip to content

Commit

Permalink
Merge pull request #4 from allenai/refactor-upgrade
Browse files Browse the repository at this point in the history
refactor
  • Loading branch information
pbeukema authored Dec 12, 2023
2 parents 797dcd8 + b776edd commit c1d2a51
Show file tree
Hide file tree
Showing 179 changed files with 390,255 additions and 573 deletions.
3 changes: 2 additions & 1 deletion .dockerignore
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
data/*
.env/*
.env/*
**/.git
114 changes: 43 additions & 71 deletions .github/workflows/build_and_push_docker_images.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,79 +4,52 @@ on:
push:

env:
GHCR: ghcr.io
GHCR_IMAGE_NAME: ${{ github.repository }}
GCR: gcr.io
GCR_IMAGE_NAME: skylight-docker-images/gh-actions/${{ github.repository }}/${{ github.ref_name }}
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}

jobs:
build-and-push-image-ghcr-vessel-detection-download:
runs-on: ubuntu-latest
steps:
- name: Checkout source code
uses: actions/checkout@v3

- name: Log in to the Container registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Extract GHCR metadata (tags, labels) for Docker vessel-detection-download
id: meta
uses: docker/metadata-action@v4
with:
images: ${{ env.GHCR }}/${{ env.GHCR_IMAGE_NAME }}/vessel-detection-download
tags: |
type=sha,format=short
type=raw,value=latest
- name: Build and push Docker image vessel-detection-download to GHCR
uses: docker/build-push-action@v3
with:
context: .
file: docker/downloader.dockerfile
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-and-push-image:
runs-on: ubuntu-20.04

build-and-push-image-ghcr-vessel-detection-train:
runs-on: ubuntu-latest
steps:
- name: Check disk space
run: df . -h
- name: Free disk space
run: |
sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
sudo rm -rf \
/usr/share/dotnet /usr/local/lib/android /opt/ghc \
/usr/local/share/powershell /usr/share/swift /usr/local/.ghcup \
/usr/lib/jvm || true
echo "some directories deleted"
sudo apt install aptitude -y >/dev/null 2>&1
sudo aptitude purge aria2 ansible azure-cli shellcheck rpm xorriso zsync \
esl-erlang firefox gfortran-8 gfortran-9 google-chrome-stable \
imagemagick \
libmagickcore-dev libmagickwand-dev libmagic-dev ant ant-optional kubectl \
mercurial apt-transport-https mono-complete libmysqlclient \
unixodbc-dev yarn chrpath libssl-dev libxft-dev \
libfreetype6 libfreetype6-dev libfontconfig1 libfontconfig1-dev \
snmp pollinate libpq-dev postgresql-client powershell ruby-full \
sphinxsearch subversion mongodb-org azure-cli microsoft-edge-stable \
-y -f >/dev/null 2>&1
sudo aptitude purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true
sudo apt purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true
sudo aptitude purge '~n ^mysql' -f -y >/dev/null 2>&1
sudo aptitude purge '~n ^php' -f -y >/dev/null 2>&1
sudo aptitude purge '~n ^dotnet' -f -y >/dev/null 2>&1
sudo apt-get autoremove -y >/dev/null 2>&1
sudo apt-get autoclean -y >/dev/null 2>&1
echo "some packages purged"
- name: Check disk space
run: |
sudo dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -nr | head
df . -h
sudo du /usr/ -hx -d 4 --threshold=1G | sort -hr | head
- name: Checkout source code
uses: actions/checkout@v3

- name: Log in to the Container registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Extract GHCR metadata (tags, labels) for Docker vessel-detection-train
id: meta
uses: docker/metadata-action@v4
with:
images: ${{ env.GHCR }}/${{ env.GHCR_IMAGE_NAME }}/vessel-detection-train
tags: |
type=sha,format=short
type=raw,value=latest
- name: Build and push Docker image vessel-detection-train to GHCR
uses: docker/build-push-action@v3
with:
context: .
file: docker/train.dockerfile
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}

build-and-push-image-ghcr-vessel-detection:
runs-on: ubuntu-latest
steps:
- name: Checkout source code
uses: actions/checkout@v3
lfs: true

- name: Log in to the Container registry
uses: docker/login-action@v2
Expand All @@ -85,20 +58,19 @@ jobs:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Extract GHCR metadata (tags, labels) for Docker vessel-detection
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@v4
with:
images: ${{ env.GHCR }}/${{ env.GHCR_IMAGE_NAME }}/vessel-detection
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
tags: |
type=sha,format=short
type=raw,value=latest
type=raw,value=latest,enable={{is_default_branch}}
- name: Build and push Docker image vessel-detection to GHCR
- name: Build and push Docker image
uses: docker/build-push-action@v3
with:
context: .
file: docker/inference.dockerfile
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
39 changes: 37 additions & 2 deletions .github/workflows/tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,46 @@ on:

jobs:
unit-tests:
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
env:
COMPOSE_FILE: docker-compose.yml

steps:
- name: Check disk space
run: df . -h
- name: Free disk space
run: |
sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
sudo rm -rf \
/usr/share/dotnet /usr/local/lib/android /opt/ghc \
/usr/local/share/powershell /usr/share/swift /usr/local/.ghcup \
/usr/lib/jvm || true
echo "some directories deleted"
sudo apt install aptitude -y >/dev/null 2>&1
sudo aptitude purge aria2 ansible azure-cli shellcheck rpm xorriso zsync \
esl-erlang firefox gfortran-8 gfortran-9 google-chrome-stable \
imagemagick \
libmagickcore-dev libmagickwand-dev libmagic-dev ant ant-optional kubectl \
mercurial apt-transport-https mono-complete libmysqlclient \
unixodbc-dev yarn chrpath libssl-dev libxft-dev \
libfreetype6 libfreetype6-dev libfontconfig1 libfontconfig1-dev \
snmp pollinate libpq-dev postgresql-client powershell ruby-full \
sphinxsearch subversion mongodb-org azure-cli microsoft-edge-stable \
-y -f >/dev/null 2>&1
sudo aptitude purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true
sudo apt purge microsoft-edge-stable -f -y >/dev/null 2>&1 || true
sudo aptitude purge '~n ^mysql' -f -y >/dev/null 2>&1
sudo aptitude purge '~n ^php' -f -y >/dev/null 2>&1
sudo aptitude purge '~n ^dotnet' -f -y >/dev/null 2>&1
sudo apt-get autoremove -y >/dev/null 2>&1
sudo apt-get autoclean -y >/dev/null 2>&1
echo "some packages purged"
- name: Check disk space
run: |
sudo dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -nr | head
df . -h
sudo du /usr/ -hx -d 4 --threshold=1G | sort -hr | head
- name: Checkout repository
uses: actions/checkout@v3
with:
Expand All @@ -38,4 +73,4 @@ jobs:
- name: Run unit and integration tests

run: |
docker compose run test
docker compose run test pytest --ignore=/src/tests/test_main.py -v
7 changes: 6 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
*.pyc
.vscode
.env
.env
**/**/.DS_Store
**/**/nohup.out
**/.ipynb_checkpoints/
**/.venv/
**/*.ipynb
76 changes: 76 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
exclude: ais/data/machine_annotations/tsr/standard_rendezvous.csv
exclude_types: ["*.geojson"]
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
hooks:
- id: check-yaml
- id: end-of-file-fixer
- id: trailing-whitespace
- id: check-json
- id: mixed-line-ending
- id: requirements-txt-fixer
- id: pretty-format-json
args: ["--autofix"]
- id: check-case-conflict
- id: check-docstring-first
- id: check-added-large-files
- id: check-ast
- id: check-byte-order-marker
- id: check-executables-have-shebangs
- id: check-merge-conflict
- id: check-toml
- id: debug-statements
- id: detect-aws-credentials
args: [--allow-missing-credentials]
- id: detect-private-key

- repo: https://github.com/pre-commit/mirrors-mypy
rev: v1.1.1
hooks:
- id: mypy
args:
[
--install-types,
--ignore-missing-imports,
--disallow-untyped-defs,
--ignore-missing-imports,
--non-interactive,
]
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
hooks:
- id: detect-private-key
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
hooks:
- id: check-added-large-files
- repo: https://github.com/PyCQA/bandit
rev: "1.7.5"
hooks:
- id: bandit
exclude: ^tests/
args:
- -s
- B101
- repo: local
hooks:
- id: interrogate
name: interrogate
language: system
entry: interrogate
types: [python]
args:
[
--ignore-init-method,
--ignore-init-module,
-p,
-vv,
ais,
--fail-under=80,
]
- repo: https://github.com/charliermarsh/ruff-pre-commit
rev: "v0.0.257"
hooks:
- id: ruff
args: [--fix, --exit-non-zero-on-fix, --ignore, "E501"]
39 changes: 39 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
# Base Image
FROM nvidia/cuda:11.3.1-cudnn8-runtime-ubuntu20.04@sha256:9ccfe38d9cb31ae23f6f8f9595b450565da18399c2e71ebc9a5c079f786319e3

# Environment Variables
ENV WANDB_API_KEY=$WANDB_API_KEY \
RUN_TAG=$RUN_TAG \
WANDB_MODE=$WANDB_MODE \
WANDB_START_METHOD="thread" \
WANDB_PROJECT="vessel-detection" \
COPERNICUS_USERNAME=$COPERNICUS_USERNAME \
COPERNICUS_PASSWORD=$COPERNICUS_PASSWORD

# System Dependencies and Cleanup
RUN apt-get update -y && \
DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y tzdata && \
apt-get install -y software-properties-common ffmpeg libsm6 libxext6 libhdf5-serial-dev netcdf-bin libnetcdf-dev && \
add-apt-repository ppa:ubuntugis/ubuntugis-unstable && \
apt-get update && \
apt-get install -y curl build-essential gdal-bin libgdal-dev libpq-dev python3-gdal python3-pip apt-transport-https ca-certificates gnupg && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*

# Copy only the necessary files
COPY requirements.txt /home/vessel_detection/requirements.txt

# Install Python Packages
RUN pip install --no-cache-dir -r /home/vessel_detection/requirements.txt

# Set Working Directory and Prepare App
WORKDIR /home/vessel_detection/src
COPY src /home/vessel_detection/src
COPY tests /home/vessel_detection/src/
RUN mkdir -p /root/.cache/torch/hub/checkpoints/
COPY torch_weights/swin_v2_s-637d8ceb.pth /root/.cache/torch/hub/checkpoints/swin_v2_s-637d8ceb.pth
COPY torch_weights/resnet50-0676ba61.pth /root/.cache/torch/hub/checkpoints/resnet50-0676ba61.pth
COPY torch_weights/swin_v2_t-b137f0e2.pth /root/.cache/torch/hub/checkpoints/swin_v2_t-b137f0e2.pth

# CMD
CMD ["python3", "main.py"]

This file was deleted.

Loading

0 comments on commit c1d2a51

Please sign in to comment.