task/WG-379: upgrade python to 3.11; reorg Dockerfiles; reorg `pyproj…

…ect.toml` (first pass) (#225) * python black formatting; flake8 on everything * upgrade python to 3.11; reorg Dockerfiles; reorg pyproject * update github actions to 3.11; formatting * gh actions dev dependenceis * upgrade packages * move laspy to main group; fix conda env name * fix tests * fix packages * fix initdb * upgrade packages; update import * resolve sqalchemy warnings * sqlalchemy 2.x syntax * laspy upgrades * update test; add rabbitmq healthcheck * use poetry flake8 and black * update workflows * Update devops/docker-compose.local.yml Co-authored-by: Nathan Franklin <[email protected]> * add make build-dev instruction to readme * Update devops/docker-compose.local.yml Co-authored-by: Nathan Franklin <[email protected]> * update gh action pg * add lazrs * task/WG-379: upgrade python to 3.11; reorg Dockerfiles; reorg `pyproject.toml` (first pass) (#222) * upgrade python to 3.11; reorg Dockerfiles; reorg pyproject * update github actions to 3.11; formatting * gh actions dev dependenceis * upgrade packages * move laspy to main group; fix conda env name * fix tests * fix packages * fix initdb * upgrade packages; update import * resolve sqalchemy warnings * sqlalchemy 2.x syntax * laspy upgrades * update test; add rabbitmq healthcheck * use poetry flake8 and black * update workflows * Update devops/docker-compose.local.yml Co-authored-by: Nathan Franklin <[email protected]> * add make build-dev instruction to readme * Update devops/docker-compose.local.yml Co-authored-by: Nathan Franklin <[email protected]> * update gh action pg * add lazrs --------- Co-authored-by: Nathan Franklin <[email protected]> * Revert "task/WG-379: upgrade python to 3.11; reorg Dockerfiles; reorg `pyproject.toml` (first pass) (#222)" This reverts commit 404e8d9. * hotfix: add unit test using compressed laz file (#224) * Add compressed laz file Changed some file names as the previous .laz files were uncompressed. * Add lidar_medium_subset_las1pt2_utmzone13N_compressed.laz * Fix formatting --------- Co-authored-by: Sal Tijerina <[email protected]> --------- Co-authored-by: Nathan Franklin <[email protected]>
TACC-Cloud · Nov 27, 2024 · bdb1ac3 · bdb1ac3
1 parent ca4f917
commit bdb1ac3
Show file tree

Hide file tree

Showing 37 changed files with 1,944 additions and 1,278 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -9,18 +9,33 @@ jobs:
   Linting:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+
+      - name: Fetch base and install Poetry
+        run: |
+          git fetch origin ${{github.base_ref}}
+          pipx install poetry
+
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.9
-      - name: Install flake8 and black
-        run: pip install flake8 black
+          python-version: "3.11"
+          cache: "poetry"
+
+      - name: Install Python Packages
+        run: |
+          cd devops
+          poetry install --only dev
+
       - name: Run flake8
-        run: flake8 geoapi
+        run: |
+          cd devops
+          poetry run flake8 ../geoapi
+
       - name: Formatting with black
         run: |
-          black --check .
+          cd devops
+          poetry run black --check ../geoapi
 
   Geoapi_Unit_Tests:
     runs-on: ubuntu-latest
@@ -42,38 +57,39 @@ jobs:
           --health-timeout 5s
           --health-retries 5
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+
+      - name: Fetch base and install Poetry
+        run: |
+          git fetch origin ${{github.base_ref}}
+          pipx install poetry
+
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.9
+          python-version: "3.11"
+          cache: "poetry"
+
       - name: Install ffmpeg
         run: |
           sudo apt-get update
           sudo apt-get install -y ffmpeg
-      - name: Setup Poetry
-        uses: abatilo/[email protected]
-        with:
-          poetry-version: 1.8.3
-      - uses: actions/cache@v1
-        with:
-          path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-poetry-
+
       - name: Install Python Packages
         run: |
           cd devops
-          poetry install
+          poetry install --with dev,flask,worker
+
       - name: Run server-side unit tests
         run: |
           cd devops
           poetry run pytest ../geoapi
+
   Workers_Unit_Tests:
     runs-on: ubuntu-latest
     services:
       postgres:
-        image: mdillon/postgis:11-alpine
+        image: postgis/postgis:11-3.3-alpine
         env:
           POSTGRES_USER: dev
           POSTGRES_PASSWORD: dev
@@ -86,15 +102,15 @@ jobs:
           --health-timeout 5s
           --health-retries 5
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: docker/login-action@v1
         with:
           password: ${{ secrets.DOCKER_PASSWORD }}
           username: ${{ secrets.DOCKER_USER  }}
       - name: Build worker image
         run: |
           docker pull taccaci/geoapi-workers:latest
-          docker build --cache-from taccaci/geoapi-workers:latest -t taccaci/geoapi-workers:latest -f devops/Dockerfile.worker .
+          docker build --cache-from taccaci/geoapi-workers:latest -t taccaci/geoapi-workers:latest -f devops/Dockerfile.worker --target development .
       - name: Run worker test
         run: |
           docker run --network="host" -e APP_ENV='testing' -e DB_HOST='localhost'  taccaci/geoapi-workers:latest pytest -m "worker"
diff --git a/Makefile b/Makefile
@@ -53,3 +53,10 @@ deploy-geoapi:
 .PHONY: deploy-workers
 deploy-workers:
 	docker push $(GEOAPI_WORKERS):$(TAG)
+
+.PHONY: build-dev
+build-dev:
+	docker build -t $(GEOAPI_IMAGE):$(TAG) --target development -f devops/Dockerfile .
+	docker build -t $(GEOAPI_WORKERS):$(TAG) --target development -f devops/Dockerfile.worker .
+	docker tag $(GEOAPI_WORKERS):$(TAG) $(GEOAPI_WORKERS):local
+	docker tag $(GEOAPI_IMAGE):$(TAG) $(GEOAPI_IMAGE):local
diff --git a/README.md b/README.md
@@ -3,8 +3,8 @@
 ## Overview and Architecture
 
 GeoAPI is a restful API to create geospatial features in a PostGIS database. Users create a map "project" then
-can add features to it. The development docker-compose file has 3 containers: 
-* a PostGIS database exposing 5432, 
+can add features to it. The development docker-compose file has 3 containers:
+* a PostGIS database exposing 5432,
 * the api which exposes port 8000 behind gunicorn
 * a nginx server to serve static files and proxy to the api, running on port 8080.
 
@@ -22,7 +22,7 @@ information. An .env file for developers can be found on [UT Stache](https://sta
 The API is built with flask and flask-restplus. It is running in its own container
 under gunicorn on port 8000
 
-`make build`
+`make build-dev`
 `make start`
 
 ###### Initialize the database (for local development and unit testing)
@@ -97,4 +97,4 @@ on Jenkins deployment workflows.
 
 ## Python client
 
-The python package can be found at [PyPi](https://pypi.org/project/geoapi-client/).  More details can be found in [Python Client](./PYTHON_CLIENT.md)
+The python package can be found at [PyPi](https://pypi.org/project/geoapi-client/).  More details can be found in [Python Client](./PYTHON_CLIENT.md)
diff --git a/devops/Dockerfile b/devops/Dockerfile
@@ -1,24 +1,59 @@
-FROM python:3.9-slim
-RUN apt-get update -q && apt-get install -q -y \
-  build-essential \
-  software-properties-common \
-  libgdal-dev \
-  ffmpeg \
-  curl \
-  git
-
-ENV POETRY_VERSION=1.8.3
-ENV POETRY_HOME=/opt/poetry
-ENV PATH="$POETRY_HOME/bin:$PATH"
-RUN curl -sSL https://install.python-poetry.org | python3 -
-RUN poetry config virtualenvs.create false
-
-WORKDIR /opt
+FROM python:3.11-slim AS python-base
+
+LABEL maintainer="DesignSafe-CI <[email protected]>"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    software-properties-common \
+    libgdal-dev \
+    ffmpeg \
+    curl \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+
+# https://python-poetry.org/docs/configuration/#using-environment-variables
+ENV POETRY_VERSION=1.8 \
+    POETRY_HOME="/opt/poetry" \
+    POETRY_VIRTUALENVS_IN_PROJECT=true \
+    PYSETUP_PATH="/opt/pysetup" \
+    VENV_PATH="/opt/pysetup/.venv"
+
+ENV PATH="$VENV_PATH/bin:$POETRY_HOME/bin:$PATH"
+
+# Install poetry version $POETRY_VERSION to $POETRY_HOME
+RUN pip3 install --no-cache-dir --upgrade pip setuptools wheel \
+    && python3 -m venv "$POETRY_HOME" \
+    && "$POETRY_HOME/bin/pip" install --no-cache-dir poetry=="$POETRY_VERSION"
+
+# Copy project requirement files to ensure they will be cached.
+WORKDIR $PYSETUP_PATH
+
 COPY devops/poetry.lock devops/pyproject.toml ./
-RUN poetry install
 
-RUN mkdir /app
+##############
+# `development` image target is used for local development
+FROM python-base AS development
+
+# Install dev dependencies
+RUN "$POETRY_HOME/bin/poetry" install --with dev,flask,worker
+
 COPY geoapi /app/geoapi
+
+ENV PYTHONPATH=/app
+
+WORKDIR /app/geoapi
+
+##############
+# `production` image target is used for deployed runtime environments
+FROM python-base AS production
+
+# Install runtime dependencies
+RUN "$POETRY_HOME/bin/poetry" install --with flask,worker
+
+COPY geoapi /app/geoapi
+
 ENV PYTHONPATH=/app
 
 WORKDIR /app/geoapi
diff --git a/devops/Dockerfile.worker b/devops/Dockerfile.worker
@@ -1,4 +1,6 @@
-FROM ubuntu:22.04
+FROM python:3.11-slim AS python-base
+
+LABEL maintainer="DesignSafe-CI <[email protected]>"
 
 ENV DEBIAN_FRONTEND=noninteractive
 
@@ -16,66 +18,73 @@ RUN apt-get update && apt-get install -y \
     git \
     cmake \
     build-essential \
-    python3.9 \
-    python3-pip \
-    python3-dev \
     ffmpeg \
     unzip \
     wget \
     libc6-dev \
     libtbb-dev\
-    libcgal-dev
+    libcgal-dev \
+    && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /opt
 
 # Install PotreeConverter
 # c2328c4 is v2.1.1 and some additional fixes
-RUN git clone -b develop https://github.com/potree/PotreeConverter.git && cd PotreeConverter && git checkout c2328c4 && \
-mkdir build && cd build && \
-cmake .. -DCMAKE_BUILD_TYPE=Release && \
-make
+RUN git clone -b develop https://github.com/potree/PotreeConverter.git \
+    && cd PotreeConverter \
+    && git checkout c2328c4 \
+    && mkdir build \
+    && cd build \
+    && cmake .. -DCMAKE_BUILD_TYPE=Release \
+    && make
 
 # Setup our page template for PotreeConverter
-ADD devops/misc/potree/page_template/nsf_logo.png /opt/PotreeConverter/build/resources/page_template/
-ADD devops/misc/potree/page_template/nsf_logo_snippet.txt /tmp/
+COPY devops/misc/potree/page_template/nsf_logo.png /opt/PotreeConverter/build/resources/page_template/
+COPY devops/misc/potree/page_template/nsf_logo_snippet.txt /tmp/
+
 #  - add nsf logo
 RUN sed -i '/<body>/r /tmp/nsf_logo_snippet.txt' /opt/PotreeConverter/build/resources/page_template/viewer_template.html
+
 #  - remove reference to background image
 RUN sed -i 's/style="[^"]*background-image:[^"]*"//' /opt/PotreeConverter/build/resources/page_template/viewer_template.html
 
-
 # Install Miniforge for our Python environment (provides easier PDAL installation)
-RUN wget -q -O miniforge.sh https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-$(uname -m).sh && \
-    sh miniforge.sh -b -p /opt/conda && \
-    rm miniforge.sh
+RUN wget -q -O miniforge.sh https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-$(uname -m).sh \
+    && sh miniforge.sh -b -p /opt/conda \
+    && rm miniforge.sh
+
 ENV PATH="/opt/conda/bin:${PATH}"
 
-# Create a conda environment with Python 3.9 and activate it
-RUN conda create -n py39env python=3.9 -y
-SHELL ["conda", "run", "-n", "py39env", "/bin/bash", "-c"]
+# Create a conda environment with Python 3.11 and activate it
+RUN conda create -n py311env python=3.11 -y
+SHELL ["conda", "run", "-n", "py311env", "/bin/bash", "-c"]
 
 # Install PDAL using conda
 RUN conda install -c conda-forge pdal -y
 
-# Install needed python packages using poetry
-RUN pip install poetry==1.8.3
-RUN poetry config virtualenvs.create false
-COPY devops/pyproject.toml devops/poetry.lock ./
-RUN poetry install
+# https://python-poetry.org/docs/configuration/#using-environment-variables
+ENV POETRY_VERSION=1.8 \
+    POETRY_HOME="/opt/poetry" \
+    POETRY_VIRTUALENVS_IN_PROJECT=true \
+    PYSETUP_PATH="/opt/pysetup" \
+    VENV_PATH="/opt/pysetup/.venv"
 
-# Populate image with geoapi and set PYTHONPATH
-RUN mkdir app
-COPY geoapi /app/geoapi
-WORKDIR /app/geoapi
-ENV PYTHONPATH=/app
+ENV PATH="$VENV_PATH/bin:$POETRY_HOME/bin:$PATH"
+
+# Install poetry version $POETRY_VERSION to $POETRY_HOME
+RUN pip3 install --no-cache-dir --upgrade pip setuptools wheel \
+    && python3 -m venv "$POETRY_HOME" \
+    && "$POETRY_HOME/bin/pip" install --no-cache-dir poetry=="$POETRY_VERSION"
+
+COPY devops/pyproject.toml devops/poetry.lock ./
 
 # Create an entrypoint script that activates our conda environment
 RUN echo '#!/bin/bash' > /usr/local/bin/entrypoint.sh && \
     echo 'set -e' >> /usr/local/bin/entrypoint.sh && \
     echo '' >> /usr/local/bin/entrypoint.sh && \
     echo '# Activate conda and the specific environment' >> /usr/local/bin/entrypoint.sh && \
     echo '. /opt/conda/etc/profile.d/conda.sh' >> /usr/local/bin/entrypoint.sh && \
-    echo 'conda activate py39env' >> /usr/local/bin/entrypoint.sh && \
+    echo 'conda activate py311env' >> /usr/local/bin/entrypoint.sh && \
     echo '' >> /usr/local/bin/entrypoint.sh && \
     echo '# Execute the passed command' >> /usr/local/bin/entrypoint.sh && \
     echo 'exec "$@"' >> /usr/local/bin/entrypoint.sh && \
@@ -86,7 +95,33 @@ ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
 
 # activate conda (to handle when user starts a bash via docker exec)
 RUN echo '. /opt/conda/etc/profile.d/conda.sh' >> /root/.bashrc && \
-    echo 'conda activate py39env' >> /root/.bashrc
+    echo 'conda activate py311env' >> /root/.bashrc
 
 # Set a default command (can be overridden by docker-compose)
 CMD ["bash"]
+
+##############
+# `development` image target is used for local development
+FROM python-base AS development
+
+# Install dev dependencies
+RUN "$POETRY_HOME/bin/poetry" install --with dev,flask,worker
+
+COPY geoapi /app/geoapi
+
+ENV PYTHONPATH=/app
+
+WORKDIR /app/geoapi
+
+##############
+# `production` image target is used for deployed runtime environments
+FROM python-base AS production
+
+# Install runtime dependencies
+RUN "$POETRY_HOME/bin/poetry" install --with flask,worker
+
+COPY geoapi /app/geoapi
+
+ENV PYTHONPATH=/app
+
+WORKDIR /app/geoapi