#97: CI refactoring to remove duplicate commands

DARMA-tasking · Sep 6, 2024 · 283f0b8 · 283f0b8
1 parent 82fa119
commit 283f0b8
Show file tree

Hide file tree

Showing 8 changed files with 129 additions and 144 deletions.
diff --git a/.github/workflows/build-and-test-macos.yml b/.github/workflows/build-and-test-macos.yml
@@ -46,7 +46,7 @@ jobs:
         VT_TV_OUTPUT_DIR: ${{ github.workspace }}/output
         VT_TV_TESTS_OUTPUT_DIR: ${{ github.workspace }}/output/tests
         VT_TV_ARTIFACTS_DIR: /tmp/artifacts
-        CONDA_PATH: /opt/miniconda # ~/miniconda3
+        CONDA_PREFIX: /opt/miniconda3
         CC: ~
         CXX: ~
         GCOV: ~
@@ -117,7 +117,7 @@ jobs:
           path: |
             ${{ env.VTK_SRC_DIR }}
             ${{ env.VTK_BUILD_DIR }}
-            ${{ env.CONDA_PATH }}
+            ${{ env.CONDA_PREFIX }}
             ~/.zshrc
             ~/.bash_profile
           key: ${{ env.CACHE_KEY }}
@@ -126,59 +126,17 @@ jobs:
       - name: Setup Miniconda3
         if: ${{steps.base-cache.outputs.cache-hit != 'true'}}
         run: |
-          mkdir -p ~/miniconda3
-          if [[ $(uname -a) == *"Darwin"* ]]; then
-            if [[ $(arch) == 'arm64' ]]; then
-              curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o ~/miniconda.sh
-            else
-              curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o ~/miniconda.sh
-            fi
-          else
-            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -o ~/miniconda.sh
-          fi
-          bash ~/miniconda.sh -b -u -p ${{ env.CONDA_PATH }}
-          rm -rf ~/miniconda.sh
-
-          ${{ env.CONDA_PATH }}/bin/conda init bash
-          ${{ env.CONDA_PATH }}/bin/conda init zsh
-          if [ -f ~/.zshrc ]; then
-            . ~/.zshrc
-          fi
-          if [ -f ~/.profile ]; then
-            . ~/.profile
-          fi
-          if [ -f ~/.bashrc ]; then
-            . ~/.bashrc
-          fi
-
-          echo "Conda path: $(which conda)"
-          echo "Conda version: $(conda --version)"
-
-          PYTHON_VERSIONS=(`echo '${{ join(matrix.host.python) }}' | sed 's/,/\n/g'`)
-          for python_version in "${PYTHON_VERSIONS[@]}"
-          do
-            echo "::group::Create conda environment (py${python_version})"
-            conda create -y -n py${python_version} python=${python_version}
-
-            . ${{ env.CONDA_PATH }}/etc/profile.d/conda.sh && conda activate py${python_version}
-            echo "Python version: $(python --version)"
-            pip install nanobind
-            conda deactivate
-            echo "::endgroup::"
-          done
+          sudo CONDA_PREFIX=${{ env.CONDA_PREFIX }} bash ./ci/setup_conda.sh ${{ join(matrix.host.python) }}
 
       - name: Setup VTK ${{ matrix.host.vtk }}
         if: ${{steps.base-cache.outputs.cache-hit != 'true'}}
         run: |
           mkdir -p ${{ env.VTK_SRC_DIR }}
           git clone --recursive --branch v${{ matrix.host.vtk }} https://gitlab.kitware.com/vtk/vtk.git ${{ env.VTK_SRC_DIR }}
-          VTK_DIR=${{ env.VTK_BUILD_DIR }} VTK_SRC_DIR=${{ env.VTK_SRC_DIR }} bash ${{ github.workspace }}/ci/vtk_build.sh
+          VTK_DIR=${{ env.VTK_BUILD_DIR }} VTK_SRC_DIR=${{ env.VTK_SRC_DIR }} bash ./ci/vtk_build.sh
 
       - name: Build
         run: |
-          cd ${{ github.workspace }}
-          chmod +x ./ci/build.sh
-
           mkdir -p ${{ env.VT_TV_BUILD_DIR }}
 
           CC="${{ env.CC }}" \
@@ -191,56 +149,22 @@ jobs:
           GCOV="${{ env.GCOV }}" \
           VT_TV_PYTHON_BINDINGS_ENABLED=OFF \
           VT_TV_WERROR_ENABLED=ON \
-          ./build.sh
+          bash ./build.sh
 
       - name: Test
         run: |
-          cd ${{ github.workspace }}
-          chmod +x ./ci/test.sh
-
           VTK_DIR=${{ env.VTK_BUILD_DIR }} \
           VT_TV_COVERAGE_ENABLED=${{ env.VT_TV_COVERAGE_ENABLED }} \
           VT_TV_OUTPUT_DIR="${{ env.VT_TV_OUTPUT_DIR }}" \
-          ./ci/test.sh
+          bash ./ci/test.sh
 
       - name: Build Python package (${{ join(matrix.host.python, ', ') }})
         run: |
-          cd ${{ github.workspace }}
-          chmod +x ./ci/python_build.sh
-
-          PYTHON_VERSIONS=(`echo '${{ join(matrix.host.python) }}' | sed 's/,/\n/g'`)
-          export VTK_DIR=${{ env.VTK_BUILD_DIR }}
-          for python_version in "${PYTHON_VERSIONS[@]}"
-          do
-            echo "::group::Build Python package (${python_version})"
-            VTK_DIR=${{ env.VTK_BUILD_DIR }} \
-            CONDA_PATH=${{ env.CONDA_PATH }} \
-            VT_TV_CONDA_ENV=py${python_version} \
-            ./ci/python_build.sh
-            echo "::endgroup::"
-          done
+          VTK_DIR=${{ env.VTK_BUILD_DIR }} bash ./ci/python_build.sh
 
       - name: Test Python bindings (${{ join(matrix.host.python) }})
         run: |
-          cd ${{ github.workspace }}
-          chmod +x ./ci/python_test.sh
-
-          # Create vizualization output directory (required).
-          sudo mkdir -p ${{ env.VT_TV_OUTPUT_DIR }}/python_tests
-
-          PYTHON_VERSIONS=(`echo '${{ join(matrix.host.python) }}' | sed 's/,/\n/g'`)
-          for python_version in "${PYTHON_VERSIONS[@]}"
-          do
-            # Clear vizualization output directory
-            rm -rf ${{ env.VT_TV_OUTPUT_DIR }}/python_tests/*
-
-            echo "::group::Test Python Bindings (${python_version})"
-            CONDA_PATH=${{ env.CONDA_PATH }} \
-            VT_TV_CONDA_ENV=py${python_version} \
-            VT_TV_OUTPUT_DIR="${{ env.VT_TV_OUTPUT_DIR }}/python_tests" \
-            ./ci/python_test.sh
-            echo "::endgroup::"
-          done
+          VTK_DIR=${{ env.VTK_BUILD_DIR }} bash ./ci/python_test.sh
 
       - name: Collect artifacts
         run: |

diff --git a/ci/docker/build-and-test-ubuntu.dockerfile b/ci/docker/build-and-test-ubuntu.dockerfile
@@ -5,9 +5,6 @@ ARG VT_TV_TEST_PYTHON_BINDINGS=OFF
 
 FROM ${BASE_IMAGE} AS base
 
-# setup requirements for rendering tests (xvfb) + coverage report (lcov)
-RUN apt-get update && apt-get install -y
-
 COPY . /opt/src/vt-tv
 RUN mkdir -p /opt/build/vt-tv
 
@@ -30,6 +27,7 @@ RUN mkdir -p /opt/src/vt-tv/output/python_tests
 RUN bash /opt/src/vt-tv/ci/python_build.sh
 RUN bash /opt/src/vt-tv/ci/python_test.sh
 
+
 # Artifacts
 FROM scratch AS artifacts
 COPY --from=test-cpp /tmp/artifacts /tmp/artifacts

diff --git a/ci/docker/make-base-ubuntu.dockerfile b/ci/docker/make-base-ubuntu.dockerfile
@@ -8,19 +8,22 @@ FROM ${BASE_IMAGE} AS base
 
 # Arguments
 ARG VTK_VERSION=9.2.2
-ARG PYTHON_VERSION=3.8
+ARG PYTHON_VERSIONS=3.8,3.9,3.10,3.11,3.12
 ARG CC=gcc-11
 ARG CXX=g++-11
 ARG GCOV=gcov-11
 
-# Copy scripts
+# Copy setup scripts
 RUN mkdir -p /opt/scripts
 COPY ci/setup_mesa.sh /opt/scripts/setup_mesa.sh
-COPY ci/vtk_build.sh /opt/scripts/vtk_build.sh
+COPY ci/setup_mesa.sh /opt/scripts/setup_conda.sh
+COPY ci/setup_vtk.sh /opt/scripts/setup_vtk.sh
 
 ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update -y -q && \
-  apt-get install -y -q --no-install-recommends \
+
+# Setup common tools and compiler
+RUN apt update -y -q && \
+  apt install -y -q --no-install-recommends \
   ${CC} \
   ${CXX} \
   git \
@@ -49,43 +52,25 @@ RUN apt-get update -y -q && \
   xvfb \
   lcov
 
+# Setup MESA (opengl)
 RUN bash /opt/scripts/setup_mesa.sh
 RUN xvfb-run bash -c "glxinfo | grep 'OpenGL version'"
 
-RUN apt-get clean && rm -rf /var/lib/apt/lists/*
+# Setup conda and python environments
+RUN bash /opt/scripts/setup_conda.sh ${PYTHON_VERSIONS}
 
-# Share environment variables for use in images based on this.
+# Setup compiler using environment variables
 ENV CC=/usr/bin/$CC
 ENV CXX=/usr/bin/$CXX
 ENV GCOV=/usr/bin/$GCOV
-ENV VTK_DIR=/opt/build/vtk
-
-# Setup python 3.8 with conda
-
-# Download and install Miniconda
-RUN curl -LO https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
-    bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda && \
-    rm Miniconda3-latest-Linux-x86_64.sh
-
-# Update PATH so that conda and the installed packages are usable
-ENV PATH="/opt/conda/bin:${PATH}"
-
-# Create a new environment and install necessary packages
-RUN conda create -y -n deves python=${PYTHON_VERSION} && \
-    echo "source activate deves" > ~/.bashrc && \
-    /bin/bash -c ". /opt/conda/etc/profile.d/conda.sh && conda activate deves && pip install nanobind"
-
-# Set the environment to deves on container run
-ENV CONDA_DEFAULT_ENV=deves
-ENV CONDA_PREFIX=/opt/conda/envs/$CONDA_DEFAULT_ENV
-ENV PATH=$PATH:$CONDA_PREFIX/bin
-ENV CONDA_AUTO_UPDATE_CONDA=false
 
-# Clone VTK source
-RUN mkdir -p /opt/src/vtk
-RUN git clone --recursive --branch v${VTK_VERSION} https://gitlab.kitware.com/vtk/vtk.git /opt/src/vtk
+# Setup VTK
+RUN VTK_VERSION=${VTK_VERSION} \
+  VTK_DIR=${VTK_DIR} \
+  VTK_SRC_DIR=/opt/src/vtk \
+  bash /opt/scripts/setup_vtk.sh
 
-# Build VTK
-RUN VTK_DIR=${VTK_DIR} bash /opt/scripts/vtk_build.sh
+# Clean apt
+RUN apt clean && rm -rf /var/lib/apt/lists/*
 
 RUN echo "Base creation success"
diff --git a/ci/python_build.sh b/ci/python_build.sh
@@ -7,16 +7,22 @@ set -ex
 CURRENT_DIR="$(dirname -- "$(realpath -- "$0")")"
 PARENT_DIR="$(dirname "$CURRENT_DIR")"
 
-CONDA_PATH=${CONDA_PATH:-/opt/conda}
-VT_TV_CONDA_ENV=${VT_TV_CONDA_ENV:-deves}
 VT_TV_SRC_DIR=${VT_TV_SRC_DIR:-$PARENT_DIR}
 
-# Activate conda environment
-. ${CONDA_PATH}/etc/profile.d/conda.sh && conda activate $VT_TV_CONDA_ENV
+for env in $(conda env list | grep py | cut -d" " -f1); do if == "#" ; then continue; fi;
+    # Clear vizualization output directory
+    rm -rf $VT_TV_OUTPUT_DIR/python_tests/*
 
-# Build
-pip install PyYAML
-pip install $VT_TV_SRC_DIR
+    echo "::group::Test Python Bindings (${python_version})"
 
-# Deactivate conda environment
-conda deactivate
+    # Activate conda environment
+    . $CONDA_PREFIX/etc/profile.d/conda.sh && conda activate env
+
+    # Build
+    pip install PyYAML
+    pip install $VT_TV_SRC_DIR
+
+    conda deactivate
+
+    echo "::endgroup::"
+done
diff --git a/ci/python_test.sh b/ci/python_test.sh
@@ -7,19 +7,35 @@ set -ex
 CURRENT_DIR="$(dirname -- "$(realpath -- "$0")")"
 PARENT_DIR="$(dirname "$CURRENT_DIR")"
 
-CONDA_PATH=${CONDA_PATH:-/opt/conda}
-VT_TV_CONDA_ENV=${VT_TV_CONDA_ENV:-deves}
 VT_TV_SRC_DIR=${VT_TV_SRC_DIR:-$PARENT_DIR}
 VT_TV_OUTPUT_DIR=${VT_TV_OUTPUT_DIR:-"$VT_TV_SRC_DIR/output"}
 
-# Activate conda environment
-. ${CONDA_PATH}/etc/profile.d/conda.sh && conda activate $VT_TV_CONDA_ENV
+pushd $VT_TV_SRC_DIR
+chmod +x ./ci/python_test.sh
 
+# Create vizualization output directory (required).
+sudo mkdir -p $VT_TV_OUTPUT_DIR/python_tests
 
-# Run test
-if [[ $(uname -a) != *"Darwin"* ]]; then
-    # Start virtual display (Linux)
-    xvfb-run python $VT_TV_SRC_DIR/tests/test_bindings.py
-else
-    python $VT_TV_SRC_DIR/tests/test_bindings.py
-fi
+for env in $(conda env list | grep py | cut -d" " -f1); do if == "#" ; then continue; fi;
+    # Clear vizualization output directory
+    rm -rf $VT_TV_OUTPUT_DIR/python_tests/*
+
+    echo "::group::Test Python Bindings (${python_version})"
+
+    # Activate conda environment
+    . $CONDA_PREFIX/etc/profile.d/conda.sh && conda activate env
+
+    # Run test
+    if [[ $(uname -a) != *"Darwin"* ]]; then
+        # Start virtual display (Linux)
+        xvfb-run python $VT_TV_SRC_DIR/tests/test_bindings.py
+    else
+        python $VT_TV_SRC_DIR/tests/test_bindings.py
+    fi
+
+    conda deactivate
+
+    echo "::endgroup::"
+done
+
+popd
diff --git a/ci/setup_conda.sh b/ci/setup_conda.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# This script installs Conda and setup conda environments on the host machine for the given python versions
+# Example: setup_conda.sh 3.8,3.9,3.10,3.11,3.12
+
+CONDA_PREFIX=${CONDA_PREFIX:-"/opt/miniconda3"}
+PYTHON_VERSIONS=${1:-"3.8,3.9,3.10,3.11,3.12"}
+
+echo "::group::Install conda"
+mkdir -p ~/miniconda3
+if [[ $(uname -a) == *"Darwin"* ]]; then
+    if [[ $(arch) == 'arm64' ]]; then
+        curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o ~/miniconda.sh
+    else
+        curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o ~/miniconda.sh
+    fi
+else
+    curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -o ~/miniconda.sh
+fi
+bash ~/miniconda.sh -b -u -p $CONDA_PREFIX
+rm -rf ~/miniconda.sh
+
+$CONDA_PREFIX/bin/conda init bash
+$CONDA_PREFIX/bin/conda init zsh
+if [ -f ~/.zshrc ]; then
+. ~/.zshrc
+fi
+if [ -f ~/.profile ]; then
+. ~/.profile
+fi
+if [ -f ~/.bashrc ]; then
+. ~/.bashrc
+fi
+
+echo "Conda path: $(which conda)"
+echo "Conda version: $(conda --version)"
+
+echo "::endgroup::"
+
+versions=(`echo $PYTHON_VERSIONS | sed 's/,/\n/g'`)
+for python_version in "${versions[@]}"
+do
+    echo "::group::Create conda environment (py${python_version})"
+    conda create -y -n py${python_version} python=${python_version}
+
+    . $CONDA_PREFIX/etc/profile.d/conda.sh && conda activate py${python_version}
+    echo "Python version: $(python --version)"
+    pip install nanobind
+    conda deactivate
+    echo "::endgroup::"
+done
diff --git a/ci/setup_mesa.sh b/ci/setup_mesa.sh
@@ -2,10 +2,10 @@
 
 # This script installs Mesa libraries and utilities
 
-. /etc/lsb-release
 
-# Ubuntu 24.04 FIX.
+# FIX MESA driver (Ubuntu 24.04).
 # Error: MESA: error: ZINK: vkCreateInstance failed (VK_ERROR_INCOMPATIBLE_DRIVER)
+. /etc/lsb-release
 if [ "$DISTRIB_RELEASE" == "24.04" ]; then
     echo "FIX: Using latest MESA drivers (dev) for Ubuntu 24.04 to fix MESA errors !"
     add-apt-repository ppa:oibaf/graphics-drivers -y