From 283f0b8fa4499da2368c87265d74a699d07182ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20Dutheillet-Lamonth=C3=A9zie?= Date: Fri, 6 Sep 2024 12:42:47 +0200 Subject: [PATCH] #97: CI refactoring to remove duplicate commands --- .github/workflows/build-and-test-macos.yml | 92 ++-------------------- ci/docker/build-and-test-ubuntu.dockerfile | 4 +- ci/docker/make-base-ubuntu.dockerfile | 53 +++++-------- ci/python_build.sh | 24 +++--- ci/python_test.sh | 38 ++++++--- ci/setup_conda.sh | 51 ++++++++++++ ci/setup_mesa.sh | 4 +- ci/{vtk_build.sh => setup_vtk.sh} | 7 +- 8 files changed, 129 insertions(+), 144 deletions(-) create mode 100644 ci/setup_conda.sh rename ci/{vtk_build.sh => setup_vtk.sh} (62%) diff --git a/.github/workflows/build-and-test-macos.yml b/.github/workflows/build-and-test-macos.yml index 60e16d260..6aa655a26 100644 --- a/.github/workflows/build-and-test-macos.yml +++ b/.github/workflows/build-and-test-macos.yml @@ -46,7 +46,7 @@ jobs: VT_TV_OUTPUT_DIR: ${{ github.workspace }}/output VT_TV_TESTS_OUTPUT_DIR: ${{ github.workspace }}/output/tests VT_TV_ARTIFACTS_DIR: /tmp/artifacts - CONDA_PATH: /opt/miniconda # ~/miniconda3 + CONDA_PREFIX: /opt/miniconda3 CC: ~ CXX: ~ GCOV: ~ @@ -117,7 +117,7 @@ jobs: path: | ${{ env.VTK_SRC_DIR }} ${{ env.VTK_BUILD_DIR }} - ${{ env.CONDA_PATH }} + ${{ env.CONDA_PREFIX }} ~/.zshrc ~/.bash_profile key: ${{ env.CACHE_KEY }} @@ -126,59 +126,17 @@ jobs: - name: Setup Miniconda3 if: ${{steps.base-cache.outputs.cache-hit != 'true'}} run: | - mkdir -p ~/miniconda3 - if [[ $(uname -a) == *"Darwin"* ]]; then - if [[ $(arch) == 'arm64' ]]; then - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o ~/miniconda.sh - else - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o ~/miniconda.sh - fi - else - curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -o ~/miniconda.sh - fi - bash ~/miniconda.sh -b -u -p ${{ env.CONDA_PATH }} - rm -rf ~/miniconda.sh - - ${{ env.CONDA_PATH }}/bin/conda init bash - ${{ env.CONDA_PATH }}/bin/conda init zsh - if [ -f ~/.zshrc ]; then - . ~/.zshrc - fi - if [ -f ~/.profile ]; then - . ~/.profile - fi - if [ -f ~/.bashrc ]; then - . ~/.bashrc - fi - - echo "Conda path: $(which conda)" - echo "Conda version: $(conda --version)" - - PYTHON_VERSIONS=(`echo '${{ join(matrix.host.python) }}' | sed 's/,/\n/g'`) - for python_version in "${PYTHON_VERSIONS[@]}" - do - echo "::group::Create conda environment (py${python_version})" - conda create -y -n py${python_version} python=${python_version} - - . ${{ env.CONDA_PATH }}/etc/profile.d/conda.sh && conda activate py${python_version} - echo "Python version: $(python --version)" - pip install nanobind - conda deactivate - echo "::endgroup::" - done + sudo CONDA_PREFIX=${{ env.CONDA_PREFIX }} bash ./ci/setup_conda.sh ${{ join(matrix.host.python) }} - name: Setup VTK ${{ matrix.host.vtk }} if: ${{steps.base-cache.outputs.cache-hit != 'true'}} run: | mkdir -p ${{ env.VTK_SRC_DIR }} git clone --recursive --branch v${{ matrix.host.vtk }} https://gitlab.kitware.com/vtk/vtk.git ${{ env.VTK_SRC_DIR }} - VTK_DIR=${{ env.VTK_BUILD_DIR }} VTK_SRC_DIR=${{ env.VTK_SRC_DIR }} bash ${{ github.workspace }}/ci/vtk_build.sh + VTK_DIR=${{ env.VTK_BUILD_DIR }} VTK_SRC_DIR=${{ env.VTK_SRC_DIR }} bash ./ci/vtk_build.sh - name: Build run: | - cd ${{ github.workspace }} - chmod +x ./ci/build.sh - mkdir -p ${{ env.VT_TV_BUILD_DIR }} CC="${{ env.CC }}" \ @@ -191,56 +149,22 @@ jobs: GCOV="${{ env.GCOV }}" \ VT_TV_PYTHON_BINDINGS_ENABLED=OFF \ VT_TV_WERROR_ENABLED=ON \ - ./build.sh + bash ./build.sh - name: Test run: | - cd ${{ github.workspace }} - chmod +x ./ci/test.sh - VTK_DIR=${{ env.VTK_BUILD_DIR }} \ VT_TV_COVERAGE_ENABLED=${{ env.VT_TV_COVERAGE_ENABLED }} \ VT_TV_OUTPUT_DIR="${{ env.VT_TV_OUTPUT_DIR }}" \ - ./ci/test.sh + bash ./ci/test.sh - name: Build Python package (${{ join(matrix.host.python, ', ') }}) run: | - cd ${{ github.workspace }} - chmod +x ./ci/python_build.sh - - PYTHON_VERSIONS=(`echo '${{ join(matrix.host.python) }}' | sed 's/,/\n/g'`) - export VTK_DIR=${{ env.VTK_BUILD_DIR }} - for python_version in "${PYTHON_VERSIONS[@]}" - do - echo "::group::Build Python package (${python_version})" - VTK_DIR=${{ env.VTK_BUILD_DIR }} \ - CONDA_PATH=${{ env.CONDA_PATH }} \ - VT_TV_CONDA_ENV=py${python_version} \ - ./ci/python_build.sh - echo "::endgroup::" - done + VTK_DIR=${{ env.VTK_BUILD_DIR }} bash ./ci/python_build.sh - name: Test Python bindings (${{ join(matrix.host.python) }}) run: | - cd ${{ github.workspace }} - chmod +x ./ci/python_test.sh - - # Create vizualization output directory (required). - sudo mkdir -p ${{ env.VT_TV_OUTPUT_DIR }}/python_tests - - PYTHON_VERSIONS=(`echo '${{ join(matrix.host.python) }}' | sed 's/,/\n/g'`) - for python_version in "${PYTHON_VERSIONS[@]}" - do - # Clear vizualization output directory - rm -rf ${{ env.VT_TV_OUTPUT_DIR }}/python_tests/* - - echo "::group::Test Python Bindings (${python_version})" - CONDA_PATH=${{ env.CONDA_PATH }} \ - VT_TV_CONDA_ENV=py${python_version} \ - VT_TV_OUTPUT_DIR="${{ env.VT_TV_OUTPUT_DIR }}/python_tests" \ - ./ci/python_test.sh - echo "::endgroup::" - done + VTK_DIR=${{ env.VTK_BUILD_DIR }} bash ./ci/python_test.sh - name: Collect artifacts run: | diff --git a/ci/docker/build-and-test-ubuntu.dockerfile b/ci/docker/build-and-test-ubuntu.dockerfile index 0ce9e664c..465735c51 100644 --- a/ci/docker/build-and-test-ubuntu.dockerfile +++ b/ci/docker/build-and-test-ubuntu.dockerfile @@ -5,9 +5,6 @@ ARG VT_TV_TEST_PYTHON_BINDINGS=OFF FROM ${BASE_IMAGE} AS base -# setup requirements for rendering tests (xvfb) + coverage report (lcov) -RUN apt-get update && apt-get install -y - COPY . /opt/src/vt-tv RUN mkdir -p /opt/build/vt-tv @@ -30,6 +27,7 @@ RUN mkdir -p /opt/src/vt-tv/output/python_tests RUN bash /opt/src/vt-tv/ci/python_build.sh RUN bash /opt/src/vt-tv/ci/python_test.sh + # Artifacts FROM scratch AS artifacts COPY --from=test-cpp /tmp/artifacts /tmp/artifacts diff --git a/ci/docker/make-base-ubuntu.dockerfile b/ci/docker/make-base-ubuntu.dockerfile index 3591f6e86..6e03bd853 100644 --- a/ci/docker/make-base-ubuntu.dockerfile +++ b/ci/docker/make-base-ubuntu.dockerfile @@ -8,19 +8,22 @@ FROM ${BASE_IMAGE} AS base # Arguments ARG VTK_VERSION=9.2.2 -ARG PYTHON_VERSION=3.8 +ARG PYTHON_VERSIONS=3.8,3.9,3.10,3.11,3.12 ARG CC=gcc-11 ARG CXX=g++-11 ARG GCOV=gcov-11 -# Copy scripts +# Copy setup scripts RUN mkdir -p /opt/scripts COPY ci/setup_mesa.sh /opt/scripts/setup_mesa.sh -COPY ci/vtk_build.sh /opt/scripts/vtk_build.sh +COPY ci/setup_mesa.sh /opt/scripts/setup_conda.sh +COPY ci/setup_vtk.sh /opt/scripts/setup_vtk.sh ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update -y -q && \ - apt-get install -y -q --no-install-recommends \ + +# Setup common tools and compiler +RUN apt update -y -q && \ + apt install -y -q --no-install-recommends \ ${CC} \ ${CXX} \ git \ @@ -49,43 +52,25 @@ RUN apt-get update -y -q && \ xvfb \ lcov +# Setup MESA (opengl) RUN bash /opt/scripts/setup_mesa.sh RUN xvfb-run bash -c "glxinfo | grep 'OpenGL version'" -RUN apt-get clean && rm -rf /var/lib/apt/lists/* +# Setup conda and python environments +RUN bash /opt/scripts/setup_conda.sh ${PYTHON_VERSIONS} -# Share environment variables for use in images based on this. +# Setup compiler using environment variables ENV CC=/usr/bin/$CC ENV CXX=/usr/bin/$CXX ENV GCOV=/usr/bin/$GCOV -ENV VTK_DIR=/opt/build/vtk - -# Setup python 3.8 with conda - -# Download and install Miniconda -RUN curl -LO https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ - bash Miniconda3-latest-Linux-x86_64.sh -b -p /opt/conda && \ - rm Miniconda3-latest-Linux-x86_64.sh - -# Update PATH so that conda and the installed packages are usable -ENV PATH="/opt/conda/bin:${PATH}" - -# Create a new environment and install necessary packages -RUN conda create -y -n deves python=${PYTHON_VERSION} && \ - echo "source activate deves" > ~/.bashrc && \ - /bin/bash -c ". /opt/conda/etc/profile.d/conda.sh && conda activate deves && pip install nanobind" - -# Set the environment to deves on container run -ENV CONDA_DEFAULT_ENV=deves -ENV CONDA_PREFIX=/opt/conda/envs/$CONDA_DEFAULT_ENV -ENV PATH=$PATH:$CONDA_PREFIX/bin -ENV CONDA_AUTO_UPDATE_CONDA=false -# Clone VTK source -RUN mkdir -p /opt/src/vtk -RUN git clone --recursive --branch v${VTK_VERSION} https://gitlab.kitware.com/vtk/vtk.git /opt/src/vtk +# Setup VTK +RUN VTK_VERSION=${VTK_VERSION} \ + VTK_DIR=${VTK_DIR} \ + VTK_SRC_DIR=/opt/src/vtk \ + bash /opt/scripts/setup_vtk.sh -# Build VTK -RUN VTK_DIR=${VTK_DIR} bash /opt/scripts/vtk_build.sh +# Clean apt +RUN apt clean && rm -rf /var/lib/apt/lists/* RUN echo "Base creation success" diff --git a/ci/python_build.sh b/ci/python_build.sh index 107720f4f..7b138eda6 100644 --- a/ci/python_build.sh +++ b/ci/python_build.sh @@ -7,16 +7,22 @@ set -ex CURRENT_DIR="$(dirname -- "$(realpath -- "$0")")" PARENT_DIR="$(dirname "$CURRENT_DIR")" -CONDA_PATH=${CONDA_PATH:-/opt/conda} -VT_TV_CONDA_ENV=${VT_TV_CONDA_ENV:-deves} VT_TV_SRC_DIR=${VT_TV_SRC_DIR:-$PARENT_DIR} -# Activate conda environment -. ${CONDA_PATH}/etc/profile.d/conda.sh && conda activate $VT_TV_CONDA_ENV +for env in $(conda env list | grep py | cut -d" " -f1); do if == "#" ; then continue; fi; + # Clear vizualization output directory + rm -rf $VT_TV_OUTPUT_DIR/python_tests/* -# Build -pip install PyYAML -pip install $VT_TV_SRC_DIR + echo "::group::Test Python Bindings (${python_version})" -# Deactivate conda environment -conda deactivate \ No newline at end of file + # Activate conda environment + . $CONDA_PREFIX/etc/profile.d/conda.sh && conda activate env + + # Build + pip install PyYAML + pip install $VT_TV_SRC_DIR + + conda deactivate + + echo "::endgroup::" +done \ No newline at end of file diff --git a/ci/python_test.sh b/ci/python_test.sh index f28c92951..ba722daa6 100644 --- a/ci/python_test.sh +++ b/ci/python_test.sh @@ -7,19 +7,35 @@ set -ex CURRENT_DIR="$(dirname -- "$(realpath -- "$0")")" PARENT_DIR="$(dirname "$CURRENT_DIR")" -CONDA_PATH=${CONDA_PATH:-/opt/conda} -VT_TV_CONDA_ENV=${VT_TV_CONDA_ENV:-deves} VT_TV_SRC_DIR=${VT_TV_SRC_DIR:-$PARENT_DIR} VT_TV_OUTPUT_DIR=${VT_TV_OUTPUT_DIR:-"$VT_TV_SRC_DIR/output"} -# Activate conda environment -. ${CONDA_PATH}/etc/profile.d/conda.sh && conda activate $VT_TV_CONDA_ENV +pushd $VT_TV_SRC_DIR +chmod +x ./ci/python_test.sh +# Create vizualization output directory (required). +sudo mkdir -p $VT_TV_OUTPUT_DIR/python_tests -# Run test -if [[ $(uname -a) != *"Darwin"* ]]; then - # Start virtual display (Linux) - xvfb-run python $VT_TV_SRC_DIR/tests/test_bindings.py -else - python $VT_TV_SRC_DIR/tests/test_bindings.py -fi +for env in $(conda env list | grep py | cut -d" " -f1); do if == "#" ; then continue; fi; + # Clear vizualization output directory + rm -rf $VT_TV_OUTPUT_DIR/python_tests/* + + echo "::group::Test Python Bindings (${python_version})" + + # Activate conda environment + . $CONDA_PREFIX/etc/profile.d/conda.sh && conda activate env + + # Run test + if [[ $(uname -a) != *"Darwin"* ]]; then + # Start virtual display (Linux) + xvfb-run python $VT_TV_SRC_DIR/tests/test_bindings.py + else + python $VT_TV_SRC_DIR/tests/test_bindings.py + fi + + conda deactivate + + echo "::endgroup::" +done + +popd \ No newline at end of file diff --git a/ci/setup_conda.sh b/ci/setup_conda.sh new file mode 100644 index 000000000..528aea6ea --- /dev/null +++ b/ci/setup_conda.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# This script installs Conda and setup conda environments on the host machine for the given python versions +# Example: setup_conda.sh 3.8,3.9,3.10,3.11,3.12 + +CONDA_PREFIX=${CONDA_PREFIX:-"/opt/miniconda3"} +PYTHON_VERSIONS=${1:-"3.8,3.9,3.10,3.11,3.12"} + +echo "::group::Install conda" +mkdir -p ~/miniconda3 +if [[ $(uname -a) == *"Darwin"* ]]; then + if [[ $(arch) == 'arm64' ]]; then + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-arm64.sh -o ~/miniconda.sh + else + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -o ~/miniconda.sh + fi +else + curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -o ~/miniconda.sh +fi +bash ~/miniconda.sh -b -u -p $CONDA_PREFIX +rm -rf ~/miniconda.sh + +$CONDA_PREFIX/bin/conda init bash +$CONDA_PREFIX/bin/conda init zsh +if [ -f ~/.zshrc ]; then +. ~/.zshrc +fi +if [ -f ~/.profile ]; then +. ~/.profile +fi +if [ -f ~/.bashrc ]; then +. ~/.bashrc +fi + +echo "Conda path: $(which conda)" +echo "Conda version: $(conda --version)" + +echo "::endgroup::" + +versions=(`echo $PYTHON_VERSIONS | sed 's/,/\n/g'`) +for python_version in "${versions[@]}" +do + echo "::group::Create conda environment (py${python_version})" + conda create -y -n py${python_version} python=${python_version} + + . $CONDA_PREFIX/etc/profile.d/conda.sh && conda activate py${python_version} + echo "Python version: $(python --version)" + pip install nanobind + conda deactivate + echo "::endgroup::" +done diff --git a/ci/setup_mesa.sh b/ci/setup_mesa.sh index 88e8012da..585ce3c15 100644 --- a/ci/setup_mesa.sh +++ b/ci/setup_mesa.sh @@ -2,10 +2,10 @@ # This script installs Mesa libraries and utilities -. /etc/lsb-release -# Ubuntu 24.04 FIX. +# FIX MESA driver (Ubuntu 24.04). # Error: MESA: error: ZINK: vkCreateInstance failed (VK_ERROR_INCOMPATIBLE_DRIVER) +. /etc/lsb-release if [ "$DISTRIB_RELEASE" == "24.04" ]; then echo "FIX: Using latest MESA drivers (dev) for Ubuntu 24.04 to fix MESA errors !" add-apt-repository ppa:oibaf/graphics-drivers -y diff --git a/ci/vtk_build.sh b/ci/setup_vtk.sh similarity index 62% rename from ci/vtk_build.sh rename to ci/setup_vtk.sh index c9e959cfc..8072b2e85 100755 --- a/ci/vtk_build.sh +++ b/ci/setup_vtk.sh @@ -6,6 +6,10 @@ set -ex VTK_SRC_DIR=${VTK_SRC_DIR:-"/opt/src/vtk"} VTK_DIR=${VTK_DIR:-"/opt/build/vtk"} +VTK_VERSION=${VTK_VERSION:-"9.3.1"} + +echo "Setup VTK $VTK_VERSION from source..." +git clone --recursive --branch v${VTK_VERSION} https://gitlab.kitware.com/vtk/vtk.git /opt/src/vtk mkdir -p $VTK_DIR pushd $VTK_DIR @@ -18,5 +22,6 @@ cmake \ cmake --build "$VTK_DIR" -j$(nproc) echo "VTK build success" - popd + +echo "VTK $VTK_VERSION has been installed successfully."