Azure CI: Cache (ECP-WarpX#2615)

* run_test: WARPX_CI_TMP

Allow to use a fixed instead of a unique temporary directory.
This will help `ccache` to cache compliation, because absolute
paths do not change anymore between builds.

* Azure CI: Cache

Try to use caching for as much as possible on Azure.
This might help to reuse AMReX objects between our weekly updates.
It might also be just way too large and get evicted quickly.

.azure-pipelines.yml

@@ -10,13 +10,16 @@ pr:
 jobs:
 - job:
   variables:
-    WARPX_CI_NUM_MAKE_JOBS: 2
+    BLASPP_HOME: '/usr/local'
+    CEI_SUDO: 'sudo'
+    CEI_TMP: '/tmp/cei'
+    FFTW_HOME: '/usr'
+    LAPACKPP_HOME: '/usr/local'
+    OMP_NUM_THREADS: 1
     WARPX_CI_CCACHE: 'TRUE'
+    WARPX_CI_NUM_MAKE_JOBS: 2
     WARPX_CI_OPENPMD: 'TRUE'
-    FFTW_HOME: '/usr/'
-    BLASPP_HOME: '/usr/local/'
-    LAPACKPP_HOME: '/usr/local/'
-    OMP_NUM_THREADS: 1
+    WARPX_CI_TMP: '/tmp/ci'
 
   strategy:
     matrix:
@@ -42,12 +45,39 @@ jobs:
   timeoutInMinutes: 90
 
   steps:
-  - script: |
+  # set up caches:
+  # - once stored under a key, they become immutable (even if cache content changes)
+  # - for a refresh the key has to change, e.g., hash of a tracked file in the key
+  - task: Cache@2
+    inputs:
+      key: 'Ccache | "$(System.JobName)" | cmake/dependencies/AMReX.cmake | run_test.sh'
+      restoreKeys: |
+         Ccache | "$(System.JobName)" | cmake/dependencies/AMReX.cmake | run_test.sh
+         Ccache | "$(System.JobName)" | cmake/dependencies/AMReX.cmake
+         Ccache | "$(System.JobName)"
+         Ccache
+      path: /home/vsts/.ccache
+      cacheHitVar: CCACHE_CACHE_RESTORED
+    displayName: Cache Ccache Objects
+
+  - task: Cache@2
+    inputs:
+      key: 'Python3 | "$(System.JobName)" | run_test.sh'
+      restoreKeys: |
+         Python3 | "$(System.JobName)" | run_test.sh
+         Python3 | "$(System.JobName)"
+         Python3
+      path: /home/vsts/.local/lib/python3.8
+      cacheHitVar: PYTHON38_CACHE_RESTORED
+    displayName: Cache Python Libraries
+
+  - bash: |
       cat /proc/cpuinfo | grep "model name" | sort -u
-      sudo apt update
+      df -h
       sudo apt install -y ccache gcc gfortran g++ openmpi-bin libopenmpi-dev \
         libfftw3-dev libfftw3-mpi-dev libhdf5-openmpi-dev pkg-config make \
         python3 python3-pip python3-venv python3-setuptools libblas-dev liblapack-dev
+      ccache --set-config=max_size=10.0G
       sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 2
       sudo update-alternatives --set python /usr/bin/python3
       python -m pip install --upgrade pip
@@ -57,22 +87,31 @@ jobs:
       python -m pipx ensurepath
       export PATH="$HOME/.local/bin:$PATH"
       python -m pip install --upgrade matplotlib mpi4py numpy scipy yt
-      export CEI_SUDO="sudo"
       sudo curl -L -o /usr/local/bin/cmake-easyinstall https://git.io/JvLxY
       sudo chmod a+x /usr/local/bin/cmake-easyinstall
       if [ "${WARPX_CI_OPENPMD:-FALSE}" == "TRUE" ]; then
-        cmake-easyinstall --prefix=/usr/local git+https://github.com/openPMD/[email protected] \
+        cmake-easyinstall --prefix=/usr/local                   \
+          git+https://github.com/openPMD/[email protected] \
+          -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache)         \
+          -DCMAKE_VERBOSE_MAKEFILE=ON                           \
           -DopenPMD_USE_PYTHON=OFF -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI_TOOLS=OFF
         python -m pip install --upgrade openpmd-api
       fi
       if [[ "${WARPX_CI_RZ_OR_NOMPI:-FALSE}" == "TRUE" || "${WARPX_CI_PYTHON_MAIN:-FALSE}" == "TRUE" ]]; then
         cmake-easyinstall --prefix=/usr/local git+https://bitbucket.org/icl/blaspp.git \
+          -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \
           -Duse_openmp=OFF -Dbuild_tests=OFF -DCMAKE_VERBOSE_MAKEFILE=ON
         cmake-easyinstall --prefix=/usr/local git+https://bitbucket.org/icl/lapackpp.git \
+          -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \
           -Duse_cmake_find_lapack=ON -Dbuild_tests=OFF -DCMAKE_VERBOSE_MAKEFILE=ON
       fi
+      rm -rf ${CEI_TMP}
+      df -h
     displayName: 'Install dependencies'
 
-  - script: |
+  - bash: |
+      df -h
       ./run_test.sh
+      rm -rf ${WARPX_CI_TMP}
+      df -h
     displayName: 'Build & test'

run_test.sh

@@ -32,14 +32,19 @@ tests_run=${tests_arg:+--tests=${tests_arg}}
 # Remove contents and link to a previous test directory (intentionally two arguments)
 rm -rf test_dir/* test_dir
 # Create a temporary test directory
-tmp_dir=$(mktemp --help >/dev/null 2>&1 && mktemp -d -t ci-XXXXXXXXXX || mktemp -d "${TMPDIR:-/tmp}"/ci-XXXXXXXXXX)
-if [ $? -ne 0 ]; then
-    echo "Cannot create temporary directory"
-    exit 1
+if [ -z "${WARPX_CI_TMP}" ]; then
+    tmp_dir=$(mktemp --help >/dev/null 2>&1 && mktemp -d -t ci-XXXXXXXXXX || mktemp -d "${TMPDIR:-/tmp}"/ci-XXXXXXXXXX)
+    if [ $? -ne 0 ]; then
+        echo "Cannot create temporary directory"
+        exit 1
+    fi
+else
+    tmp_dir=${WARPX_CI_TMP}
 fi
 
 # Copy WarpX into current test directory
-mkdir ${tmp_dir}/warpx
+rm -rf ${tmp_dir}/warpx
+mkdir -p ${tmp_dir}/warpx
 cp -r ./* ${tmp_dir}/warpx
 
 # Link the test directory