From 83d3095e2c4272e9c8b98f1b1d0a4d7e4a99e7d9 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 3 Mar 2022 09:51:51 -0800 Subject: [PATCH 1/3] CI: Newer CCache for CUDA ccache on Ubuntu 18 and 20 is << v4.2 --- .github/workflows/cuda.yml | 4 +++- .github/workflows/dependencies/icc.sh | 4 +++- .github/workflows/dependencies/nvcc11.sh | 22 +++++++++++++++++++++- .github/workflows/dependencies/nvhpc.sh | 19 ++++++++++++++++++- .github/workflows/dependencies/pyfull.sh | 3 +++ 5 files changed, 48 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 8ea1d0660df..04c4ba76b06 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -40,12 +40,14 @@ jobs: - name: install openPMD-api run: | export CEI_SUDO="sudo" + export CEI_TMP="/tmp/cei" cmake-easyinstall --prefix=/usr/local \ git+https://github.com/openPMD/openPMD-api.git@0.14.3 \ -DopenPMD_USE_PYTHON=OFF \ -DBUILD_TESTING=OFF \ -DBUILD_EXAMPLES=OFF \ -DBUILD_CLI_TOOLS=OFF \ + -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ -DCMAKE_VERBOSE_MAKEFILE=ON - name: build WarpX run: | @@ -91,7 +93,7 @@ jobs: git clone https://github.com/AMReX-Codes/amrex.git ../amrex cd amrex && git checkout --detach 22.03 && cd - - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE -j 2 + make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2 build_nvhpc21-9-nvcc: name: NVHPC@21.9 NVCC/NVC++ Release [tests] diff --git a/.github/workflows/dependencies/icc.sh b/.github/workflows/dependencies/icc.sh index f8c2e916809..8e1b8201294 100755 --- a/.github/workflows/dependencies/icc.sh +++ b/.github/workflows/dependencies/icc.sh @@ -35,6 +35,7 @@ set -eu sudo curl -L -o /usr/local/bin/cmake-easyinstall https://git.io/JvLxY sudo chmod a+x /usr/local/bin/cmake-easyinstall export CEI_SUDO="sudo" +export CEI_TMP="/tmp/cei" # openPMD-api CXX=$(which icpc) CC=$(which icc) \ @@ -44,4 +45,5 @@ CXX=$(which icpc) CC=$(which icc) \ -DopenPMD_USE_PYTHON=OFF \ -DBUILD_TESTING=OFF \ -DBUILD_EXAMPLES=OFF \ - -DBUILD_CLI_TOOLS=OFF + -DBUILD_CLI_TOOLS=OFF \ + -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) diff --git a/.github/workflows/dependencies/nvcc11.sh b/.github/workflows/dependencies/nvcc11.sh index 1576879eb40..ed626c91766 100755 --- a/.github/workflows/dependencies/nvcc11.sh +++ b/.github/workflows/dependencies/nvcc11.sh @@ -11,10 +11,11 @@ sudo apt-get -qqq update sudo apt-get install -y \ build-essential \ ca-certificates \ - ccache \ cmake \ gnupg \ + libhiredis-dev \ libopenmpi-dev \ + libzstd-dev \ ninja-build \ openmpi-bin \ pkg-config \ @@ -37,7 +38,26 @@ sudo apt-get install -y \ libcurand-dev-11-0 sudo ln -s cuda-11.0 /usr/local/cuda +# if we run out of temporary storage in CI: +#du -sh /usr/local/cuda-11.0 +#echo "+++ REDUCING CUDA Toolkit install size +++" +#sudo rm -rf /usr/local/cuda-11.0/targets/x86_64-linux/lib/libcu{fft,pti,rand}_static.a +#sudo rm -rf /usr/local/cuda-11.0/targets/x86_64-linux/lib/libnvperf_host_static.a +#du -sh /usr/local/cuda-11.0/ +#df -h + # cmake-easyinstall # sudo curl -L -o /usr/local/bin/cmake-easyinstall https://git.io/JvLxY sudo chmod a+x /usr/local/bin/cmake-easyinstall +export CEI_SUDO="sudo" +export CEI_TMP="/tmp/cei" + +# ccache 4.2+ +# +cmake-easyinstall --prefix=/usr/local \ + git+https://github.com/ccache/ccache.git@v4.6 \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_DOCUMENTATION=OFF \ + -DENABLE_TESTING=OFF \ + -DWARNINGS_AS_ERRORS=OFF diff --git a/.github/workflows/dependencies/nvhpc.sh b/.github/workflows/dependencies/nvhpc.sh index 684b2440e5c..6b594bbdbec 100755 --- a/.github/workflows/dependencies/nvhpc.sh +++ b/.github/workflows/dependencies/nvhpc.sh @@ -10,10 +10,11 @@ sudo apt-get -qqq update sudo apt-get install -y \ build-essential \ ca-certificates \ - ccache \ cmake \ environment-modules \ gnupg \ + libhiredis-dev \ + libzstd-dev \ ninja-build \ pkg-config \ wget @@ -29,3 +30,19 @@ rm -rf ./nvhpc-21-9_21.9_amd64.deb ./nvhpc-2021_21.9_amd64.deb # activation via: # source /etc/profile.d/modules.sh # module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/21.9 + +# cmake-easyinstall +# +sudo curl -L -o /usr/local/bin/cmake-easyinstall https://git.io/JvLxY +sudo chmod a+x /usr/local/bin/cmake-easyinstall +export CEI_SUDO="sudo" +export CEI_TMP="/tmp/cei" + +# ccache 4.2+ +# +cmake-easyinstall --prefix=/usr/local \ + git+https://github.com/ccache/ccache.git@v4.6 \ + -DCMAKE_BUILD_TYPE=Release \ + -DENABLE_DOCUMENTATION=OFF \ + -DENABLE_TESTING=OFF \ + -DWARNINGS_AS_ERRORS=OFF diff --git a/.github/workflows/dependencies/pyfull.sh b/.github/workflows/dependencies/pyfull.sh index 8d5de8e04ce..fce778b9ca5 100755 --- a/.github/workflows/dependencies/pyfull.sh +++ b/.github/workflows/dependencies/pyfull.sh @@ -35,6 +35,7 @@ sudo apt-get install -y \ sudo curl -L -o /usr/local/bin/cmake-easyinstall https://git.io/JvLxY sudo chmod a+x /usr/local/bin/cmake-easyinstall export CEI_SUDO="sudo" +export CEI_TMP="/tmp/cei" # BLAS++ & LAPACK++ cmake-easyinstall \ @@ -42,6 +43,7 @@ cmake-easyinstall \ git+https://bitbucket.org/icl/blaspp.git \ -Duse_openmp=OFF \ -Dbuild_tests=OFF \ + -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ -DCMAKE_VERBOSE_MAKEFILE=ON cmake-easyinstall \ @@ -49,4 +51,5 @@ cmake-easyinstall \ git+https://bitbucket.org/icl/lapackpp.git \ -Duse_cmake_find_lapack=ON \ -Dbuild_tests=OFF \ + -DCMAKE_CXX_COMPILER_LAUNCHER=$(which ccache) \ -DCMAKE_VERBOSE_MAKEFILE=ON From a008caf5c2c3aebc53666e3933e84de423c72811 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 3 Mar 2022 19:42:40 -0800 Subject: [PATCH 2/3] pip: also use ccache Modern `pip` does not build in a random temporary dir anymore but in `$PWD/build`. With absolute paths established, we can use `ccache` to speed up this big build as well. --- .github/workflows/ubuntu.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 2be792f3cb7..318d19f55db 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -91,6 +91,16 @@ jobs: - name: install dependencies run: | .github/workflows/dependencies/pyfull.sh + - name: CCache Cache + uses: actions/cache@v2 + # - once stored under a key, they become immutable (even if local cache path content changes) + # - for a refresh the key has to change, e.g., hash of a tracked file in the key + with: + path: ~/.ccache + key: ccache-openmp-pyfull-${{ hashFiles('.github/workflows/ubuntu.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} + restore-keys: | + ccache-openmp-pyfull-${{ hashFiles('.github/workflows/ubuntu.yml') }}- + ccache-openmp-pyfull- - name: build WarpX run: | python3 -m pip install --upgrade pip setuptools wheel From 5880e95973ae3bd3f079ca5d21dd25f2439e43fb Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 3 Mar 2022 20:53:54 -0800 Subject: [PATCH 3/3] macOS: Fix `cache_dir` Not in $HOME. --- .github/workflows/macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 2c8889aa40b..24484e612c8 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -40,7 +40,7 @@ jobs: # - once stored under a key, they become immutable (even if local cache path content changes) # - for a refresh the key has to change, e.g., hash of a tracked file in the key with: - path: ~/.ccache + path: /Users/runner/Library/Caches/ccache key: ccache-macos-appleclang-${{ hashFiles('.github/workflows/macos.yml') }}-${{ hashFiles('cmake/dependencies/AMReX.cmake') }} restore-keys: | ccache-macos-appleclang-${{ hashFiles('.github/workflows/macos.yml') }}-