Merge with upstream development

.codespell-ignore-words

@@ -0,0 +1,7 @@
+Blocs
+BOUN
+inout
+parms
+pres
+rime
+wth

.codespellrc

@@ -0,0 +1,4 @@
+[codespell]
+skip = .git,*.ipynb,*.bib,*.ps,*.patch,*~,CHANGES,*/Submodules,*/tmp_build_dir,Cell_D_*
+ignore-words = .codespell-ignore-words
+uri-ignore-words-list = *

.github/dependabot.yml

@@ -0,0 +1,9 @@
+version: 2
+
+updates:
+  - package-ecosystem: gitsubmodule
+    schedule:
+        interval: "monthly"
+    directory: /
+    allow:
+      - dependency-name: "Submodules/AMReX"

.github/workflows/c-linter.yml

@@ -3,9 +3,9 @@ name: cpp-linter
 on: [pull_request]
 jobs:
   cpp-linter:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
 
@@ -30,7 +30,7 @@ jobs:
         run: python3 Submodules/cpp-linter-action/run_on_changed_files.py ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} -header-filter=Source/ -ignore-files="AMReX|GoogleTest" -run-linter
 
       - name: Archive clang tidy report
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v4
         with:
           name: clang-tidy-report
           path: clang-tidy-report.txt

.github/workflows/ci.yml

@@ -31,7 +31,7 @@ jobs:
 
     steps:
 
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
 
@@ -46,12 +46,24 @@ jobs:
         # Install MPI
         ${{matrix.install_deps}}
 
+    - name: Install CCache
+      run: Submodules/AMReX/.github/workflows/dependencies/dependencies_ccache.sh
+
+    - name: Set Up Cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/ccache
+        key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+        restore-keys: |
+             ccache-${{ github.workflow }}-${{ github.job }}-git-
+
     - name: Configure CMake
       run: |
         cmake \
           -B${{runner.workspace}}/ERF/build-${{matrix.os}} \
           -DCMAKE_INSTALL_PREFIX:PATH=${{runner.workspace}}/ERF/install \
           -DCMAKE_BUILD_TYPE:STRING=Debug \
+          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
           -DERF_DIM:STRING=3 \
           -DERF_ENABLE_MPI:BOOL=ON \
           -DERF_ENABLE_TESTS:BOOL=ON \
@@ -63,7 +75,25 @@ jobs:
 
     - name: Build
       run: |
-        cmake --build ${{runner.workspace}}/ERF/build-${{matrix.os}} --parallel ${{env.NPROCS}};
+        export CCACHE_COMPRESS=1
+        export CCACHE_COMPRESSLEVEL=10
+        export CCACHE_MAXSIZE=600M
+        ccache -z
+
+        cmake --build ${{runner.workspace}}/ERF/build-${{matrix.os}} --parallel ${{env.NPROCS}} \
+        2>&1 | tee -a ${{runner.workspace}}/build-output.txt;
+
+        ccache -s
+        du -hs ~/.cache/ccache
+
+    - name: Report
+      run: |
+        egrep "warning:|error:" ${{runner.workspace}}/build-output.txt \
+          | egrep -v "Submodules/amrex" | egrep -v "lto-wrapper: warning:" | sort | uniq \
+          | awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > ${{runner.workspace}}/build-output-warnings.txt
+        cat ${{runner.workspace}}/build-output-warnings.txt
+        export return=$(tail -n 1 ${{runner.workspace}}/build-output-warnings.txt | awk '{print $2}')
+        exit ${return}
 
     - name: Regression Tests
       run: |
@@ -81,14 +111,14 @@ jobs:
     #     cd ..
     #     gcovr -g -k -r . --xml regressioncov.html  # -v
     # - name: Success artifacts
-    #   uses: actions/upload-artifact@v3
+    #   uses: actions/upload-artifact@v4
     #   if: success()
     #   with:
     #     name: build-and-test
     #     path: |
     #       ${{runner.workspace}}/ERF/regressioncov.html
     # - name: Failing test artifacts
-    #   uses: actions/upload-artifact@v3
+    #   uses: actions/upload-artifact@v4
     #   if: failure()
     #   with:
     #     name: build-and-test

.github/workflows/cleanup-cache-postpr.yml

@@ -0,0 +1,43 @@
+name: CleanUpCachePostPR
+
+on:
+  workflow_run:
+    workflows: [PostPR]
+    types:
+      - completed
+
+jobs:
+  CleanUpCcacheCachePostPR:
+    name: Clean Up Ccache Cache Post PR
+    runs-on: ubuntu-22.04
+    permissions:
+      actions: write
+      contents: read
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Clean up ccache
+        run: |
+          gh extension install actions/gh-actions-cache
+
+          REPO=${{ github.repository }}
+
+          # For debugging cat ${GITHUB_EVENT_PATH} to see the payload.
+
+          pr_head_sha=${{ github.event.workflow_run.head_sha }}
+          pr_number=$(gh pr list --state all --search $pr_head_sha --json number --jq '.[0].number')
+          echo "Post-PR cache cleanup for PR ${pr_number}"
+          BRANCH=refs/pull/${pr_number}/merge
+
+          # Setting this to not fail the workflow while deleting cache keys.
+          set +e
+
+          keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH | cut -f 1)
+          # $keys might contain spaces. Thus we set IFS to \n.
+          IFS=$'\n'
+          for k in $keys
+          do
+            gh actions-cache delete "$k" -R $REPO -B $BRANCH --confirm
+          done
+          unset IFS

.github/workflows/cleanup-cache.yml

@@ -0,0 +1,67 @@
+name: CleanUpCache
+
+on:
+  workflow_run:
+    workflows: [ERF CI, ERF CUDA CI, ERF CI (hip), ERF CI (sycl), Linux GCC, MacOS]
+    types:
+      - completed
+
+jobs:
+  CleanUpCcacheCache:
+    name: Clean Up Ccache Cache for ${{ github.event.workflow_run.name }}
+    runs-on: ubuntu-22.04
+    permissions:
+      actions: write
+      contents: read
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Clean up ccache
+        run: |
+          gh extension install actions/gh-actions-cache
+
+          REPO=${{ github.repository }}
+
+          # push or pull_request or schedule or ...
+          EVENT=${{ github.event.workflow_run.event }}
+
+          # Triggering workflow run name (e.g., LinuxClang)
+          WORKFLOW_NAME="${{ github.event.workflow_run.name }}"
+
+          # For debugging, cat ${GITHUB_EVENT_PATH} to see the payload.
+
+          if [[ $EVENT == "pull_request" ]]; then
+            pr_head_sha=${{ github.event.workflow_run.head_sha }}
+            pr_number=$(gh pr list --search $pr_head_sha --json number --jq '.[0].number')
+            echo "Clean up cache for PR ${pr_number}"
+            BRANCH=refs/pull/${pr_number}/merge
+          else
+            BRANCH=refs/heads/${{ github.event.workflow_run.head_branch }}
+          fi
+
+          # Setting this to not fail the workflow while deleting cache keys.
+          set +e
+
+          # In our cache keys, substring after `-git-` is git hash, substring
+          # before that is a unique id for jobs (e.g., `ccache-LinuxClang-configure-2d`).
+          # The goal is to keep the last used key of each job and delete all others.
+
+          # something like ccache-LinuxClang-
+          keyprefix="ccache-${WORKFLOW_NAME}-"
+
+          cached_jobs=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "$keyprefix" | awk -F '-git-' '{print $1}' | sort | uniq)
+
+          # cached_jobs is something like "ccache-LinuxClang-configure-1d ccache-LinuxClang-configure-2d".
+          # It might also contain spaces. Thus we set IFS to \n.
+          IFS=$'\n'
+          for j in $cached_jobs
+          do
+            # Delete all entries except the last used one
+            old_keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "${j}-git-" --sort last-used | cut -f 1 | tail -n +2)
+            for k in $old_keys
+            do
+              gh actions-cache delete "$k" -R $REPO -B $BRANCH --confirm
+            done
+          done
+          unset IFS

.github/workflows/codespell.yml

@@ -0,0 +1,23 @@
+name: codespell
+
+on: [push, pull_request]
+
+concurrency:
+  group: ${{ github.ref }}-${{ github.head_ref }}-codespell
+  cancel-in-progress: true
+
+jobs:
+  codespell:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install codespell
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends python3-pip
+          pip3 install --user codespell
+
+      - name: Run codespell
+        run: codespell

.github/workflows/cuda-ci.yml

@@ -30,22 +30,51 @@ jobs:
         uses: styfle/[email protected]
         with:
           access_token: ${{github.token}}
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
       - name: Prepare CUDA environment
         run: Submodules/AMReX/.github/workflows/dependencies/dependencies_nvcc.sh ${{matrix.cuda_ver}}
+      - name: Install CCache
+        run: Submodules/AMReX/.github/workflows/dependencies/dependencies_ccache.sh
+      - name: Set Up Cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/ccache
+          key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }}
+          restore-keys: |
+               ccache-${{ github.workflow }}-${{ github.job }}-git-
       - name: Configure and build
         run: |
+          export CCACHE_COMPRESS=1
+          export CCACHE_COMPRESSLEVEL=10
+          export CCACHE_MAXSIZE=600M
+          ccache -z
+
           export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
           export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH}
           which nvcc || echo "nvcc not in PATH!"
           cmake -Bbuild-${{matrix.cuda_pkg}} \
             -DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+            -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache \
             -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \
             -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON           \
             -DERF_DIM:STRING=3 \
             -DERF_ENABLE_MPI:BOOL=ON \
             -DERF_ENABLE_CUDA:BOOL=ON \
             -DERF_ENABLE_REGRESSION_TESTS_ONLY:BOOL=ON .
-          cmake --build build-${{matrix.cuda_pkg}} -- -j $(nproc)
+          cmake --build build-${{matrix.cuda_pkg}} -- -j $(nproc) \
+          2>&1 | tee -a ${{runner.workspace}}/build-${{matrix.cuda_pkg}}-output.txt;
+
+          ccache -s
+          du -hs ~/.cache/ccache
+
+      - name: Report
+        run: |
+          egrep "warning:|error:" ${{runner.workspace}}/build-${{matrix.cuda_pkg}}-output.txt \
+            | egrep -v "Submodules/amrex" | egrep -v "lto-wrapper: warning:" | sort | uniq \
+            | awk 'BEGIN{i=0}{print $0}{i++}END{print "Warnings: "i}' > ${{runner.workspace}}/build-${{matrix.cuda_pkg}}-output-warnings.txt
+          cat ${{runner.workspace}}/build-${{matrix.cuda_pkg}}-output-warnings.txt
+          export return=$(tail -n 1 ${{runner.workspace}}/build-${{matrix.cuda_pkg}}-output-warnings.txt | awk '{print $2}')
+          exit ${return}

.github/workflows/docs.yml

@@ -8,10 +8,10 @@ concurrency:
 
 jobs:
   build-and-deploy:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Dependencies
         run: |

.github/workflows/draft-pdf.yml

@@ -2,19 +2,19 @@ on: [push]
 
 jobs:
   paper:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     name: Paper Draft
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Build draft PDF
         uses: openjournals/openjournals-draft-action@master
         with:
           journal: joss
           # This should be the path to the paper within your repo.
           paper-path: paper/paper.md
       - name: Upload
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v4
         with:
           name: paper
           # This is the output path where Pandoc will write the compiled