From 19385de01a981da6543ed05379981402aae28608 Mon Sep 17 00:00:00 2001
From: Richard Berger <rberger@lanl.gov>
Date: Tue, 27 Aug 2024 11:20:24 -0600
Subject: [PATCH 1/3] Use new CI setup

---
 .gitlab-ci.yml                         | 169 ++++------------------
 .gitlab-ci/config/spack/upstreams.yaml |   5 -
 .gitlab/build_and_test.sh              | 186 +++++++++++++++++++++++++
 spack-repo/packages/spiner/package.py  |   2 +
 4 files changed, 214 insertions(+), 148 deletions(-)
 delete mode 100644 .gitlab-ci/config/spack/upstreams.yaml
 create mode 100644 .gitlab/build_and_test.sh

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bb4696779..ede93f582 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,166 +1,49 @@
-###################
-# Global Settings #
-###################
-
 stages:
   - build_n_test
 
-variables:
-  XCAP_PROJECT_DIR: "/usr/projects/xcap/oss"
-  TESTING_CLUSTER: darwin
-  SPINER_GCC_VERSION: "10.3.0"
-  SPINER_CUDA_VERSION: "11.6.0"
-  SPINER_OPENMPI_VERSION: "4.1.1"
-  SPINER_SPACK_SPEC: "spiner@main%gcc@=${SPINER_GCC_VERSION}+python+test"
-  SPINER_SPACK_FULL_SPEC: "${SPINER_SPACK_SPEC} ^openblas"
-  COLOR_CYAN: "\e[1;36m"
-  COLOR_PLAIN: "\e[0m"
-  # uncomment to have the CI Spack installation for debugging
-  # PROJECT_TMP_CI_DIR: "${CI_PROJECT_DIR}"
-  # uncomment & set to different spack upstream for testing
-  # PROJECT_TMP_SPACK_DIR: "deployment/CI/spack-v0.20.1-8"
-  # uncomment for spack debug output
-  # PROJECT_SPACK_DEBUG_FLAG: "-d"
+default:
+  interruptible: true
 
-workflow:
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-    - if: $CI_PIPELINE_SOURCE == "pipeline"
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - if: $CI_COMMIT_TAG
+variables:
+  ENABLED_CLUSTERS: "darwin"
+  GIT_SUBMODULE_STRATEGY: normal
 
-default:
-  tags:
-    - darwin-slurm-shared
+.ascgit_job:
   id_tokens:
     SITE_ID_TOKEN:
       aud: https://asc-git.lanl.gov
 
-##########################
-# Build Matrix Variables #
-##########################
-
-.buildtype_debug: &buildtype_debug
-  BUILD_TYPE: "Debug"
-
-.buildtype_release: &buildtype_release
-  BUILD_TYPE: "Release"
-
-######################
-# Platform Variables #
-######################
-
-.skylake: &skylake
-    SCHEDULER_PARAMETERS: "--nodes=1 --partition=skylake-gold,skylake-platinum"
-
-.a100: &a100
-    SCHEDULER_PARAMETERS: "--nodes=1 --partition=shared-gpu-ampere"
-
-#######################
-# Job Script Template #
-#######################
+.darwin_job:
+  rules:
+    - if: $ENABLED_CLUSTERS =~ /darwin/
+  variables:
+    CLUSTER: darwin
+    SCHEDULER_PARAMETERS: "-N 1 --qos=debug -p general,skylake-gold,skylake-platinum --constraint=\"(cpu_family:skylake)&ib:edr\""
+  tags:
+    - darwin-slurm-shared
 
-.test:
+.build_and_test:
   stage: build_n_test
-  before_script:
-    - echo "Running on $(hostname)"
-    - section() { echo $'\e[0K'"section_$1:$(date +%s):$2"$'\r\e[0K'"${3+${COLOR_CYAN}$3${COLOR_PLAIN}}"; }
-    - export PYTHONNOUSERSITE=1
-    - export SPACK_DISABLE_LOCAL_CONFIG=true
-    - export SPACK_SKIP_MODULES=true
-    - export TMP_USER_PROJECT_DIR="/tmp/${USER}/${CI_PROJECT_NAME}/${CI_JOB_NAME}"
-    - export PROJECT_TMP_CI_DIR=${PROJECT_TMP_CI_DIR:-${TMP_USER_PROJECT_DIR}}
-    - export SPACK_USER_CACHE_PATH="${PROJECT_TMP_CI_DIR}/spack-local"
-    - export DEVEL_SPACK_ROOT="${PROJECT_TMP_CI_DIR}/spack"
-    - export DEVEL_SPACK_MIRROR="${CI_PROJECT_DIR}/spack-mirror"
-    - rm -rf ${SPACK_USER_CACHE_PATH} ${DEVEL_SPACK_ROOT}
-    - |
-      if [[ -z "${PROJECT_TMP_SPACK_DIR}" ]]; then
-        export PROJECT_SPACK_ROOT="${XCAP_PROJECT_DIR}/spack-${TESTING_CLUSTER}"
-      else
-        export PROJECT_SPACK_ROOT="${XCAP_PROJECT_DIR}/${PROJECT_TMP_SPACK_DIR}-${TESTING_CLUSTER}"
-      fi
-    - section start "spack_init[collapsed=true]" "Creating Spack instance at ${DEVEL_SPACK_ROOT}"
-    - echo "Upstream ${PROJECT_SPACK_ROOT}"
-    - mkdir -m 2770 -p ${PROJECT_TMP_CI_DIR}
-    - |
-      rsync -rpl \
-          --exclude=".git" \
-          --exclude={"*__pycache__*","*.pyc"} \
-          --include="etc/spack/**" \
-          --include="lib/spack/**" \
-          --exclude-from=${PROJECT_SPACK_ROOT}/.gitignore \
-          ${PROJECT_SPACK_ROOT}/ ${DEVEL_SPACK_ROOT}/
-    - sed -i "s;xcap-admin;xcap;g" "${DEVEL_SPACK_ROOT}/etc/spack/packages.yaml"
-    - source ${DEVEL_SPACK_ROOT}/share/spack/setup-env.sh
-    - spack --version
-    - spack compiler list
-    - section end spack_init
   script:
-    - module load gcc/${SPINER_GCC_VERSION}
-    - module load cuda/${SPINER_CUDA_VERSION}
-    - |
-      if [[ ${CI_JOB_NAME} =~ "a100" ]];
-      then
-        module load openmpi/${SPINER_OPENMPI_VERSION}-gcc_${SPINER_GCC_VERSION}
-        export SPINER_SPACK_FULL_SPEC="${SPINER_SPACK_SPEC}+hdf5+mpi+kokkos ^kokkos+wrapper+cuda cuda_arch=80 ^openmpi@${SPINER_OPENMPI_VERSION} ^openblas";
-      fi
-    - |
-      section start "spack_build[collapsed=true]" "Building via Spack"
-      spack env create spack_build_env
-      spack env activate spack_build_env
-      spack repo add ${DEVEL_SPACK_ROOT}/var/spack/repos/xcap_deployment
-      spack repo add spack-repo
-      spack repo list
-      spack config add upstreams:default:install_tree:${PROJECT_SPACK_ROOT}/opt/spack/
-      spack dev-build -q -j $(nproc) ${SPINER_SPACK_FULL_SPEC}
-      spack env deactivate
-      section end spack_build
-    - section start "spack_env[collapsed=true]" "Creating Spack environment"
-    - spack env create spack_env
-    - spack env activate spack_env
-    - spack repo add ${DEVEL_SPACK_ROOT}/var/spack/repos/xcap_deployment
-    - spack repo add spack-repo
-    - spack repo list
-    - spack config add upstreams:default:install_tree:${PROJECT_SPACK_ROOT}/opt/spack/
-    - spack spec -I ${SPINER_SPACK_FULL_SPEC}
-    - spack add ${SPINER_SPACK_FULL_SPEC}
-    - spack install -j $(nproc) --show-log-on-error --no-checksum --yes-to-all -u cmake
-    - section end spack_env
-    - mkdir -p build
-    - cd build
-    - |
-      cmake \
-            -DSPINER_BUILD_TESTS=ON \
-            -DCMAKE_INSTALL_PREFIX=${CI_PROJECT_DIR}/install \
-            -DSPINER_USE_HDF=ON \
-            -DSPINER_TEST_USE_KOKKOS=$([[ ${CI_JOB_NAME} =~ "a100" ]] && echo ON || echo OFF) \
-            -DCMAKE_CXX_COMPILER=$([[ ${CI_JOB_NAME} =~ "a100" ]] && echo nvcc_wrapper || g++) \
-            ..
-    - make -j $(nproc)
-    - make install
-    - make test
-    - export CTEST_OUTPUT_ON_FAILURE=1
-    - ctest --output-junit tests.xml
-    - make convergence
+    - source .gitlab/build_and_test.sh ${CLUSTER} ${SPACK_ENV_NAME}
   artifacts:
+    expire_in: 2 weeks
     paths:
-      - build/tests.xml
+      - ${CI_PROJECT_DIR}/build/tests.xml
     reports:
-      junit: build/tests.xml
+      junit: ${CI_PROJECT_DIR}/build/tests.xml
 
 ########
 # Jobs #
 ########
 
-gnu_skylake:
-  extends: .test
+openmpi_gcc:
+  extends: [.ascgit_job, .darwin_job, .build_and_test]
   variables:
-    <<: *skylake
+    SPACK_ENV_NAME: openmpi-gcc
 
-gnu_a100:
-  extends: .test
+openmpi_cuda_gcc_ampere:
+  extends: [.ascgit_job, .darwin_job, .build_and_test]
   variables:
-    <<: *a100
-
+    SPACK_ENV_NAME: openmpi-cuda-gcc-ampere
+    SCHEDULER_PARAMETERS: "-N 1 --qos=debug -p shared-gpu-ampere"
diff --git a/.gitlab-ci/config/spack/upstreams.yaml b/.gitlab-ci/config/spack/upstreams.yaml
deleted file mode 100644
index ca96de527..000000000
--- a/.gitlab-ci/config/spack/upstreams.yaml
+++ /dev/null
@@ -1,5 +0,0 @@
-upstreams:
-  oss-spack:
-    install_tree: XCAP_OSS_SPACK_DIR_TMP/opt/spack
-    modules:
-      tcl: XCAP_OSS_SPACK_DIR_TMP/share/spack/modules
diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh
new file mode 100644
index 000000000..d80b90d63
--- /dev/null
+++ b/.gitlab/build_and_test.sh
@@ -0,0 +1,186 @@
+#!/bin/bash
+PROJECT_NAME=spiner
+PROJECT_DEFAULT_BRANCH=main
+PROJECT_GROUP=oss
+BUILD_DIR=${BUILD_DIR:-build}
+SOURCE_DIR=${CI_PROJECT_DIR:-$PWD}
+PROJECT_SPACK_ENV=${PROJECT_SPACK_ENV:-/usr/projects/xcap/spack-env/oss/2024-08-01}
+CI_SPACK_ENV=${CI_SPACK_ENV:-$TMPDIR/$USER-ci-env}
+UNTIL=${UNTIL:-install}
+
+# colors
+COLOR_BLUE='\033[1;34m'
+COLOR_MAGENTA='\033[1;35m'
+COLOR_CYAN='\033[1;36m'
+COLOR_PLAIN='\033[0m'
+
+section() {
+  if [ -z "${CI}" ]; then
+    echo -e "${3:+${COLOR_CYAN}$3${COLOR_PLAIN}}"
+  else
+    echo -e $'\e[0K'"section_$1:$(date +%s):$2"$'\r\e[0K'"${3:+${COLOR_CYAN}$3${COLOR_PLAIN}}"
+  fi
+}
+
+print_usage() {
+  echo "usage: build_and_test [-h] [-u {spack,env,cmake,build,test,install}] system environment"
+  echo
+  echo "After the 'env' phase has run, you will be in an active Spack environment."
+  echo "You can then use the 'activate_build_env' command to enter a sub-shell"
+  echo "with the Spack build environment for your project."
+  echo
+  echo "positional arguments:"
+  echo "  system         name of the system"
+  echo "  environment    name of the Spack environment"
+  echo
+  echo "options:"
+  echo "  -u {spack,env,cmake,build,test,install}, --until {spack,env,cmake,build,test,install}"
+  echo "                 run script until the given phase"
+  echo "  -h, --help     show this help message and exit"
+}
+
+VALID_CMD=false
+if [ $# -eq 2 ] && [[ ! "$1" =~ ^-.*  ]] && [[ ! "$2" =~ ^-.*  ]]; then
+  VALID_CMD=true
+elif [ $# -eq 4 ]; then
+  if [ "$1" = "-u" ] || [ "$1" = "--until" ]; then
+    if [[ "$2" =~ ^(spack|env|cmake|build|test|install)$ ]]; then
+      UNTIL=$2
+      VALID_CMD=true
+      shift 2
+    fi
+  fi
+fi
+
+if ! $VALID_CMD; then
+    print_usage
+    return
+fi
+
+SYSTEM_NAME=$1
+SPACK_ENV_NAME=$2
+
+###############################################################################
+# Generic steps
+###############################################################################
+
+#------------------------------------------------------------------------------
+# Prepare Spack instance
+#------------------------------------------------------------------------------
+prepare_spack() {
+  section start "prepare_spack[collapsed=true]" "Prepare Spack"
+  umask 0007
+  source $PROJECT_SPACK_ENV/replicate.sh $CI_SPACK_ENV
+  section end "prepare_spack"
+}
+
+#------------------------------------------------------------------------------
+# Setup up Spack environment
+#------------------------------------------------------------------------------
+prepare_env() {
+  section start "prepare_env[collapsed=true]" "Create environment"
+  echo "Activating ${SPACK_ENV_NAME} environment on ${SYSTEM_NAME}"
+  source ${CI_SPACK_ENV}/systems/${SYSTEM_NAME}/activate.sh ${PROJECT_GROUP}/${PROJECT_NAME}/${SPACK_ENV_NAME}
+  if [ -d ${SOURCE_DIR}/spack-repo ]; then
+    spack repo add ${SOURCE_DIR}/spack-repo
+  fi
+
+  spack develop -b ${BUILD_DIR} -p ${SOURCE_DIR} --no-clone ${PROJECT_NAME}@${PROJECT_DEFAULT_BRANCH}
+  spack install --include-build-deps --only dependencies -j $(nproc)
+  section end "prepare_env"
+}
+
+#------------------------------------------------------------------------------
+# Run CMake (configured via Project Spack package and selected Spec)
+#------------------------------------------------------------------------------
+spack_cmake_configure() {
+  section start "spack_cmake_configure[collapsed=true]" "Initial Spack CMake Configure"
+  spack install --test root --include-build-deps -u cmake -v ${PROJECT_NAME}
+  BUILD_ENV=${BUILD_DIR}/build_env.sh
+  spack build-env --dump ${BUILD_ENV} ${PROJECT_NAME}
+  section end "spack_cmake_configure"
+}
+
+#------------------------------------------------------------------------------
+# Build project
+#------------------------------------------------------------------------------
+cmake_build() {
+  section start "cmake_build[collapsed=false]" "CMake Build"
+  (
+  source ${BUILD_ENV}
+  cmake -DCMAKE_VERBOSE_MAKEFILE=off -DCMAKE_INSTALL_PREFIX=$PWD/build/install build
+  cmake --build ${BUILD_DIR} --parallel
+  )
+  section end "cmake_build"
+}
+
+#------------------------------------------------------------------------------
+# Run tests
+#------------------------------------------------------------------------------
+cmake_test() {
+  section start "testing[collapsed=false]" "Tests"
+  (
+  source ${BUILD_ENV}
+  export CTEST_OUTPUT_ON_FAILURE=1
+  ctest --test-dir ${BUILD_DIR} --output-junit tests.xml
+  )
+  section end "testing"
+}
+
+#------------------------------------------------------------------------------
+# Install project
+#------------------------------------------------------------------------------
+cmake_install() {
+  section start install "Install"
+  (
+  source ${BUILD_ENV}
+  cmake --build ${BUILD_DIR} --target install
+  )
+  section end install
+}
+
+#------------------------------------------------------------------------------
+# Enter build environment shell
+#------------------------------------------------------------------------------
+activate_build_env() {
+  spack build-env ${PROJECT_NAME} bash
+}
+
+###############################################################################
+# Run script
+###############################################################################
+echo "Running on $(hostname)"
+
+if [ ${CI} ]; then
+  echo " "
+  echo -e "${COLOR_BLUE}######################################################################${COLOR_PLAIN}"
+  echo " "
+  echo -e "${COLOR_BLUE}To recreate this CI run, follow these steps:${COLOR_PLAIN}"
+  echo " "
+  echo -e "${COLOR_BLUE}ssh ${CLUSTER}${COLOR_PLAIN}"
+  echo -e "${COLOR_BLUE}salloc ${SCHEDULER_PARAMETERS}${COLOR_PLAIN}"
+  echo -e "${COLOR_BLUE}source .gitlab/build_and_test.sh ${CLUSTER} ${SPACK_ENV_NAME}${COLOR_PLAIN}"
+  echo " "
+  echo -e "${COLOR_BLUE}See 'source .gitlab/build_and_test.sh -h' for more options.${COLOR_PLAIN}"
+  echo " "
+  echo -e "${COLOR_BLUE}######################################################################${COLOR_PLAIN}"
+  echo " "
+fi
+
+prepare_spack
+if [[ "${UNTIL}" == "spack" ]]; then return; fi
+
+prepare_env
+if [[ "${UNTIL}" == "env" ]]; then return; fi
+
+spack_cmake_configure
+if [[ "${UNTIL}" == "cmake" ]]; then return; fi
+
+cmake_build
+if [[ "${UNTIL}" == "build" ]]; then return; fi
+
+cmake_test
+if [[ "${UNTIL}" == "test" ]]; then return; fi
+
+cmake_install
+if [[ "${UNTIL}" == "install" ]]; then return; fi
diff --git a/spack-repo/packages/spiner/package.py b/spack-repo/packages/spiner/package.py
index 57062f6cc..b7627e829 100644
--- a/spack-repo/packages/spiner/package.py
+++ b/spack-repo/packages/spiner/package.py
@@ -85,6 +85,8 @@ def cmake_args(self):
 
         args = [
             self.define("BUILD_TESTING", self.run_tests),
+            self.define("SPINER_BUILD_TESTS", self.run_tests),
+            self.define("SPINER_TEST_USE_KOKKOS", self.run_tests and self.spec.satisfies("+kokkos")),
             self.define_from_variant(use_kokkos_option, "kokkos"),
             self.define_from_variant("SPINER_USE_HDF", "hdf5"),
         ]

From a0bb545ab4e55401a2052d513b4d63d60ef4b167 Mon Sep 17 00:00:00 2001
From: Richard Berger <rberger@lanl.gov>
Date: Tue, 3 Sep 2024 23:03:25 -0600
Subject: [PATCH 2/3] Update spackage

---
 spack-repo/packages/spiner/package.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/spack-repo/packages/spiner/package.py b/spack-repo/packages/spiner/package.py
index b7627e829..74feaf2b3 100644
--- a/spack-repo/packages/spiner/package.py
+++ b/spack-repo/packages/spiner/package.py
@@ -63,8 +63,9 @@ class Spiner(CMakePackage):
     # Currently the raw cuda backend of ports-of-call is not supported.
     depends_on("ports-of-call portability_strategy=Kokkos", when="@:1.5.1 +kokkos")
     depends_on("ports-of-call portability_strategy=None", when="@:1.5.1 ~kokkos")
+    depends_on("kokkos@3.3.00:", when="+kokkos")
     depends_on(
-        "kokkos@3.3.00: ~shared+cuda_lambda+cuda_constexpr",
+        "kokkos ~shared+cuda_lambda+cuda_constexpr",
         when="+kokkos ^kokkos+cuda",
     )
 

From a54c447bc6c658ee155191fd7bf723f02d294d65 Mon Sep 17 00:00:00 2001
From: Richard Berger <rberger@lanl.gov>
Date: Wed, 4 Sep 2024 10:52:54 -0600
Subject: [PATCH 3/3] Update build_and_test.sh

---
 .gitlab/build_and_test.sh | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/.gitlab/build_and_test.sh b/.gitlab/build_and_test.sh
index d80b90d63..07b2cf150 100644
--- a/.gitlab/build_and_test.sh
+++ b/.gitlab/build_and_test.sh
@@ -4,7 +4,7 @@ PROJECT_DEFAULT_BRANCH=main
 PROJECT_GROUP=oss
 BUILD_DIR=${BUILD_DIR:-build}
 SOURCE_DIR=${CI_PROJECT_DIR:-$PWD}
-PROJECT_SPACK_ENV=${PROJECT_SPACK_ENV:-/usr/projects/xcap/spack-env/oss/2024-08-01}
+PROJECT_SPACK_ENV=${PROJECT_SPACK_ENV:-/usr/projects/xcap/spack-env/oss/current}
 CI_SPACK_ENV=${CI_SPACK_ENV:-$TMPDIR/$USER-ci-env}
 UNTIL=${UNTIL:-install}
 
@@ -25,10 +25,23 @@ section() {
 print_usage() {
   echo "usage: build_and_test [-h] [-u {spack,env,cmake,build,test,install}] system environment"
   echo
+  echo "This script allows you to active a Spack environment on a given system,"
+  echo "configure your project's CMake based on its Spack variants, and then build, test and"
+  echo "install it. Use the -u/--until option to stop the script early at a given phase."
+  echo
   echo "After the 'env' phase has run, you will be in an active Spack environment."
   echo "You can then use the 'activate_build_env' command to enter a sub-shell"
   echo "with the Spack build environment for your project."
   echo
+  echo "You can also make use of the following commands to manually run each phase and customize"
+  echo "the environment in between:"
+  echo " - prepare_spack"
+  echo " - prepare_env"
+  echo " - spack_cmake_configure"
+  echo " - cmake_build [EXTRA_CMAKE_ARGS]"
+  echo " - cmake_test [EXTRA_CTEST_ARGS]"
+  echo " - cmake_install"
+  echo
   echo "positional arguments:"
   echo "  system         name of the system"
   echo "  environment    name of the Spack environment"
@@ -108,7 +121,7 @@ cmake_build() {
   section start "cmake_build[collapsed=false]" "CMake Build"
   (
   source ${BUILD_ENV}
-  cmake -DCMAKE_VERBOSE_MAKEFILE=off -DCMAKE_INSTALL_PREFIX=$PWD/build/install build
+  cmake -DCMAKE_VERBOSE_MAKEFILE=off -DCMAKE_INSTALL_PREFIX=$PWD/${BUILD_DIR}/install $@ ${BUILD_DIR}
   cmake --build ${BUILD_DIR} --parallel
   )
   section end "cmake_build"
@@ -122,7 +135,7 @@ cmake_test() {
   (
   source ${BUILD_ENV}
   export CTEST_OUTPUT_ON_FAILURE=1
-  ctest --test-dir ${BUILD_DIR} --output-junit tests.xml
+  ctest --test-dir ${BUILD_DIR} --output-junit tests.xml $@
   )
   section end "testing"
 }
@@ -158,8 +171,9 @@ if [ ${CI} ]; then
   echo -e "${COLOR_BLUE}To recreate this CI run, follow these steps:${COLOR_PLAIN}"
   echo " "
   echo -e "${COLOR_BLUE}ssh ${CLUSTER}${COLOR_PLAIN}"
+  echo -e "${COLOR_BLUE}cd /your/${PROJECT_NAME}/checkout${COLOR_PLAIN}"
   echo -e "${COLOR_BLUE}salloc ${SCHEDULER_PARAMETERS}${COLOR_PLAIN}"
-  echo -e "${COLOR_BLUE}source .gitlab/build_and_test.sh ${CLUSTER} ${SPACK_ENV_NAME}${COLOR_PLAIN}"
+  echo -e "${COLOR_BLUE}source .gitlab/build_and_test.sh --until ${UNTIL} ${CLUSTER} ${SPACK_ENV_NAME}${COLOR_PLAIN}"
   echo " "
   echo -e "${COLOR_BLUE}See 'source .gitlab/build_and_test.sh -h' for more options.${COLOR_PLAIN}"
   echo " "