From c15e1510cf1c0b14c5555649406c76bdea16788b Mon Sep 17 00:00:00 2001
From: Richard Berger <rberger@lanl.gov>
Date: Fri, 5 Apr 2024 11:15:16 -0600
Subject: [PATCH 1/4] Fixup CI after deployment change

---
 .gitlab-ci.yml | 134 +++++++++++++++++++++++++++----------------------
 1 file changed, 73 insertions(+), 61 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 77f25e14a..a9f4a05d9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -6,39 +6,35 @@ stages:
   - build_n_test
 
 variables:
-  GIT_SUBMODULE_STRATEGY: recursive
-  SPINER_GCC_VERSION: "9.3.0"
+  GIT_SUBMODULE_STRATEGY: normal
+  XCAP_PROJECT_DIR: "/usr/projects/xcap/oss"
+  TESTING_CLUSTER: darwin
+  SPINER_GCC_VERSION: "10.3.0"
   SPINER_CUDA_VERSION: "11.6.0"
-  SPINER_SPACK_SPEC: "spiner@main+python+test%gcc@${SPINER_GCC_VERSION}"
+  SPINER_SPACK_SPEC: "spiner@main+python+test%gcc@=${SPINER_GCC_VERSION}"
   COLOR_CYAN: "\e[1;36m"
   COLOR_PLAIN: "\e[0m"
-  # set to different spack upstream for testing
-  SPINER_TEMP_SPACK_DIR: ""
-  # uncomment for debugging the CI and persist the Spack installation
-  # SPACK_USER_CACHE_PATH: "${CI_PROJECT_DIR}/spack-local"
-  # DEVEL_SPACK_CHECKOUT: "${CI_PROJECT_DIR}/spack"
-
-before_script:
-  - echo "Running on $(hostname)"
-  - section() { echo $'\e[0K'"section_$1:$(date +%s):$2"$'\r\e[0K'"${3+${COLOR_CYAN}$3${COLOR_PLAIN}}"; }
-  - export SPACK_DISABLE_LOCAL_CONFIG=true
-  - export SPACK_USER_CACHE_PATH=${SPACK_USER_CACHE_PATH:-/tmp/spack-local}
-  - export DEVEL_SPACK_CHECKOUT=${DEVEL_SPACK_CHECKOUT:-/tmp/spack}
-  - export SPINER_SPACK_DIR=${SPINER_TEMP_SPACK_DIR:=$XCAP_OSS_SPACK_DIR}
-  - section start "spack_init[collapsed=true]" "Creating Spack instance at ${DEVEL_SPACK_CHECKOUT}"
-  - |
-    rsync -rpl \
-        --exclude=".git" \
-        --exclude="etc/spack/repos.yaml" \
-        --exclude="etc/spack/packages.yaml" \
-        --include="etc/spack/**" \
-        --include="lib/spack/**" \
-        --exclude-from=${SPINER_SPACK_DIR}/.gitignore \
-        ${SPINER_SPACK_DIR}/ ${DEVEL_SPACK_CHECKOUT}/
-  - source ${DEVEL_SPACK_CHECKOUT}/share/spack/setup-env.sh
-  - spack --version
-  - spack compiler list
-  - section end spack_init
+  # uncomment to have the CI Spack installation for debugging
+  # PROJECT_TMP_CI_DIR: "${CI_PROJECT_DIR}"
+  # uncomment & set to different spack upstream for testing
+  # PROJECT_TMP_SPACK_DIR: "deployment/CI/spack-v0.20.1-8"
+  # uncomment for spack debug output
+  # PROJECT_SPACK_DEBUG_FLAG: "-d"
+
+workflow:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_PIPELINE_SOURCE == "pipeline"
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_COMMIT_TAG
+
+default:
+  tags:
+    - darwin-slurm-shared
+  id_tokens:
+    SITE_ID_TOKEN:
+      aud: https://asc-git.lanl.gov
 
 ##########################
 # Build Matrix Variables #
@@ -50,39 +46,55 @@ before_script:
 .buildtype_release: &buildtype_release
   BUILD_TYPE: "Release"
 
+######################
+# Platform Variables #
+######################
+
 .skylake: &skylake
     SCHEDULER_PARAMETERS: "--nodes=1 --partition=skylake-gold,skylake-platinum"
 
 .a100: &a100
     SCHEDULER_PARAMETERS: "--nodes=1 --partition=shared-gpu-ampere"
 
-#################
-# General Setup #
-#################
-
-.job: &job_def
-  id_tokens:
-    SITE_ID_TOKEN:
-      aud: https://asc-git.lanl.gov
-  tags:
-    - darwin-slurm-shared
-  variables:
-    SCHEDULER_PARAMETERS: "--nodes=1 --partition=skylake-gold,skylake-platinum"
-  only:
-    refs:
-      - merge_requests
-      - pipelines
-  except:
-    refs:
-
 #######################
 # Job Script Template #
 #######################
 
 .test:
   stage: build_n_test
-  extends:
-    - .job
+  before_script:
+    - echo "Running on $(hostname)"
+    - section() { echo $'\e[0K'"section_$1:$(date +%s):$2"$'\r\e[0K'"${3+${COLOR_CYAN}$3${COLOR_PLAIN}}"; }
+    - export PYTHONNOUSERSITE=1
+    - export SPACK_DISABLE_LOCAL_CONFIG=true
+    - export TMP_USER_PROJECT_DIR="/tmp/${USER}/${CI_PROJECT_NAME}/${CI_JOB_NAME}"
+    - export PROJECT_TMP_CI_DIR=${PROJECT_TMP_CI_DIR:-${TMP_USER_PROJECT_DIR}}
+    - export SPACK_USER_CACHE_PATH="${PROJECT_TMP_CI_DIR}/spack-local"
+    - export DEVEL_SPACK_ROOT="${PROJECT_TMP_CI_DIR}/spack"
+    - export DEVEL_SPACK_MIRROR="${CI_PROJECT_DIR}/spack-mirror"
+    - rm -rf ${SPACK_USER_CACHE_PATH} ${DEVEL_SPACK_ROOT}
+    - |
+      if [[ -z "${PROJECT_TMP_SPACK_DIR}" ]]; then
+        export PROJECT_SPACK_ROOT=${XCAP_PROJECT_DIR}/spack
+      else
+        export PROJECT_SPACK_ROOT="${XCAP_PROJECT_DIR}/${PROJECT_TMP_SPACK_DIR}-${TESTING_CLUSTER}"
+      fi
+    - section start "spack_init[collapsed=true]" "Creating Spack instance at ${DEVEL_SPACK_ROOT}"
+    - echo "Upstream ${PROJECT_SPACK_ROOT}"
+    - mkdir -m 2770 -p ${PROJECT_TMP_CI_DIR}
+    - |
+      rsync -rpl \
+          --exclude=".git" \
+          --exclude={"*__pycache__*","*.pyc"} \
+          --include="etc/spack/**" \
+          --include="lib/spack/**" \
+          --exclude-from=${PROJECT_SPACK_ROOT}/.gitignore \
+          ${PROJECT_SPACK_ROOT}/ ${DEVEL_SPACK_ROOT}/
+    - sed -i "s;xcap-admin;xcap;g" "${DEVEL_SPACK_ROOT}/etc/spack/packages.yaml"
+    - source ${DEVEL_SPACK_ROOT}/share/spack/setup-env.sh
+    - spack --version
+    - spack compiler list
+    - section end spack_init
   script:
     - module load gcc/${SPINER_GCC_VERSION}
     - module load cuda/${SPINER_CUDA_VERSION}
@@ -93,22 +105,24 @@ before_script:
       fi
     - |
       section start "spack_build[collapsed=true]" "Building via Spack"
-      mkdir -p spack_build_env
-      spack env create -d spack_build_env
-      spack env activate -d spack_build_env
+      spack env create spack_build_env
+      spack env activate spack_build_env
+      spack repo add ${DEVEL_SPACK_ROOT}/var/spack/repos/xcap_deployment
       spack repo add spack-repo
       spack repo list
+      spack develop --no-clone -p ${CI_PROJECT_DIR}/ports-of-call ports-of-call@main
       spack config add upstreams:default:install_tree:${SPINER_SPACK_DIR}/opt/spack/
       spack dev-build -q -j $(nproc) ${SPINER_SPACK_SPEC}
       spack env deactivate
       section end spack_build
     - section start "spack_env[collapsed=true]" "Creating Spack environment"
-    - mkdir -p spack_env
-    - spack env create -d spack_env
-    - spack env activate -d spack_env
+    - spack env create spack_env
+    - spack env activate spack_env
+    - spack repo add ${DEVEL_SPACK_ROOT}/var/spack/repos/xcap_deployment
     - spack repo add spack-repo
     - spack repo list
-    - spack config add upstreams:default:install_tree:${SPINER_SPACK_DIR}/opt/spack/
+    - spack develop --no-clone -p ${CI_PROJECT_DIR}/ports-of-call ports-of-call@main
+    - spack config add upstreams:default:install_tree:${PROJECT_SPACK_ROOT}/opt/spack/
     - spack spec -I ${SPINER_SPACK_SPEC}
     - spack add ${SPINER_SPACK_SPEC}
     - spack install -j $(nproc) --show-log-on-error --no-checksum --yes-to-all -u cmake
@@ -116,7 +130,7 @@ before_script:
     - mkdir -p build
     - cd build
     - |
-      cmake --log-level=DEBUG \
+      cmake \
             -DSPINER_BUILD_TESTS=ON \
             -DCMAKE_INSTALL_PREFIX=${CI_PROJECT_DIR}/install \
             -DSPINER_USE_HDF=ON \
@@ -140,13 +154,11 @@ before_script:
 ########
 
 gnu_skylake:
-  <<: *job_def
   extends: .test
   variables:
     <<: *skylake
 
 gnu_a100:
-  <<: *job_def
   extends: .test
   variables:
     <<: *a100

From 84f7f5fc3f5e66d5a06e2639c0175a372e02c0f7 Mon Sep 17 00:00:00 2001
From: Richard Berger <rberger@lanl.gov>
Date: Fri, 5 Apr 2024 12:20:44 -0600
Subject: [PATCH 2/4] Remove incomplete ports-of-call submodule

---
 .gitlab-ci.yml | 3 ---
 .gitmodules    | 3 ---
 2 files changed, 6 deletions(-)
 delete mode 100644 .gitmodules

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a9f4a05d9..7742163e5 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -6,7 +6,6 @@ stages:
   - build_n_test
 
 variables:
-  GIT_SUBMODULE_STRATEGY: normal
   XCAP_PROJECT_DIR: "/usr/projects/xcap/oss"
   TESTING_CLUSTER: darwin
   SPINER_GCC_VERSION: "10.3.0"
@@ -110,7 +109,6 @@ default:
       spack repo add ${DEVEL_SPACK_ROOT}/var/spack/repos/xcap_deployment
       spack repo add spack-repo
       spack repo list
-      spack develop --no-clone -p ${CI_PROJECT_DIR}/ports-of-call ports-of-call@main
       spack config add upstreams:default:install_tree:${SPINER_SPACK_DIR}/opt/spack/
       spack dev-build -q -j $(nproc) ${SPINER_SPACK_SPEC}
       spack env deactivate
@@ -121,7 +119,6 @@ default:
     - spack repo add ${DEVEL_SPACK_ROOT}/var/spack/repos/xcap_deployment
     - spack repo add spack-repo
     - spack repo list
-    - spack develop --no-clone -p ${CI_PROJECT_DIR}/ports-of-call ports-of-call@main
     - spack config add upstreams:default:install_tree:${PROJECT_SPACK_ROOT}/opt/spack/
     - spack spec -I ${SPINER_SPACK_SPEC}
     - spack add ${SPINER_SPACK_SPEC}
diff --git a/.gitmodules b/.gitmodules
deleted file mode 100644
index 30012419f..000000000
--- a/.gitmodules
+++ /dev/null
@@ -1,3 +0,0 @@
-[submodule "ports-of-call"]
-	path = ports-of-call
-	url = ../ports-of-call.git

From c38604fdcc2099d4d1fed37da5b67d20228d6f04 Mon Sep 17 00:00:00 2001
From: Richard Berger <rberger@lanl.gov>
Date: Fri, 5 Apr 2024 12:31:11 -0600
Subject: [PATCH 3/4] Speed up spack by disabling MODULEPATH manipulation

---
 .gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7742163e5..0f8cf5b9c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -66,6 +66,7 @@ default:
     - section() { echo $'\e[0K'"section_$1:$(date +%s):$2"$'\r\e[0K'"${3+${COLOR_CYAN}$3${COLOR_PLAIN}}"; }
     - export PYTHONNOUSERSITE=1
     - export SPACK_DISABLE_LOCAL_CONFIG=true
+    - export SPACK_SKIP_MODULES=true
     - export TMP_USER_PROJECT_DIR="/tmp/${USER}/${CI_PROJECT_NAME}/${CI_JOB_NAME}"
     - export PROJECT_TMP_CI_DIR=${PROJECT_TMP_CI_DIR:-${TMP_USER_PROJECT_DIR}}
     - export SPACK_USER_CACHE_PATH="${PROJECT_TMP_CI_DIR}/spack-local"

From 82eeed868ec9c62c4fbfebe4bed9f056ceea2110 Mon Sep 17 00:00:00 2001
From: Richard Berger <rberger@lanl.gov>
Date: Tue, 9 Apr 2024 15:45:17 -0600
Subject: [PATCH 4/4] ci: add missing mpi module load

---
 .gitlab-ci.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0f8cf5b9c..7eba7ad06 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -10,6 +10,7 @@ variables:
   TESTING_CLUSTER: darwin
   SPINER_GCC_VERSION: "10.3.0"
   SPINER_CUDA_VERSION: "11.6.0"
+  SPINER_OPENMPI_VERSION: "4.1.1"
   SPINER_SPACK_SPEC: "spiner@main+python+test%gcc@=${SPINER_GCC_VERSION}"
   COLOR_CYAN: "\e[1;36m"
   COLOR_PLAIN: "\e[0m"
@@ -101,7 +102,8 @@ default:
     - |
       if [[ ${CI_JOB_NAME} =~ "a100" ]];
       then
-        export SPINER_SPACK_SPEC="${SPINER_SPACK_SPEC}+hdf5+mpi+kokkos ^kokkos+wrapper+cuda cuda_arch=80";
+        module load openmpi/${SPINER_OPENMPI_VERSION}-gcc_${SPINER_GCC_VERSION}
+        export SPINER_SPACK_SPEC="${SPINER_SPACK_SPEC}+hdf5+mpi+kokkos ^kokkos+wrapper+cuda cuda_arch=80 ^openmpi@${SPINER_OPENMPI_VERSION}";
       fi
     - |
       section start "spack_build[collapsed=true]" "Building via Spack"