Merge pull request #105 from lanl/rberger/ci_update

ci: update
lanl · Nov 15, 2024 · aca8824 · aca8824
2 parents 2ed8581 + 4fcd191
commit aca8824
Show file tree

Hide file tree

Showing 2 changed files with 124 additions and 18 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -3,15 +3,36 @@ stages:
 
 variables:
   PROJECT_NAME: spiner
-  ENABLE_CDASH: false
+  ENABLE_CDASH: true
+  # use this variable if there is a xcap/deployment MR you want to use instead
+  # PROJECT_SPACK_ENV_VERSION: mr/62/2024-10-15
 
 include:
  - .gitlab/common.yml
 
 
-########
-# Jobs #
-########
+# Spack environments are defined in
+# https://re-git.lanl.gov/xcap/deployment/-/tree/xcap-spack-env/systems
+# for each system and for each XCAP project.
+#
+# The project configuration can be found at the top of
+# .gitlab/build_and_test.sh.
+#
+# Each environment is uniquely identified by
+# ${SYSTEM_NAME}/${PROJECT_GROUP}/${PROJECT_NAME}/${SPACK_ENV_NAME}
+#
+# For creating a new custom environment for testing use the special
+# 'custom-spec' or 'custom-file` environments.
+#
+# The 'custom-spec` environment allows you specify an arbitrary Spack spec to
+# be added to a system-specific empty Spack environment. Use the SPACK_ENV_SPEC
+# environment variable to define the spec you want to add.
+
+# The 'custom-file` environment allows you to manually define an environment
+# from scratch.  Use the SPACK_ENV_FILE environment variable to specify the
+# path to the YAML file you want to use. Note, the environment will be placed
+# in ${SYSTEM_NAME}/${PROJECT_GROUP}/${PROJECT_NAME}/custom-file/spack.yaml and
+# can therefore make relative includes to common configuration files.
 
 prereq_offline_deps:
   stage: build_n_test
@@ -20,7 +41,7 @@ prereq_offline_deps:
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_PIPELINE_SOURCE == "web"
-    - if: $CI_PIPELINE_SOURCE == "push"
+    - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
   needs: []
   script:
     - mkdir extern
@@ -34,29 +55,115 @@ openmpi_gcc:
   extends: [.ascgit_job, .darwin_job, .darwin_regular_job, .build_and_test]
   variables:
     SPACK_ENV_NAME: openmpi-gcc
+    SUBMIT_TO_CDASH: "${ENABLE_CDASH}"
 
 openmpi_cuda_gcc_volta:
   extends: [.ascgit_job, .darwin_job, .darwin_regular_job, .build_and_test]
   variables:
     SPACK_ENV_NAME: openmpi-cuda-gcc-volta
     SCHEDULER_PARAMETERS: "-N 1 --qos=debug -p volta-x86 -C cpu_family:haswell --time=02:00:00"
+    SUBMIT_TO_CDASH: "${ENABLE_CDASH}"
 
 openmpi_cuda_gcc_ampere:
   extends: [.ascgit_job, .darwin_job, .darwin_regular_job, .build_and_test]
   variables:
     SPACK_ENV_NAME: openmpi-cuda-gcc-ampere
     SCHEDULER_PARAMETERS: "-N 1 --qos=debug -p shared-gpu-ampere"
+    SUBMIT_TO_CDASH: "${ENABLE_CDASH}"
+
+rzansel_spectrummpi_cuda_volta_gcc:
+  extends: [.ascgit_job, .rzansel_job, .rzansel_regular_job, .build_and_test]
+  needs:
+   - prereq_offline_deps
+  variables:
+    SPACK_ENV_NAME: spectrummpi-cuda-volta-gcc
+    SUBMIT_TO_CDASH: false              # disable CDash submission, since LLNL systems can't reach them
+    BUILD_WITH_CTEST: "${ENABLE_CDASH}" # but do build with CTest to create the necessary files
+    SUBMIT_ON_ERROR: false              # no network access, don't try to send from compute node
+    SUBMIT_AFTER: false                 # submit host also doesn't have access to CDash server
+    CREATE_SUBMIT_ARTIFACT: "${ENABLE_CDASH}" # archive results
+
+rzansel_spectrummpi_cuda_volta_xl:
+  extends: [.ascgit_job, .rzansel_job, .rzansel_regular_job, .build_and_test]
+  needs:
+   - prereq_offline_deps
+  variables:
+    SPACK_ENV_NAME: spectrummpi-cuda-volta-xl
+    SUBMIT_TO_CDASH: false              # disable CDash submission, since LLNL systems can't reach them
+    BUILD_WITH_CTEST: "${ENABLE_CDASH}" # but do build with CTest to create the necessary files
+    SUBMIT_ON_ERROR: false              # no network access, don't try to send from compute node
+    SUBMIT_AFTER: false                 # submit host also doesn't have access to CDash server
+    CREATE_SUBMIT_ARTIFACT: "${ENABLE_CDASH}" # archive results
 
-rzvernal_craympich_rocm_mi250_gcc:
-  extends: [.ascgit_job, .rzadams_job, .rzvernal_regular_job, .build_and_test]
+rzvernal_craympich_rocm_mi250_cce:
+  extends: [.ascgit_job, .rzvernal_job, .rzvernal_regular_job, .build_and_test]
   needs:
    - prereq_offline_deps
   variables:
-    SPACK_ENV_NAME: craympich-rocm-gfx90a-gcc
+    SPACK_ENV_NAME: craympich-rocm-gfx90a-cce
+    SUBMIT_TO_CDASH: false              # disable CDash submission, since LLNL systems can't reach them
+    BUILD_WITH_CTEST: "${ENABLE_CDASH}" # but do build with CTest to create the necessary files
+    SUBMIT_ON_ERROR: false              # no network access, don't try to send from compute node
+    SUBMIT_AFTER: false                 # submit host also doesn't have access to CDash server
+    CREATE_SUBMIT_ARTIFACT: "${ENABLE_CDASH}" # archive results
 
-rzadams_craympich_rocm_mi300_gcc:
+rzadams_craympich_rocm_mi300_cce:
   extends: [.ascgit_job, .rzadams_job, .rzadams_regular_job, .build_and_test]
   needs:
    - prereq_offline_deps
   variables:
-    SPACK_ENV_NAME: craympich-rocm-gfx942-gcc
+    SPACK_ENV_NAME: craympich-rocm-gfx942-cce
+    SUBMIT_TO_CDASH: false              # disable CDash submission, since LLNL systems can't reach them
+    BUILD_WITH_CTEST: "${ENABLE_CDASH}" # but do build with CTest to create the necessary files
+    SUBMIT_ON_ERROR: false              # no network access, don't try to send from compute node
+    SUBMIT_AFTER: false                 # submit host also doesn't have access to CDash server
+    CREATE_SUBMIT_ARTIFACT: "${ENABLE_CDASH}" # archive results
+
+# collect job results from RZansel and submit them to CDash
+post_rzansel_results_to_cdash:
+  extends: [.post_results_to_cdash]
+  allow_failure: true
+  rules:
+    - if: $ENABLE_CDASH == "true" && $ENABLED_CLUSTERS =~ /rzansel/ && $GITLAB_USER_LOGIN =~ $RZANSEL_USERS && $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: always
+    - if: $ENABLE_CDASH == "true" && $ENABLED_CLUSTERS =~ /rzansel/ && $GITLAB_USER_LOGIN =~ $RZANSEL_USERS && $CI_PIPELINE_SOURCE == "schedule"
+      when: always
+    - if: $ENABLE_CDASH == "true" && $ENABLED_CLUSTERS =~ /rzansel/ && $GITLAB_USER_LOGIN =~ $RZANSEL_USERS && $CI_PIPELINE_SOURCE == "web"
+      when: always
+    - if: $ENABLE_CDASH == "true" && $ENABLED_CLUSTERS =~ /rzansel/ && $GITLAB_USER_LOGIN =~ $RZANSEL_USERS && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+      when: always
+  needs:
+    - rzansel_spectrummpi_cuda_volta_gcc
+    - rzansel_spectrummpi_cuda_volta_xl
+
+# collect job results from RZvernal and submit them to CDash
+post_rzvernal_results_to_cdash:
+  extends: [.post_results_to_cdash]
+  allow_failure: true
+  rules:
+    - if: $ENABLE_CDASH == "true" && $ENABLED_CLUSTERS =~ /rzvernal/ && $GITLAB_USER_LOGIN =~ $RZVERNAL_USERS && $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: always
+    - if: $ENABLE_CDASH == "true" && $ENABLED_CLUSTERS =~ /rzvernal/ && $GITLAB_USER_LOGIN =~ $RZVERNAL_USERS && $CI_PIPELINE_SOURCE == "schedule"
+      when: always
+    - if: $ENABLE_CDASH == "true" && $ENABLED_CLUSTERS =~ /rzvernal/ && $GITLAB_USER_LOGIN =~ $RZVERNAL_USERS && $CI_PIPELINE_SOURCE == "web"
+      when: always
+    - if: $ENABLE_CDASH == "true" && $ENABLED_CLUSTERS =~ /rzvernal/ && $GITLAB_USER_LOGIN =~ $RZVERNAL_USERS && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+      when: always
+  needs:
+    - rzvernal_craympich_rocm_mi250_cce
+
+# collect job results from RZadams and submit them to CDash
+post_rzadams_results_to_cdash:
+  extends: [.post_results_to_cdash]
+  allow_failure: true
+  rules:
+    - if: $ENABLE_CDASH == "true" && $ENABLED_CLUSTERS =~ /rzadams/ && $GITLAB_USER_LOGIN =~ $RZADAMS_USERS && $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: always
+    - if: $ENABLE_CDASH == "true" && $ENABLED_CLUSTERS =~ /rzadams/ && $GITLAB_USER_LOGIN =~ $RZADAMS_USERS && $CI_PIPELINE_SOURCE == "schedule"
+      when: always
+    - if: $ENABLE_CDASH == "true" && $ENABLED_CLUSTERS =~ /rzadams/ && $GITLAB_USER_LOGIN =~ $RZADAMS_USERS && $CI_PIPELINE_SOURCE == "web"
+      when: always
+    - if: $ENABLE_CDASH == "true" && $ENABLED_CLUSTERS =~ /rzadams/ && $GITLAB_USER_LOGIN =~ $RZADAMS_USERS && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+      when: always
+  needs:
+    - rzadams_craympich_rocm_mi300_cce
diff --git a/.gitlab/common.yml b/.gitlab/common.yml
@@ -25,7 +25,7 @@ default:
     - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_PIPELINE_SOURCE == "schedule"
     - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_PIPELINE_SOURCE == "pipeline"
     - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_PIPELINE_SOURCE == "web"
-    - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_PIPELINE_SOURCE == "push"
+    - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
 
 .darwin_regular_mr_only_job:
   variables:
@@ -40,7 +40,6 @@ default:
     - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_PIPELINE_SOURCE == "merge_request_event"
     - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_PIPELINE_SOURCE == "pipeline"
     - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_PIPELINE_SOURCE == "web"
-    - if: $ENABLED_CLUSTERS =~ /darwin/ && $CI_PIPELINE_SOURCE == "push"
 
 .darwin_regular_default_branch_only_job:
   variables:
@@ -74,13 +73,13 @@ default:
     - if: $ENABLED_CLUSTERS =~ /rocinante/ && $CI_PIPELINE_SOURCE == "merge_request_event"
     - if: $ENABLED_CLUSTERS =~ /rocinante/ && $CI_PIPELINE_SOURCE == "schedule"
     - if: $ENABLED_CLUSTERS =~ /rocinante/ && $CI_PIPELINE_SOURCE == "web"
-    - if: $ENABLED_CLUSTERS =~ /rocinante/ && $CI_PIPELINE_SOURCE == "push"
+    - if: $ENABLED_CLUSTERS =~ /rocinante/ && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
 
 .venado_job:
   allow_failure: true
   variables:
     CLUSTER: venado
-    SCHEDULER_PARAMETERS: "-N 1 -p standard --time=02:00:00"
+    SCHEDULER_PARAMETERS: "-N 1 -A lanl_ai_g -p gpu --time=02:00:00"
   tags:
     - venado
     - batch
@@ -100,7 +99,7 @@ default:
     - if: $ENABLED_CLUSTERS =~ /venado/ && $GITLAB_USER_LOGIN =~ $VENADO_USERS && $CI_PIPELINE_SOURCE == "merge_request_event"
     - if: $ENABLED_CLUSTERS =~ /venado/ && $GITLAB_USER_LOGIN =~ $VENADO_USERS && $CI_PIPELINE_SOURCE == "schedule"
     - if: $ENABLED_CLUSTERS =~ /venado/ && $GITLAB_USER_LOGIN =~ $VENADO_USERS && $CI_PIPELINE_SOURCE == "web"
-    - if: $ENABLED_CLUSTERS =~ /venado/ && $GITLAB_USER_LOGIN =~ $VENADO_USERS && $CI_PIPELINE_SOURCE == "push"
+    - if: $ENABLED_CLUSTERS =~ /venado/ && $GITLAB_USER_LOGIN =~ $VENADO_USERS && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
 
 .rzansel_job:
   allow_failure: true
@@ -118,7 +117,7 @@ default:
     - if: $ENABLED_CLUSTERS =~ /rzansel/ && $GITLAB_USER_LOGIN =~ $RZANSEL_USERS && $CI_PIPELINE_SOURCE == "merge_request_event"
     - if: $ENABLED_CLUSTERS =~ /rzansel/ && $GITLAB_USER_LOGIN =~ $RZANSEL_USERS && $CI_PIPELINE_SOURCE == "schedule"
     - if: $ENABLED_CLUSTERS =~ /rzansel/ && $GITLAB_USER_LOGIN =~ $RZANSEL_USERS && $CI_PIPELINE_SOURCE == "web"
-    - if: $ENABLED_CLUSTERS =~ /rzansel/ && $GITLAB_USER_LOGIN =~ $RZANSEL_USERS && $CI_PIPELINE_SOURCE == "push"
+    - if: $ENABLED_CLUSTERS =~ /rzansel/ && $GITLAB_USER_LOGIN =~ $RZANSEL_USERS && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
 
 .rzvernal_job:
   allow_failure: true
@@ -137,7 +136,7 @@ default:
     - if: $ENABLED_CLUSTERS =~ /rzvernal/ && $GITLAB_USER_LOGIN =~ $RZVERNAL_USERS && $CI_PIPELINE_SOURCE == "merge_request_event"
     - if: $ENABLED_CLUSTERS =~ /rzvernal/ && $GITLAB_USER_LOGIN =~ $RZVERNAL_USERS && $CI_PIPELINE_SOURCE == "schedule"
     - if: $ENABLED_CLUSTERS =~ /rzvernal/ && $GITLAB_USER_LOGIN =~ $RZVERNAL_USERS && $CI_PIPELINE_SOURCE == "web"
-    - if: $ENABLED_CLUSTERS =~ /rzvernal/ && $GITLAB_USER_LOGIN =~ $RZVERNAL_USERS && $CI_PIPELINE_SOURCE == "push"
+    - if: $ENABLED_CLUSTERS =~ /rzvernal/ && $GITLAB_USER_LOGIN =~ $RZVERNAL_USERS && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
 
 .rzadams_job:
   allow_failure: true
@@ -156,7 +155,7 @@ default:
     - if: $ENABLED_CLUSTERS =~ /rzadams/ && $GITLAB_USER_LOGIN =~ $RZADAMS_USERS && $CI_PIPELINE_SOURCE == "merge_request_event"
     - if: $ENABLED_CLUSTERS =~ /rzadams/ && $GITLAB_USER_LOGIN =~ $RZADAMS_USERS && $CI_PIPELINE_SOURCE == "schedule"
     - if: $ENABLED_CLUSTERS =~ /rzadams/ && $GITLAB_USER_LOGIN =~ $RZADAMS_USERS && $CI_PIPELINE_SOURCE == "web"
-    - if: $ENABLED_CLUSTERS =~ /rzadams/ && $GITLAB_USER_LOGIN =~ $RZADAMS_USERS && $CI_PIPELINE_SOURCE == "push"
+    - if: $ENABLED_CLUSTERS =~ /rzadams/ && $GITLAB_USER_LOGIN =~ $RZADAMS_USERS && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
 
 .build_and_test:
   stage: build_n_test