From 3d001e317a76d24abec0e50e8932ae4d4a889bc5 Mon Sep 17 00:00:00 2001
From: jakob-fritz <37077134+jakob-fritz@users.noreply.github.com>
Date: Mon, 24 Jun 2024 14:02:44 +0200
Subject: [PATCH] Fix Gitlab-CI (#441)

* Added Job-Token as needed by Gitlab

* Moved schedule towards nighttime

and triggered CI again, as HPC-Systems are now back

* Schedule gitlab-ci workflow at the same time as github-workflow

* Moved installation of more packages into preparation to avoid parallel access of files

* First try to use virtual envs from python

So that installed packages are available in later CI-Jobs

* Store venv on scratch

* Activate project to use SCRATCH

* Wrong usage of variables

* Try installing mpi4py-fft for later usage
---
 .github/workflows/gitlab_ci.yml |  2 +-
 .gitlab-ci.yml                  | 31 ++++++++++++++++++++++++++-----
 2 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/gitlab_ci.yml b/.github/workflows/gitlab_ci.yml
index a76d3616e0..24a3578598 100644
--- a/.github/workflows/gitlab_ci.yml
+++ b/.github/workflows/gitlab_ci.yml
@@ -7,7 +7,7 @@ on:
   pull_request_target:
     types: [opened, synchronize, reopened, labeled]
   schedule:
-    - cron: '1 5 2 * *'
+    - cron: '1 5 * * 1'
 
 jobs:
   check_permission:
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a757fb1155..196d15a6bc 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -8,12 +8,18 @@ stages:
 
 variables:
   JUWELS_ACCOUNT: "cstma"
+  JUWELS_PROJECT: "ccstma"
 
 
 prepare_JUWELS:
   stage: benchmark
   rules:
     - if: $CI_COMMIT_MESSAGE !~ /.*\[CI-no-benchmarks\]/
+  id_tokens:
+    CI_JOB_JWT:
+      aud: https://gitlab.jsc.fz-juelich.de
+    SITE_ID_TOKEN:
+      aud: https://gitlab.jsc.fz-juelich.de
   tags:
     - jacamar
     - juwels
@@ -29,8 +35,16 @@ prepare_JUWELS:
     - module load mpi4py
     - module load SciPy-Stack
     - module load CuPy
+    - jutil env activate -p ${JUWELS_PROJECT}
+    - python -m venv --clear $SCRATCH/.venv/pySDC
+    - source $SCRATCH/.venv/pySDC/bin/activate
     - pip install -e .
     - pip install pytest-benchmark coverage
+    - git submodule add -f https://github.com/brownbaerchen/mpi4py-fft.git
+    - cd mpi4py-fft
+    - git checkout cupy_implementation
+    - FFTW_LIBRARY_DIR="/p/software/juwels/stages/2024/software/FFTW/3.3.10-GCC-12.3.0/lib64" pip install --force-reinstall .
+    - cd ../
 
 
 test_JUWELS:
@@ -39,6 +53,11 @@ test_JUWELS:
     - prepare_JUWELS
   rules:
     - if: $CI_COMMIT_MESSAGE !~ /.*\[CI-no-benchmarks\]/
+  id_tokens:
+    CI_JOB_JWT:
+      aud: https://gitlab.jsc.fz-juelich.de
+    SITE_ID_TOKEN:
+      aud: https://gitlab.jsc.fz-juelich.de
   tags:
     - jacamar
     - juwels
@@ -63,11 +82,8 @@ test_JUWELS:
     - module load mpi4py
     - module load SciPy-Stack
     - module load CuPy
-    - git submodule add -f https://github.com/brownbaerchen/mpi4py-fft.git
-    - cd mpi4py-fft
-    - git checkout cupy_implementation
-    - FFTW_LIBRARY_DIR="/p/software/juwels/stages/2024/software/FFTW/3.3.10-GCC-12.3.0/lib64" pip install --force-reinstall -e .
-    - cd ../
+    - jutil env activate -p ${JUWELS_PROJECT}
+    - source $SCRATCH/.venv/pySDC/bin/activate
   script:
     # - touch benchmarks/output.json
     - echo $SYSTEMNAME
@@ -88,6 +104,11 @@ benchmark:
   when: manual
   tags:
     - docker
+  id_tokens:
+    CI_JOB_JWT:
+      aud: https://gitlab.jsc.fz-juelich.de
+    SITE_ID_TOKEN:
+      aud: https://gitlab.jsc.fz-juelich.de
   rules:
     - if: $CI_COMMIT_MESSAGE !~ /.*\[CI-no-benchmarks\]/
   artifacts: