Changes to CI to make caching work when running on different machines…

… (in one pipeline run with same platform jobs)
GAMS-dev · Aug 15, 2024 · c8ecb8b · c8ecb8b
1 parent 6cc8812
commit c8ecb8b
Show file tree

Hide file tree

Showing 7 changed files with 46 additions and 36 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,6 +1,5 @@
 stages:
   - fetch-scripts
-  - install-gams
   - build
   - test
   - pack
@@ -9,10 +8,9 @@ stages:
 
 include:
   - project: 'devel/ciscripts'
-    file: .install-jobs.yml
+    file: '.fetch-job.yml'
   - project: 'devel/ciscripts'
     file: .shared-vars.yml
-  - 'ci/.gitlab-ci-06-gams.yml'
   - 'ci/.gitlab-ci-09-build.yml'
   - 'ci/.gitlab-ci-12-test.yml'
   - 'ci/.gitlab-ci-15-pack.yml'

diff --git a/ci/.gitlab-ci-06-gams.yml b/ci/.gitlab-ci-06-gams.yml
diff --git a/ci/.gitlab-ci-09-build.yml b/ci/.gitlab-ci-09-build.yml
@@ -4,14 +4,16 @@ build-dac:
   variables:
     qtpath: /Users/gitlab/Qt/${QT_VERSION}/macos/bin
   script:
+    - !reference [.get-gams]
+    - !reference [.gams-folder-dac]
     - printf "GAMS_DISTRIB=$HOME/cache/gams-installs/`cat gams_folder_dac.txt`\n" > gamsinclude.pri
     - printf "GAMS_DISTRIB_C_API=\$\$GAMS_DISTRIB/apifiles/C/api\n" >> gamsinclude.pri
     - printf "GAMS_DISTRIB_CPP_API=\$\$GAMS_DISTRIB/apifiles/C++/api\n" >> gamsinclude.pri
     - mkdir build && cd build
     - GAMS_CORE_PATH="${GAMS_FOR_MACOS}"
     - PATH="${qtpath}:${GAMS_FOR_MACOS}:${PATH}"
     - qmake ../gams-mii.pro CONFIG+=release && make -j4
-  needs: [install-gams-dac]
+  needs: [fetch-ci-scripts]
   artifacts:
     name: mii-dac
     expire_in: 2 hours
@@ -23,29 +25,33 @@ build-deg:
   variables:
     qtpath: /Users/gitlab/Qt/${QT_VERSION}/macos/bin
   script:
+    - !reference [.get-gams]
+    - !reference [.gams-folder-deg]
     - printf "GAMS_DISTRIB=$HOME/cache/gams-installs/`cat gams_folder_deg.txt`\n" > gamsinclude.pri
     - printf "GAMS_DISTRIB_C_API=\$\$GAMS_DISTRIB/apifiles/C/api\n" >> gamsinclude.pri
     - printf "GAMS_DISTRIB_CPP_API=\$\$GAMS_DISTRIB/apifiles/C++/api\n" >> gamsinclude.pri
     - mkdir build && cd build
     - GAMS_CORE_PATH="${GAMS_FOR_MACOS}"
     - PATH="${qtpath}:${GAMS_FOR_MACOS}:${PATH}"
     - qmake ../gams-mii.pro CONFIG+=release && make -j4
-  needs: [install-gams-deg]
+  needs: [fetch-ci-scripts]
   artifacts:
     name: mii-deg
     expire_in: 2 hours
     paths: [build]
 
 build-leg:
   stage: build
-  tags: [puma]
+  tags: [linux]
   image:
     name: $GAMS_CONTAINER_REGISTRY/qt-machines/leg/${QT_IMAGE_VERSION}:latest
     entrypoint: [""]   # prevent startup.sh
-  needs: [install-gams-leg]
+  needs: [fetch-ci-scripts]
   variables:
     DOLLAR_GAMS: $$$$GAMS_DISTRIB
   script:
+    - !reference [.get-gams]
+    - !reference [.gams-folder-leg]
     - echo "GAMS_DISTRIB=/cache/gams-installs/`cat gams_folder_leg.txt`" > gamsinclude.pri
     - echo "GAMS_DISTRIB_C_API=$DOLLAR_GAMS/apifiles/C/api" >> gamsinclude.pri
     - echo "GAMS_DISTRIB_CPP_API=$DOLLAR_GAMS/apifiles/C++/api" >> gamsinclude.pri
@@ -58,13 +64,16 @@ build-leg:
 
 build-wei:
   stage: build
-  tags: [pond]
-  needs: [install-gams-wei]
+  tags: [windows]
+  needs: [fetch-ci-scripts]
   variables:
      cmd_vcvars: C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat
   image:
     name: $GAMS_CONTAINER_REGISTRY/qt-machines/wei/${QT_IMAGE_VERSION}:latest
   script:
+    - $env:Path = "C:\Miniconda\Library\bin;C:\Miniconda\DLLs;" + $env:Path
+    - !reference [.get-gams-wei]
+    - !reference [.gams-folder-wei]
     - '& $cmd_vcvars'
     - $data = 
       "GAMS_DISTRIB=C:/Cache/gams-installs/$(Get-Content mygmsdir.tmp -Raw)",
@@ -86,13 +95,16 @@ build-wei-debug:
     - if: $PF_WEI_DEBUG == "0"
       when: never
     - when: on_success
-  tags: [pond]
-  needs: [install-gams-wei]
+  tags: [windows]
+  needs: [fetch-ci-scripts]
   variables:
      cmd_vcvars: C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat
   image:
     name: $GAMS_CONTAINER_REGISTRY/qt-machines/wei/${QT_IMAGE_VERSION}:latest
   script:
+    - $env:Path = "C:\Miniconda\Library\bin;C:\Miniconda\DLLs;" + $env:Path
+    - !reference [.get-gams-wei]
+    - !reference [.gams-folder-wei]
     - '& $cmd_vcvars'
     - $data = 
       "GAMS_DISTRIB=C:/Cache/gams-installs/$(Get-Content mygmsdir.tmp -Raw)",

diff --git a/ci/.gitlab-ci-12-test.yml b/ci/.gitlab-ci-12-test.yml
@@ -2,10 +2,12 @@ test-dac:
   stage: test
   tags: [macos-arm64]
   allow_failure: true
-  dependencies: [install-gams-dac,build-dac]
+  dependencies: [fetch-ci-scripts,build-dac]
   variables:
     qtpath: /Users/gitlab/Qt/${QT_VERSION}/macos/bin
   script:
+    - !reference [.get-gams]
+    - !reference [.gams-folder-dac]
     - GAMS_PATH=$HOME/cache/gams-installs/`cat gams_folder_dac.txt`
     - PATH="${qtpath}:${GAMS_PATH}:${PATH}"
     - cd build && mkdir reports
@@ -29,10 +31,12 @@ test-deg:
   stage: test
   tags: [macos]
   allow_failure: true
-  dependencies: [install-gams-deg,build-deg]
+  dependencies: [fetch-ci-scripts,build-deg]
   variables:
     qtpath: /Users/gitlab/Qt/${QT_VERSION}/macos/bin
   script:
+    - !reference [.get-gams]
+    - !reference [.gams-folder-deg]
     - GAMS_PATH=$HOME/cache/gams-installs/`cat gams_folder_deg.txt`
     - PATH="${qtpath}:${GAMS_PATH}:${PATH}"
     - cd build && mkdir reports
@@ -54,13 +58,15 @@ test-deg:
 
 test-leg:
   stage: test
-  tags: [puma]
+  tags: [linux]
   allow_failure: true
   image:
     name: $GAMS_CONTAINER_REGISTRY/qt-machines/leg/${QT_IMAGE_VERSION}:latest
     entrypoint: [""]   # prevent startup.sh
-  dependencies: [install-gams-leg,build-leg]
+  dependencies: [fetch-ci-scripts,build-leg]
   script:
+    - !reference [.get-gams]
+    - !reference [.gams-folder-leg]
     - GAMS_PATH=/cache/gams-installs/`cat gams_folder_leg.txt`
     - cd build && mkdir reports
     - REPORTS_DIR=$(pwd)/reports
@@ -81,12 +87,15 @@ test-leg:
 
 test-wei:
   stage: test
-  tags: [pond]
+  tags: [windows]
   allow_failure: true
   image:
     name: $GAMS_CONTAINER_REGISTRY/qt-machines/wei/${QT_IMAGE_VERSION}:latest
-  dependencies: [install-gams-wei,build-wei]
+  dependencies: [fetch-ci-scripts,build-wei]
   script:
+    - $env:Path = "C:\Miniconda\Library\bin;C:\Miniconda\DLLs;" + $env:Path
+    - !reference [.get-gams-wei]
+    - !reference [.gams-folder-wei]
     - $gmsdirname = Get-Content mygmsdir.tmp -Raw
     - $GAMS_PATH = "C:\Cache\gams-installs\$gmsdirname"
     - $WORKSPACE = $([System.IO.Directory]::GetCurrentDirectory())

diff --git a/ci/.gitlab-ci-15-pack.yml b/ci/.gitlab-ci-15-pack.yml
@@ -100,7 +100,7 @@ pack-deg:
 
 pack-leg:
   stage: pack
-  tags: [puma]
+  tags: [linux]
   image:
     name: $GAMS_CONTAINER_REGISTRY/qt-machines/leg/${QT_IMAGE_VERSION}:latest
     entrypoint: [""]   # prevent startup.sh
@@ -126,7 +126,7 @@ pack-leg:
 
 pack-wei:
   stage: pack
-  tags: [pond]
+  tags: [windows]
   image:
     name: $GAMS_CONTAINER_REGISTRY/qt-machines/wei/${QT_IMAGE_VERSION}:latest
   needs: [build-wei]
@@ -167,7 +167,7 @@ pack-wei-debug:
     - if: $PF_WEI_DEBUG == "0"
       when: never
     - when: always
-  tags: [pond]
+  tags: [windows]
   image:
     name: $GAMS_CONTAINER_REGISTRY/qt-machines/wei/${QT_IMAGE_VERSION}:latest
   needs: [build-wei-debug]

diff --git a/ci/.gitlab-ci-20-analyze.yml b/ci/.gitlab-ci-20-analyze.yml
@@ -1,14 +1,16 @@
 codechecker-leg:
   stage: analyze
-  tags: [puma]
+  tags: [linux]
   image:
     name: $GAMS_CONTAINER_REGISTRY/qt-machines/leg/analyzer:latest
     entrypoint: [""]   # prevent startup.sh
-  needs: [install-gams-leg]
+  needs: [fetch-ci-scripts]
   allow_failure: false
   variables:
     DOLLAR_GAMS: $$$$GAMS_DISTRIB
   script:
+    - !reference [.get-gams]
+    - !reference [.gams-folder-leg]
     - echo "GAMS_DISTRIB=/cache/gams-installs/`cat gams_folder_leg.txt`" > gamsinclude.pri
     - echo "GAMS_DISTRIB_C_API=$DOLLAR_GAMS/apifiles/C/api" >> gamsinclude.pri
     - echo "GAMS_DISTRIB_CPP_API=$DOLLAR_GAMS/apifiles/C++/api" >> gamsinclude.pri
@@ -23,7 +25,7 @@ codechecker-leg:
 
 license-update-leg:
   stage: analyze
-  tags: [puma]
+  tags: [linux]
   image:
     name: $GAMS_CONTAINER_REGISTRY/qt-machines/deployer:latest
     entrypoint: [""]   # prevent startup.sh

diff --git a/ci/.gitlab-ci-24-deploy.yml b/ci/.gitlab-ci-24-deploy.yml
@@ -1,6 +1,6 @@
 deploy-cloudfront:
   stage: deploy
-  tags: [puma]
+  tags: [linux]
   image:
     name: $GAMS_CONTAINER_REGISTRY/qt-machines/deployer:latest
     entrypoint: [""]   # prevent startup.sh
@@ -15,7 +15,7 @@ deploy-cloudfront:
 
 deploy-github:
   stage: deploy
-  tags: [puma]
+  tags: [linux]
   image:
     name: $GAMS_CONTAINER_REGISTRY/qt-machines/deployer:latest
     entrypoint: [""]   # prevent startup.sh