monit - use taskworker pypi image as base docker image for monitoring…

… scripts (#8505)

* monit - use crabtaskworker tw image

* monit - use crabtaskworker tw image - remove old images

* monit - use crabtaskworker tw image - apply changes suggested by wa

.gitlab-ci.yml

@@ -8,7 +8,7 @@ variables:
 
 .default_rules:
   default:
-    - if: $CI_COMMIT_TAG =~ /pypi-.*/         # pypi-devthree-1714418922
+    - if: $CI_COMMIT_TAG =~ /pypi-.*/         # pypi-(prod|preprod|test*)-1714418922
   release:
     - if: $CI_COMMIT_TAG =~ /v3\.[0-9]{6}.*/  # same as above
 
@@ -100,13 +100,52 @@ build_tw_image:
         - src/python/TaskWorker/__init__.py
       policy: pull
 
+build_monit_image:
+  rules:
+    - !reference [.default_rules, default]
+    - !reference [.default_rules, release]
+  stage: build_docker
+  needs: ["build_tw_image"]
+  image:
+    name: gcr.io/kaniko-project/executor:v1.14.0-debug
+    entrypoint: [""]
+  script:
+    - echo "{\"auths\":{\"$CMSCRAB_REGISTRY_URL\":{\"auth\":\"$(echo -n $CMSCRAB_REGISTRY_USER:$CMSCRAB_REGISTRY_PASSWORD | base64)\"}}}" > /kaniko/.docker/config.json
+    - cat /kaniko/.docker/config.json
+    - /kaniko/executor
+      --context "${CI_PROJECT_DIR}"
+      --dockerfile "${CI_PROJECT_DIR}/cicd/monit_pypi/Dockerfile"
+      --destination "registry.cern.ch/cmscrab/crabtaskworker:${IMAGE_TAG}.monit"
+      --build-arg "BASE_TAG=${IMAGE_TAG}"
+  cache:
+    - key: $CI_PIPELINE_ID
+      paths:
+        - src/python/TaskWorker/__init__.py
+      policy: pull
+
+# if release, then tag monit image with `v3.latest.monit`
+tag_monit_latest:
+  rules:
+    - !reference [.default_rules, release]
+  stage: build_docker
+  needs: ["build_monit_image"]
+  image:
+    name: registry.cern.ch/cmscrab/buildtools
+    entrypoint: [""]
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - crane auth login -u ${CMSCRAB_REGISTRY_USER} -p ${CMSCRAB_REGISTRY_PASSWORD} ${CMSCRAB_REGISTRY_URL}
+    - crane cp registry.cern.ch/cmscrab/crabtaskworker:${IMAGE_TAG}.monit registry.cern.ch/cmscrab/crabtaskworker:v3.latest.monit
+
 deploy_server:
   rules:
     - if: $SKIP_DEPLOY
       when: never
     - !reference [.default_rules, default]
     - !reference [.default_rules, release]
   stage: deploy
+  needs: ["build_rest_image"]
   image:
     name: registry.cern.ch/cmscrab/buildtools
     entrypoint: [""]

cicd/monit_pypi/Dockerfile

@@ -0,0 +1,29 @@
+ARG BASE_TAG=latest
+FROM registry.cern.ch/cmscrab/crabtaskworker:${BASE_TAG}
+
+RUN pip install --user pandas 
+
+RUN mkdir -p /data/srv/monit/
+COPY ./src/python/ServerUtilities.py           \
+     ./src/python/RESTInteractions.py          \
+     ./src/script/Monitor/GenerateMONIT.py     \
+     ./scripts/Utils/CheckTapeRecall.py        \
+     ./src/script/Monitor/aso_metrics_ora.py   \
+     ./src/script/Monitor/ReportRecallQuota.py \
+     /data/srv/monit
+
+ENV PYTHONPATH=/data/srv/current/lib/python/site-packages:${PYTHONPATH:-}
+
+ENV RUCIO_HOME=/cvmfs/cms.cern.ch/rucio/current/
+ENV RUCIO_ACCOUNT="crab_server"
+# no need to source a rucio environment from cvmfs. rucio client is already installed
+# via pip, do not pollute the python env with things you do not need!
+# ENV PYTHONPATH=$PYTHONPATH:/cvmfs/cms.cern.ch/rucio/x86_64/el9/py3/current/lib/python3.9/site-packages/
+# ENV PYTHONPATH=$PYTHONPATH:/cvmfs/cms.cern.ch/rucio/x86_64/el8/py3/current/lib/python3.6/site-packages/
+
+ENTRYPOINT ["tini", "--"]
+
+CMD python3 /data/srv/monit/GenerateMONIT.py
+# CMD python3.8 /data/srv/monit/CheckTapeRecall.py
+    # cp -v RecallRules.html /data/eos/RecallRules-docker.html
+    ## we decided to send data to opensearch instead of saving it to eos. no need to copy the file.

src/script/Monitor/ReportRecallQuota.py

@@ -83,7 +83,7 @@ def main(logger):
     logger.info("rucio client initialized: %s %s", rucioClient.ping(), rucioClient.whoami() )
 
     # prepare a JSON to be sent to MONIT
-    jsonDoc = {'producer': 'crab', 'type': 'reportrecallquota', 'hostname': gethostname()}
+    jsonDoc = {'producer': MONITUSER, 'type': 'reportrecallquota', 'hostname': gethostname()}
     # get info for the used account, each will be sent with ad-hoc tag in the JSON
     accounts = [{'name': 'crab_tape_recall', 'tag': 'tape_recall_total_TB'},
                 {'name': 'crab_input',       'tag': 'crab_input_total_TB'}]

src/script/Monitor/aso_metrics_ora.py

@@ -65,7 +65,7 @@ def printData(data, header, logger=None):
     logger.info("%-4s%-15s%-10s%-10s", header)
     logger.info("-"*70)
     for row in data:
-        logger.info("%-4d%-15s%-10d%-10d", (i, row[header[1]], row[header[2]], row[header[3]]))
+        logger.info("%-4d%-15s%-10d%-10d", i, row[header[1]], row[header[2]], row[header[3]])
         i = i+1
 
     logger.info("="*70)