From 61b7443a90735c4a3e69bdee0c0a48c3442bec94 Mon Sep 17 00:00:00 2001 From: gsmith-sas <65406958+gsmith-sas@users.noreply.github.com> Date: Fri, 15 Dec 2023 18:39:11 -0500 Subject: [PATCH] Additional changes found when deploying prom-operator --- bin/common.sh | 11 ++-- .../es-exporter_container_image.template | 2 +- logging/fb/container_image.template | 2 +- .../opensearch/os_container_image.template | 2 +- .../opensearch/osd_container_image.template | 2 +- monitoring/bin/deploy_monitoring_cluster.sh | 60 ++++++++++++------- .../prom-operator_container_image.template | 30 +++++----- .../pushgateway_container_image.template | 2 +- monitoring/tempo_container_image.template | 4 +- monitoring/values-prom-operator.yaml | 50 ++++++++-------- monitoring/values-tempo.yaml | 4 +- 11 files changed, 96 insertions(+), 73 deletions(-) diff --git a/bin/common.sh b/bin/common.sh index 79673c8e..f9c330f3 100644 --- a/bin/common.sh +++ b/bin/common.sh @@ -305,15 +305,18 @@ function doitall { fi if [ "$AIRGAP_DEPLOYMENT" == "true" ]; then - REGISTRY="$AIRGAP_REGISTRY" + GLOBAL_REGISTRY="$AIRGAP_REGISTRY" + else + GLOBAL_REGISTRY="null" fi + v4m_replace "__${prefix}GLOBAL_REGISTRY__" "$GLOBAL_REGISTRY" "$imageKeysFile" v4m_replace "__${prefix}IMAGE_REGISTRY__" "$REGISTRY" "$imageKeysFile" - v4m_replace "__${prefix}GLOBAL_REGISTRY__" "$REGISTRY" "$imageKeysFile" - v4m_replace "__${prefix}IMAGE_REPO__" "$REGISTRY\/$REPOS\/$IMAGE" "$imageKeysFile" + v4m_replace "__${prefix}IMAGE_REPO_3LEVEL__" "$REGISTRY\/$REPOS\/$IMAGE" "$imageKeysFile" + v4m_replace "__${prefix}IMAGE_REPO_2LEVEL__" "$REPOS\/$IMAGE" "$imageKeysFile" v4m_replace "__${prefix}IMAGE__" "$IMAGE" "$imageKeysFile" v4m_replace "__${prefix}IMAGE_TAG__" "$VERSION" "$imageKeysFile" - v4m_replace "__${prefix}IMAGE_PULL_POLICY__" "Always" "$imageKeysFile" + v4m_replace "__${prefix}IMAGE_PULL_POLICY__" "IfNotPresent" "$imageKeysFile" v4m_replace "__${prefix}IMAGE_PULL_SECRET__" "null" "$imageKeysFile" #Handle Single Image Pull Secret v4m_replace "__${prefix}IMAGE_PULL_SECRETS__" "[]" "$imageKeysFile" #Handle Multiple Image Pull Secrets diff --git a/logging/esexporter/es-exporter_container_image.template b/logging/esexporter/es-exporter_container_image.template index 5f535303..dda79c17 100644 --- a/logging/esexporter/es-exporter_container_image.template +++ b/logging/esexporter/es-exporter_container_image.template @@ -1,5 +1,5 @@ image: - repository: __IMAGE_REPO__ + repository: __IMAGE_REPO_3LEVEL__ tag: __IMAGE_TAG__ pullPolicy: __IMAGE_PULL_POLICY__ pullSecret: __IMAGE_PULL_SECRET__ diff --git a/logging/fb/container_image.template b/logging/fb/container_image.template index d6c82547..602dbd73 100644 --- a/logging/fb/container_image.template +++ b/logging/fb/container_image.template @@ -1,5 +1,5 @@ image: - repository: __IMAGE_REPO__ + repository: __IMAGE_REPO_3LEVEL__ tag: __IMAGE_TAG__ pullPolicy: __IMAGE_PULL_POLICY__ diff --git a/logging/opensearch/os_container_image.template b/logging/opensearch/os_container_image.template index b962aa93..800c8c04 100644 --- a/logging/opensearch/os_container_image.template +++ b/logging/opensearch/os_container_image.template @@ -1,7 +1,7 @@ global: dockerRegistry: __GLOBAL_REGISTRY__ image: - repository: __IMAGE_REPO__ + repository: __IMAGE_REPO_2LEVEL__ tag: __IMAGE_TAG__ pullPolicy: __IMAGE_PULL_POLICY__ diff --git a/logging/opensearch/osd_container_image.template b/logging/opensearch/osd_container_image.template index d6c82547..602dbd73 100644 --- a/logging/opensearch/osd_container_image.template +++ b/logging/opensearch/osd_container_image.template @@ -1,5 +1,5 @@ image: - repository: __IMAGE_REPO__ + repository: __IMAGE_REPO_3LEVEL__ tag: __IMAGE_TAG__ pullPolicy: __IMAGE_PULL_POLICY__ diff --git a/monitoring/bin/deploy_monitoring_cluster.sh b/monitoring/bin/deploy_monitoring_cluster.sh index 8c797c02..e1adc177 100755 --- a/monitoring/bin/deploy_monitoring_cluster.sh +++ b/monitoring/bin/deploy_monitoring_cluster.sh @@ -52,20 +52,35 @@ if [ "$AIRGAP_DEPLOYMENT" == "true" ]; then # Check for the image pull secret for the air gap environment and replace placeholders checkForAirgapSecretInNamespace "$AIRGAP_IMAGE_PULL_SECRET_NAME" "$MON_NS" - replaceAirgapValuesInFiles "monitoring/airgap/airgap-values-prom-operator.yaml" +### replaceAirgapValuesInFiles "monitoring/airgap/airgap-values-prom-operator.yaml" + +### airgapValuesFile=$updatedAirgapValuesFile + +### if [ "$TLS_ENABLE" == "true" ]; then +### replaceAirgapValuesInFiles "monitoring/airgap/airgap-values-prom-operator-tls.yaml" +### airgapTLSValuesFile=$updatedAirgapValuesFile +### else +### airgapTLSValuesFile=$TMP_DIR/empty.yaml +### fi +###else +### airgapValuesFile=$TMP_DIR/empty.yaml +### airgapTLSValuesFile=$TMP_DIR/empty.yaml +fi - airgapValuesFile=$updatedAirgapValuesFile +###### +echo " DDDDDDDD" #DEBUGGING-REMOVE +doitall "$PROMOP_FULL_IMAGE" "monitoring/prom-operator_container_image.template" +doitall "$ALERTMANAGER_FULL_IMAGE" "TEMPFILE" "ALERTMANAGER_" +doitall "$ADMWEBHOOK_FULL_IMAGE" "TEMPFILE" "ADMWEBHOOK_" +doitall "$KSM_FULL_IMAGE" "TEMPFILE" "KSM_" +doitall "$NODEXPORT_FULL_IMAGE" "TEMPFILE" "NODEXPORT_" +doitall "$PROMETHEUS_FULL_IMAGE" "TEMPFILE" "PROMETHEUS_" +doitall "$CONFIGRELOAD_FULL_IMAGE" "TEMPFILE" "CONFIGRELOAD_" +doitall "$GRAFANA_FULL_IMAGE" "TEMPFILE" "GRAFANA_" +doitall "$GRAFANA_SIDECAR_FULL_IMAGE" "TEMPFILE" "SIDECAR_" +cat "$imageKeysFile" #DEBUGGING-REMOVE +echo " DDDDDDDD" #DEBUGGING-REMOVE - if [ "$TLS_ENABLE" == "true" ]; then - replaceAirgapValuesInFiles "monitoring/airgap/airgap-values-prom-operator-tls.yaml" - airgapTLSValuesFile=$updatedAirgapValuesFile - else - airgapTLSValuesFile=$TMP_DIR/empty.yaml - fi -else - airgapValuesFile=$TMP_DIR/empty.yaml - airgapTLSValuesFile=$TMP_DIR/empty.yaml -fi set -e log_notice "Deploying monitoring to the [$MON_NS] namespace..." @@ -211,11 +226,10 @@ log_debug "Installing Helm chart from artifact [$chart2install]" helm $helmDebug upgrade --install $promRelease \ --namespace $MON_NS \ + -f $imageKeysFile \ -f monitoring/values-prom-operator.yaml \ - -f $airgapValuesFile \ -f $istioValuesFile \ -f $tlsValuesFile \ - -f $airgapTLSValuesFile \ -f $tlsPromAlertingEndpointFile \ -f $nodePortValuesFile \ -f $wnpValuesFile \ @@ -265,13 +279,19 @@ if [ "$TRACING_ENABLE" == "true" ]; then # Check for the image pull secret for the air gap environment and replace placeholders checkForAirgapSecretInNamespace "$AIRGAP_IMAGE_PULL_SECRET_NAME" "$MON_NS" - replaceAirgapValuesInFiles "monitoring/airgap/airgap-tempo-values.yaml" - - airgapValuesFile=$updatedAirgapValuesFile - else - airgapValuesFile=$TMP_DIR/empty.yaml +### replaceAirgapValuesInFiles "monitoring/airgap/airgap-tempo-values.yaml" +### +### airgapValuesFile=$updatedAirgapValuesFile +### else +### airgapValuesFile=$TMP_DIR/empty.yaml fi + ###### + echo " DDDDDDDD" #DEBUGGING-REMOVE + doitall "$TEMPO_FULL_IMAGE" "monitoring/tempo_container_image.template" + cat "$imageKeysFile" #DEBUGGING-REMOVE + echo " DDDDDDDD" #DEBUGGING-REMOVE + # Add the grafana helm chart repo helmRepoAdd grafana https://grafana.github.io/helm-charts helm repo update @@ -284,9 +304,9 @@ if [ "$TRACING_ENABLE" == "true" ]; then log_info "Installing tempo" helm upgrade --install v4m-tempo \ -n "$MON_NS" \ + -f $imageKeysFile \ -f monitoring/values-tempo.yaml \ -f "$TEMPO_USER_YAML" \ - -f "$airgapValuesFile" \ --version "$TEMPO_CHART_VERSION" \ $chart2install fi diff --git a/monitoring/prom-operator_container_image.template b/monitoring/prom-operator_container_image.template index f832ac6b..691dc2fa 100644 --- a/monitoring/prom-operator_container_image.template +++ b/monitoring/prom-operator_container_image.template @@ -6,14 +6,14 @@ global: prometheusOperator: image: registry: __IMAGE_REGISTRY__ - repository: __IMAGE_REPO__ + repository: __IMAGE_REPO_2LEVEL__ tag: __IMAGE_TAG__ pullPolicy: __IMAGE_PULL_POLICY__ prometheusConfigReloader: image: registry: __CONFIGRELOAD_IMAGE_REGISTRY__ - repository: __CONFIGRELOAD_IMAGE_REPO__ + repository: __CONFIGRELOAD_IMAGE_REPO_2LEVEL__ tag: __CONFIGRELOAD_IMAGE_TAG__ admissionWebhooks: @@ -21,13 +21,13 @@ prometheusOperator: # deployment: # image: # registry: __ADMWEBHOOK_IMAGE_REGISTRY__ -# repository: __ADMWEBHOOK_IMAGE_REPO__ +# repository: __ADMWEBHOOK_IMAGE_REPO_2LEVEL__ # tag: __ADMWEBHOOK_IMAGE_TAG__ # pullPolicy: __ADMWEBHOOK_IMAGE_PULL_POLICY__ patch: image: registry: __ADMWEBHOOK_IMAGE_REGISTRY__ - repository: __ADMWEBHOOK_IMAGE_REPO__ + repository: __ADMWEBHOOK_IMAGE_REPO_2LEVEL__ tag: __ADMWEBHOOK_IMAGE_TAG__ pullPolicy: __ADMWEBHOOK_IMAGE_PULL_POLICY__ @@ -50,7 +50,7 @@ prometheusOperator: ## NOTE: We do NOT use Thanos # thanosImage: # registry: __THANOS_IMAGE_REGISTRY__ -# repository: __THANOS_IMAGE_REPO__ +# repository: __THANOS_IMAGE_REPO_2LEVEL__ # tag: __THANOS_IMAGE_TAG__ @@ -59,16 +59,16 @@ alertmanager: alertmanagerSpec: image: registry: __ALERTMANAGER_IMAGE_REGISTRY__ - repository: __ALERTMANAGER_IMAGE_REPO__ + repository: __ALERTMANAGER_IMAGE_REPO_2LEVEL__ tag: __ALERTMANAGER_IMAGE_TAG__ ## Grafana grafana: image: - repository: __GRAFANA_IMAGE_REPO__ + repository: __GRAFANA_IMAGE_REPO_2LEVEL__ sidecar: image: - repository: __SIDECAR_IMAGE_REPO__ + repository: __SIDECAR_IMAGE_REPO_2LEVEL__ ## Kube-State-Metrics kube-state-metrics: @@ -78,20 +78,20 @@ kube-state-metrics: ### imagePullSecrets: __IMAGE_PULL_SECRETS__ image: registry: __KSM_IMAGE_REGISTRY__ - repository: __KSM_IMAGE_REPO__ + repository: __KSM_IMAGE_REPO_2LEVEL__ tag: __KSM_IMAGE_TAG__ - pullPolicy: __KSM_PULL_POLICY__ - imagePullSecrets: __KSM_PULL_SECRETS__ + pullPolicy: __KSM_IMAGE_PULL_POLICY__ + imagePullSecrets: __KSM_IMAGE_PULL_SECRETS__ ##Node Exporter -nodeExporter: +prometheus-node-exporter: ### TO DO: Use these globals(node exporter)? ###global: ### imageRegistry: __GLOBAL_REGISTRY__ ### imagePullSecrets: __IMAGE_PULL_SECRETS__ image: registry: __NODEXPORT_IMAGE_REGISTRY__ - repository: __NODEXPORT_IMAGE_REPO__ + repository: __NODEXPORT_IMAGE_REPO_2LEVEL__ tag: __NODEXPORT_IMAGE_TAG__ pullPolicy: __NODEXPORT_IMAGE_PULL_POLICY__ imagePullSecrets: __IMAGE_PULL_SECRETS__ @@ -101,7 +101,7 @@ prometheus: prometheusSpec: image: registry: __PROMETHEUS_IMAGE_REGISTRY__ - repository: __PROMETHEUS_IMAGE_REPO__ + repository: __PROMETHEUS_IMAGE_REPO_2LEVEL__ tag: __PROMETHEUS_IMAGE_TAG__ ##ThanosRuller @@ -110,5 +110,5 @@ prometheus: # thanosRulerSpec: # image: # registry: __THANOSRULER_IMAGE_REGISTRY__ -# repository: __THANOSRULER_IMAGE_REPO__ +# repository: __THANOSRULER_IMAGE_REPO_2LEVEL__ # tag: __THANOSRULER_IMAGE_TAG__ diff --git a/monitoring/pushgateway_container_image.template b/monitoring/pushgateway_container_image.template index 4139eee4..c0791e01 100644 --- a/monitoring/pushgateway_container_image.template +++ b/monitoring/pushgateway_container_image.template @@ -1,5 +1,5 @@ image: - repository: __IMAGE_REPO__ + repository: __IMAGE_REPO_3LEVEL__ tag: __IMAGE_TAG__ pullPolicy: __IMAGE_PULL_POLICY__ pullSecrets: __IMAGE_PULL_SECRETS__ diff --git a/monitoring/tempo_container_image.template b/monitoring/tempo_container_image.template index 6f4bc095..8b64a5af 100644 --- a/monitoring/tempo_container_image.template +++ b/monitoring/tempo_container_image.template @@ -1,12 +1,12 @@ tempo: - repository: __IMAGE_REPO__ + repository: __IMAGE_REPO_3LEVEL__ tag: __IMAGE_TAG__ pullPolicy: __IMAGE_PULL_POLICY__ pullSecrets: __IMAGE_PULL_SECRETS__ #NOTE: Tempo Query is not used #tempoQuery: -# repository: __TEMPOQ_IMAGE_REPO__ +# repository: __TEMPOQ_IMAGE_REPO_3LEVEL__ # tag: __TEMPOQ_IMAGE_TAG__ # pullPolicy: __TEMPOQ_IMAGE_PULL_POLICY__ # pullSecrets: __TEMPOQ_IMAGE_PULL_SECRETS__ diff --git a/monitoring/values-prom-operator.yaml b/monitoring/values-prom-operator.yaml index 737b8535..80470a7a 100644 --- a/monitoring/values-prom-operator.yaml +++ b/monitoring/values-prom-operator.yaml @@ -17,10 +17,10 @@ commonLabels: # =================== # https://github.com/coreos/prometheus-operator prometheusOperator: - image: - # registry: quay.io - # repository: prometheus-operator/prometheus-operator - tag: v0.69.1 +### image: +### # registry: quay.io +### # repository: prometheus-operator/prometheus-operator +### tag: v0.69.1 logFormat: json logLevel: info createCustomResource: false @@ -36,11 +36,11 @@ prometheusOperator: requests: cpu: "50m" memory: "25Mi" - prometheusConfigReloader: - image: - # registry: quay.io - # repository: prometheus-operator/prometheus-config-reloader - tag: v0.69.1 +### prometheusConfigReloader: +### image: +### # registry: quay.io +### # repository: prometheus-operator/prometheus-config-reloader +### tag: v0.69.1 # ====================== # kubelet ServiceMonitor @@ -65,8 +65,8 @@ kubeStateMetrics: # https://github.com/kubernetes/kube-state-metrics # https://github.com/helm/charts/tree/master/stable/kube-state-metrics kube-state-metrics: - image: - tag: v2.10.0 +### image: +### tag: v2.10.0 resources: requests: cpu: "25m" @@ -86,8 +86,8 @@ prometheus: type: ClusterIP nodePort: null prometheusSpec: - image: - tag: v2.47.1 +### image: +### tag: v2.47.1 logLevel: info logFormat: json podAntiAffinity: soft @@ -126,8 +126,8 @@ alertmanager: type: ClusterIP nodePort: null alertmanagerSpec: - image: - tag: v0.26.0 +### image: +### tag: v0.26.0 logFormat: json podAntiAffinity: soft retention: 240h @@ -150,10 +150,10 @@ alertmanager: # https://github.com/prometheus/node_exporter # https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-node-exporter prometheus-node-exporter: - image: - # registry: quay.io - # repository: prometheus/node-exporter - tag: v1.7.0 +### image: +### # registry: quay.io +### # repository: prometheus/node-exporter +### tag: v1.7.0 service: # Override the default port of 9100 to avoid potential conflicts port: 9110 @@ -183,8 +183,8 @@ prometheus-node-exporter: # ======= # https://github.com/grafana/helm-charts/tree/main/charts/grafana grafana: - image: - tag: 10.2.1 +### image: +### tag: 10.2.1 initChownData: enabled: false "grafana.ini": @@ -219,10 +219,10 @@ grafana: requests: cpu: "50m" memory: "100Mi" - image: - # registry: quay.io - # repository: kiwigrid/k8s-sidecar - tag: 1.25.2 +### image: +### # registry: quay.io +### # repository: kiwigrid/k8s-sidecar +### tag: 1.25.2 deploymentStrategy: type: Recreate persistence: diff --git a/monitoring/values-tempo.yaml b/monitoring/values-tempo.yaml index c0c351ec..bca75654 100644 --- a/monitoring/values-tempo.yaml +++ b/monitoring/values-tempo.yaml @@ -5,8 +5,8 @@ tempo: searchEnabled: true - repository: grafana/tempo - tag: null +### repository: grafana/tempo +### tag: null multitenancyEnabled: false # -- If true, Tempo will report anonymous usage data about the shape of a deployment to Grafana Labs