Merge pull request #67 from oracle-quickstart/daemonset

Add DaemonSet deployment support
oracle-quickstart · Apr 24, 2024 · c4bc4ed · c4bc4ed
2 parents 95f21d4 + 0dfc950
commit c4bc4ed
Show file tree

Hide file tree

Showing 13 changed files with 253 additions and 13 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Change Log
 
+## 2024-04-22
+### Added
+- Support for Management Agent Daemonset deployment
+
 ## 2024-03-08
 ### Added
 - Support for AWS EKS system and control plane logs collection.

diff --git a/charts/mgmt-agent/Chart.yaml b/charts/mgmt-agent/Chart.yaml
@@ -18,7 +18,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 3.0.1
+version: 3.0.2
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/charts/mgmt-agent/README.md b/charts/mgmt-agent/README.md
@@ -20,17 +20,34 @@ A Helm chart for collecting Kubernetes Metrics using OCI Management Agent into O
 | kubernetesCluster.compartmentId | string | `nil` | OCI Compartment Id to push Kubernetes Monitoring metrics. If not specified default is same as Agent compartment |
 | kubernetesCluster.name | string | `nil` | Kubernetes cluster name |
 | kubernetesCluster.namespace | string | `"*"` | Kubernetes cluster namespace(s) to monitor. This can be a comma-separated list of namespaces or '*' to monitor all the namespaces |
+| kubernetesCluster.monitoringNamespace | string | `nil` | OCI namespace to push Kubernetes Monitoring metrics. The namespace should match the pattern '^[a-z][a-z0-9_]*[a-z0-9]$'. By default metrics will be pushed to 'mgmtagent_kubernetes_metrics' |
+| kubernetesCluster.overrideAllowMetricsAPIServer | string | `nil` | Provide the specific list of comma separated metric names for agent computed metrics to be collected. |
+| kubernetesCluster.overrideAllowMetricsCluster | string | `nil` | Provide the specific list of comma separated metric names for agent computed metrics to be collected |
+| kubernetesCluster.overrideAllowMetricsKubelet | string | `nil` | Provide the specific list of comma separated metric names for Kubelet (/api/v1/nodes/<node_name>/proxy/metrics) metrics to be collected |
+| kubernetesCluster.overrideAllowMetricsNode | string | `nil` | Provide the specific list of comma separated metric names for Node (/api/v1/nodes/<node_name>/proxy/metrics/resource, /api/v1/nodes/<node_name>/proxy/metrics/cadvisor) metrics to be collected |
 | mgmtagent.image.secret | string | `nil` | Image secrets to use for pulling container image (base64 encoded content of ~/.docker/config.json file) |
 | mgmtagent.image.url | string | `nil` | Replace this value with actual docker image URL for Management Agent |
 | mgmtagent.installKey | string | `"resources/input.rsp"` | Copy the downloaded Management Agent Install Key file under root helm directory as resources/input.rsp |
-| mgmtagent.installKeyFileContent | string | `nil` | Provide the base64 encoded content of the Management Agent Install Key file |
+| mgmtagent.installKeyFileContent | string | `nil` | Provide the base64 encoded content of the Management Agent Install Key file (e.g. `cat input.rsp \| base64 -w 0`) |
 | namespace | string | `"{{ .Values.global.namespace }}"` | Kubernetes namespace to create and install this helm chart in |
 | oci-onm-common.createNamespace | bool | `true` | If createNamespace is set to true, it tries to create the namespace defined in 'namespace' variable. |
 | oci-onm-common.createServiceAccount | bool | `true` | By default, a cluster role, cluster role binding and serviceaccount will be created for the monitoring pods to be able to (readonly) access various objects within the cluster, to support collection of various telemetry data. You may set this to false and provide your own serviceaccount (in the parent chart(s)) which has the necessary cluster role(s) binded to it. Refer, README for the cluster role definition and other details. |
 | oci-onm-common.namespace | string | `"{{ .Values.global.namespace }}"` | Kubernetes Namespace in which the serviceaccount to be created. |
 | oci-onm-common.resourceNamePrefix | string | `"{{ .Values.global.resourceNamePrefix }}"` | Prefix to be attached to resources created through this chart. Not all resources may have this prefix. |
 | oci-onm-common.serviceAccount | string | `"{{ .Values.global.resourceNamePrefix }}"` | Name of the Kubernetes ServiceAccount |
 | serviceAccount | string | `"{{ .Values.global.resourceNamePrefix }}"` | Name of the Kubernetes ServiceAccount |
+| deployment.security.runAsUser | integer | `1000` | Processes in the Container will use the specified user ID |
+| deployment.security.runAsGroup | integer | `2000` | Processes in the Container will use the specified group ID |
+| deployment.security.fsGroup | integer | `2000` | Files created in the Container will use the specified group ID |
+| deployment.cleanupEpochTime | integer | `nil` | Please provide the current epoch time in seconds (Eg: Executing the following command in a bash shell will provide the epoch time: "date +%s") to clean up the agent installation directory from previous deployment |
+| deployment.daemonSetDeployment | bool | `false` | Setting the daemonset deployment to true, will deploy the Management Agents as a daemonset in addition to deploying the Management Agent as a statefulset. This is done to to distribute the node metrics collection to agents running on the node |
+| deployment.daemonSet.hostPath | string | `nil` |  The host path to store data, if Agent is deployed as DaemonSet. Management Agent Pod should have read-write access to it |
+| deployment.resource.request.cpuCore | string | `200m` | Minimum CPU cores(millicore) for each agent instance |
+| deployment.resource.request.memory | string | `500Mi` | Minimum memory(mebibytes) for each agent instance |
+| deployment.resource.request.storage | string | `2Gi` | Minimum storage(gibibyte) for StatefulSet's PVC |
+| deployment.resource.limit.cpuCore | string | `500m` | Maximum CPU cores(millicore) for each agent instance |
+| deployment.resource.limit.memory | string | `1Gi` | Maximum memory(gibibyte) for each agent instance |
+| deployment.storageClass | string | `nil` | The storage class for StatefulSet's PVC. If not provided then the Cluster's default storage class will be used |
 
 ----------------------------------------------
 Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0)
diff --git a/charts/mgmt-agent/templates/agent-configmap.yaml b/charts/mgmt-agent/templates/agent-configmap.yaml
@@ -0,0 +1,11 @@
+# Copyright (c) 2024, Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v1.0 as shown at https://oss.oracle.com/licenses/upl.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "mgmt-agent.resourceNamePrefix" . }}-agent
+  namespace: {{ include "mgmt-agent.namespace" . }}
+data:
+  emd.properties: |
+    # provide the key=value pair below to override
diff --git a/charts/mgmt-agent/templates/env-configmap.yaml b/charts/mgmt-agent/templates/env-configmap.yaml
@@ -0,0 +1,20 @@
+# Copyright (c) 2024, Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v1.0 as shown at https://oss.oracle.com/licenses/upl.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "mgmt-agent.resourceNamePrefix" . }}-env
+  namespace: {{ include "mgmt-agent.namespace" . }}
+data:
+  # Unmodifiable properties
+  # Please do not remove/modify the below properties
+  DAEMONSET_DEPLOYMENT: "{{ .Values.deployment.daemonSetDeployment }}"
+  HELM_DEPLOYMENT: "true"
+  INSTALL_NAMESPACE: "{{ include "mgmt-agent.namespace" . }}"
+  {{- if .Values.deployment.cleanupEpochTime }}
+  POD_CLEANUP_ID: "{{ .Values.deployment.cleanupEpochTime }}"
+  {{- end }}
+
+  # Modifiable properties
+  # Add new or change the below existing properties as required
diff --git a/charts/mgmt-agent/templates/metrics-configmap.yaml b/charts/mgmt-agent/templates/metrics-configmap.yaml
@@ -14,3 +14,23 @@ data:
     clusterName={{ include "mgmt-agent.kubernetesClusterName" . }}
     # Kubernetes Namespace to monitor
     kubernetesNamespace={{ .Values.kubernetesCluster.namespace }}
+    {{- if .Values.kubernetesCluster.monitoringNamespace }}
+    # namespace to push monitoring metrics
+    monitoringNamespace={{ .Values.kubernetesCluster.monitoringNamespace }}
+    {{- end }}
+    {{- if .Values.kubernetesCluster.overrideAllowMetricsAPIServer }}
+    # list of comma separated metric names for API server (/metrics) metrics
+    overrideAllowMetricsAPIServer={{ .Values.kubernetesCluster.overrideAllowMetricsAPIServer }}
+    {{- end }}
+    {{- if .Values.kubernetesCluster.overrideAllowMetricsCluster }}
+    # list of comma separated metric names for agent computed metrics
+    overrideAllowMetricsCluster={{ .Values.kubernetesCluster.overrideAllowMetricsCluster }}
+    {{- end }}
+    {{- if .Values.kubernetesCluster.overrideAllowMetricsKubelet }}
+    # list of comma separated metric names for Kubelet (/api/v1/nodes/<node_name>/proxy/metrics) metrics
+    overrideAllowMetricsKubelet={{ .Values.kubernetesCluster.overrideAllowMetricsKubelet }}
+    {{- end }}
+    {{- if .Values.kubernetesCluster.overrideAllowMetricsNode }}
+    # list of comma separated metric names for Node (/api/v1/nodes/<node_name>/proxy/metrics/resource, /api/v1/nodes/<node_name>/proxy/metrics/cadvisor) metrics
+    overrideAllowMetricsNode={{ .Values.kubernetesCluster.overrideAllowMetricsNode }}
+    {{- end }}
diff --git a/charts/mgmt-agent/templates/mgmt-agent-daemonset.yaml b/charts/mgmt-agent/templates/mgmt-agent-daemonset.yaml
@@ -0,0 +1,83 @@
+# Copyright (c) 2024, Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v1.0 as shown at https://oss.oracle.com/licenses/upl.
+
+{{- if .Values.deployment.daemonSetDeployment }}
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: {{ include "mgmt-agent.resourceNamePrefix" . }}-mgmt-agent
+  namespace: {{ include "mgmt-agent.namespace" . }}
+  labels:
+    app: {{ include "mgmt-agent.resourceNamePrefix" . }}-mgmt-agent
+spec:
+  selector:
+    matchLabels:
+      app: {{ include "mgmt-agent.resourceNamePrefix" . }}-mgmt-agent
+  template:
+    metadata:
+      annotations:
+        checksum/metrics-configmap: {{ include (print $.Template.BasePath "/metrics-configmap.yaml") . | sha256sum }}
+        checksum/agent-configmap: {{ include (print $.Template.BasePath "/agent-configmap.yaml") . | sha256sum }}
+        checksum/env-configmap: {{ include (print $.Template.BasePath "/env-configmap.yaml") . | sha256sum }}
+      labels:
+        app: {{ include "mgmt-agent.resourceNamePrefix" . }}-mgmt-agent
+    spec:
+      securityContext:
+        runAsUser: {{ default 0 .Values.deployment.security.runAsUser }}
+        runAsGroup: {{ default 0 .Values.deployment.security.runAsGroup }}
+        fsGroup: {{ default 0 .Values.deployment.security.fsGroup }}
+      serviceAccountName: {{ include "mgmt-agent.serviceAccount" . }}
+      {{- if .Values.mgmtagent.image.secret }}
+      imagePullSecrets:
+        - name: {{ include "mgmt-agent.resourceNamePrefix" . }}-mgmt-agent-container-registry-key
+      {{- end }}
+      restartPolicy: Always
+      containers:
+        - name: {{ include "mgmt-agent.resourceNamePrefix" . }}-mgmt-agent
+          image: {{ .Values.mgmtagent.image.url }}
+          envFrom:
+          - configMapRef:
+              name: {{ include "mgmt-agent.resourceNamePrefix" . }}-env
+          resources:
+            requests:
+              cpu: {{ .Values.deployment.resource.request.cpuCore }}
+              memory: {{ .Values.deployment.resource.request.memory }}
+            limits:
+              cpu: {{ .Values.deployment.resource.limit.cpuCore }}
+              memory: {{ .Values.deployment.resource.limit.memory }}
+          volumeMounts:
+            - name: mgmtagent-secret
+              mountPath: /opt/oracle/mgmtagent_secret
+              readOnly: true
+            - name: mgmtagent-hostpath
+              mountPath: /opt/oracle
+            - name: mgmtagent-config
+              mountPath: /opt/oracle/mgmtagent_config
+            - name: mgmtagent-env
+              mountPath: /opt/oracle/mgmtagent_env
+            - name: mgmtagent-agent-config
+              mountPath: /opt/oracle/mgmtagent_agent_config
+            - mountPath: /tmp
+              name: tmp
+          securityContext:
+           allowPrivilegeEscalation: false
+           readOnlyRootFilesystem: true
+      volumes:
+        - name: mgmtagent-secret
+          secret:
+            secretName: {{ include "mgmt-agent.resourceNamePrefix" . }}-mgmt-agent-rsp
+        - name: mgmtagent-config
+          configMap:
+            name: {{ include "mgmt-agent.resourceNamePrefix" . }}-metrics
+        - name: mgmtagent-env
+          configMap:
+            name: {{ include "mgmt-agent.resourceNamePrefix" . }}-env
+        - name: mgmtagent-agent-config
+          configMap:
+            name: {{ include "mgmt-agent.resourceNamePrefix" . }}-agent
+        - name: mgmtagent-hostpath
+          hostPath: 
+            path: {{ required "deployment.daemonSet.hostPath is required" .Values.deployment.daemonSet.hostPath }}
+        - emptyDir: {}
+          name: tmp
+{{- end }}
diff --git a/charts/mgmt-agent/templates/mgmt-agent-statefulset.yaml b/charts/mgmt-agent/templates/mgmt-agent-statefulset.yaml
@@ -16,6 +16,10 @@ spec:
       app: {{ include "mgmt-agent.resourceNamePrefix" . }}-mgmt-agent
   template:
     metadata:
+      annotations:
+        checksum/metrics-configmap: {{ include (print $.Template.BasePath "/metrics-configmap.yaml") . | sha256sum }}
+        checksum/agent-configmap: {{ include (print $.Template.BasePath "/agent-configmap.yaml") . | sha256sum }}
+        checksum/env-configmap: {{ include (print $.Template.BasePath "/env-configmap.yaml") . | sha256sum }}
       labels:
         app: {{ include "mgmt-agent.resourceNamePrefix" . }}-mgmt-agent
     spec:
@@ -24,12 +28,17 @@ spec:
         runAsGroup: {{ default 0 .Values.deployment.security.runAsGroup }}
         fsGroup: {{ default 0 .Values.deployment.security.fsGroup }}
       serviceAccountName: {{ include "mgmt-agent.serviceAccount" . }}
+      {{- if .Values.mgmtagent.image.secret }}
       imagePullSecrets:
         - name: {{ include "mgmt-agent.resourceNamePrefix" . }}-mgmt-agent-container-registry-key
+      {{- end }}
       restartPolicy: Always
       containers:
         - name: {{ include "mgmt-agent.resourceNamePrefix" . }}-mgmt-agent
           image: {{ .Values.mgmtagent.image.url }}
+          envFrom:
+          - configMapRef:
+              name: {{ include "mgmt-agent.resourceNamePrefix" . }}-env
           resources:
             requests:
               cpu: {{ .Values.deployment.resource.request.cpuCore }}
@@ -45,6 +54,10 @@ spec:
               mountPath: /opt/oracle
             - name: mgmtagent-config
               mountPath: /opt/oracle/mgmtagent_config
+            - name: mgmtagent-env
+              mountPath: /opt/oracle/mgmtagent_env
+            - name: mgmtagent-agent-config
+              mountPath: /opt/oracle/mgmtagent_agent_config
             - mountPath: /tmp
               name: tmp
           securityContext:
@@ -57,6 +70,12 @@ spec:
         - name: mgmtagent-config
           configMap:
             name: {{ include "mgmt-agent.resourceNamePrefix" . }}-metrics
+        - name: mgmtagent-env
+          configMap:
+            name: {{ include "mgmt-agent.resourceNamePrefix" . }}-env
+        - name: mgmtagent-agent-config
+          configMap:
+            name: {{ include "mgmt-agent.resourceNamePrefix" . }}-agent
         - emptyDir: {}
           name: tmp
   volumeClaimTemplates:

diff --git a/charts/mgmt-agent/values.schema.json b/charts/mgmt-agent/values.schema.json
@@ -109,16 +109,32 @@
             {
                 "name":
                 {
-                    "type": "string"
+                    "type":
+                    [
+                        "string",
+                        "null"
+                    ]
                 },
                 "namespace":
                 {
                     "type": "string"
+                },
+                "monitoringNamespace":
+                {
+                    "anyOf":
+                    [
+                        {
+                          "type": "string",
+                          "pattern": "^[a-z][a-z0-9_]*[a-z0-9]$"
+                        },
+                        {
+                          "type": "null"
+                        }
+                    ]
                 }
             },
             "required":
             [
-                "name",
                 "namespace"
             ]
         },
@@ -156,6 +172,32 @@
                         }
                     }
                 },
+                "cleanupEpochTime":
+                {
+                    "type":
+                    [
+                        "integer",
+                        "null"
+                    ]
+                },
+                "daemonSetDeployment":
+                {
+                    "type": "boolean"
+                },
+                "daemonSet":
+                {
+                    "properties":
+                    {
+                        "hostPath":
+                        {
+                            "type":
+                            [
+                                "string",
+                                "null"
+                            ]
+                        }
+                    }
+                },
                 "resource":
                 {
                     "properties":
@@ -205,7 +247,11 @@
                         }
                     }
                 }
-            }
+            },
+            "required": 
+            [
+                "daemonSetDeployment"
+            ]
         }
     },
     "required":

diff --git a/charts/mgmt-agent/values.yaml b/charts/mgmt-agent/values.yaml
@@ -22,7 +22,7 @@ oci-onm-common:
 mgmtagent:
    # Provide either installKeyFileContent or installKey as an install key. If both provided then installKeyFileContent will take higher precedence.
 
-   # -- Provide the base64 encoded content of the Management Agent Install Key file
+   # -- Provide the base64 encoded content of the Management Agent Install Key file (e.g. cat input.rsp | base64 -w 0)
    installKeyFileContent:
    # -- Copy the downloaded Management Agent Install Key file under root helm directory as resources/input.rsp
    installKey: resources/input.rsp
@@ -50,6 +50,16 @@ kubernetesCluster:
    name:
    # -- Kubernetes cluster namespace(s) to monitor. This can be a comma-separated list of namespaces or '*' to monitor all the namespaces
    namespace: '*'
+   # -- OCI namespace to push Kubernetes Monitoring metrics. The namespace should match the pattern '^[a-z][a-z0-9_]*[a-z0-9]$'. By default metrics will be pushed to 'mgmtagent_kubernetes_metrics'
+   monitoringNamespace:
+   # -- Provide the specific list of comma separated metric names for API server (/metrics) metrics to be collected.
+   overrideAllowMetricsAPIServer:
+   # -- Provide the specific list of comma separated metric names for agent computed metrics to be collected.
+   overrideAllowMetricsCluster:
+   # -- Provide the specific list of comma separated metric names for Kubelet (/api/v1/nodes/<node_name>/proxy/metrics) metrics to be collected.
+   overrideAllowMetricsKubelet:
+   # -- Provide the specific list of comma separated metric names for Node (/api/v1/nodes/<node_name>/proxy/metrics/resource, /api/v1/nodes/<node_name>/proxy/metrics/cadvisor) metrics to be collected.
+   overrideAllowMetricsNode:
 
 deployment:
    security:
@@ -60,6 +70,16 @@ deployment:
       # Files created in the Container will use group ID 2000, replace it with a different value if desired
       fsGroup: 2000
 
+   # Please provide the current epoch time in seconds (Eg: Executing the following command in a bash shell will provide the epoch time: "date +%s") to clean up the agent installation directory from previous deployment
+   cleanupEpochTime:
+
+   # Setting the daemonset deployment to true, will deploy the Management Agents as a daemonset in addition to deploying the Management Agent as a statefulset. This is done to to distribute the node metrics collection to agents running on the node
+   daemonSetDeployment: false
+
+   daemonSet:
+      # Provide the host path if Agent is deployed as DaemonSet. Management Agent Pod should have read-write access to it.
+      hostPath:
+
    # Provide the agent resources as per Kubernetes resource quantity
    resource:
       # Provide the minimum required resources