Log ingester daemonset

kubebot-monitoring-otel/Chart.lock

@@ -5,5 +5,8 @@ dependencies:
 - name: opentelemetry-operator
   repository: https://open-telemetry.github.io/opentelemetry-helm-charts
   version: 0.9.4
-digest: sha256:c903110582a4d585f87bddec1e44fa391defa0c2a3027fa68c97c1c9d5263265
-generated: "2022-07-23T17:31:40.230397413+05:30"
+- name: filebeat
+  repository: https://helm.elastic.co
+  version: 7.17.3
+digest: sha256:daff205d35bfb25c066f4f6cbc2669eee2395cec11f7faa0d63dacef2cd58189
+generated: "2022-09-12T18:42:32.12894+05:30"

kubebot-monitoring-otel/Chart.yaml

@@ -32,3 +32,7 @@ dependencies:
     condition: opentelemetry-operator.enabled
     version: 0.9.4
     repository: https://open-telemetry.github.io/opentelemetry-helm-charts
+  - name: filebeat
+    condition: filebeat.enabled
+    version: 7.17.3
+    repository: https://helm.elastic.co

kubebot-monitoring-otel/unpacked filebeat/filebeat/.helmignore

@@ -0,0 +1,2 @@
+tests/
+.pytest_cache/

kubebot-monitoring-otel/unpacked filebeat/filebeat/Chart.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+appVersion: 7.17.3
+description: Official Elastic helm chart for Filebeat
+home: https://github.com/elastic/helm-charts
+icon: https://helm.elastic.co/icons/beats.png
+maintainers:
+- email: [email protected]
+  name: Elastic
+name: filebeat
+sources:
+- https://github.com/elastic/beats
+version: 7.17.3

kubebot-monitoring-otel/unpacked filebeat/filebeat/Makefile

@@ -0,0 +1 @@
+include ../helpers/common.mk

kubebot-monitoring-otel/unpacked filebeat/filebeat/examples/default/Makefile

@@ -0,0 +1,13 @@
+default: test
+
+include ../../../helpers/examples.mk
+
+RELEASE := helm-filebeat-default
+
+install:
+	helm upgrade --wait --timeout=$(TIMEOUT) --install $(RELEASE) ../../
+
+test: install goss
+
+purge:
+	helm del $(RELEASE)

kubebot-monitoring-otel/unpacked filebeat/filebeat/examples/default/README.md

@@ -0,0 +1,27 @@
+# Default
+
+This example deploy Filebeat 7.17.3 using [default values][].
+
+
+## Usage
+
+* Deploy [Elasticsearch Helm chart][].
+
+* Deploy Filebeat chart with the default values: `make install`
+
+* You can now setup a port forward to query Filebeat indices:
+
+  ```
+  kubectl port-forward svc/elasticsearch-master 9200
+  curl localhost:9200/_cat/indices
+  ```
+
+
+## Testing
+
+You can also run [goss integration tests][] using `make test`
+
+
+[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.17/elasticsearch/examples/default/
+[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.17/filebeat/examples/default/test/goss.yaml
+[default values]: https://github.com/elastic/helm-charts/tree/7.17/filebeat/values.yaml

kubebot-monitoring-otel/unpacked filebeat/filebeat/examples/default/test/goss.yaml

@@ -0,0 +1,47 @@
+port:
+  tcp:5066:
+    listening: true
+    ip:
+      - "127.0.0.1"
+
+mount:
+  /usr/share/filebeat/data:
+    exists: true
+  /run/docker.sock:
+    exists: true
+  /var/lib/docker/containers:
+    exists: true
+    opts:
+      - ro
+  /usr/share/filebeat/filebeat.yml:
+    exists: true
+    opts:
+      - ro
+
+user:
+  filebeat:
+    exists: true
+    uid: 1000
+    gid: 1000
+
+http:
+  http://elasticsearch-master:9200/_cat/indices:
+    status: 200
+    timeout: 2000
+    body:
+      - "filebeat-7.17.3"
+
+file:
+  /usr/share/filebeat/filebeat.yml:
+    exists: true
+    contains:
+      - "add_kubernetes_metadata"
+      - "output.elasticsearch"
+      - "elasticsearch-master:9200"
+
+command:
+  cd /usr/share/filebeat && filebeat test output:
+    exit-status: 0
+    stdout:
+      - "elasticsearch: http://elasticsearch-master:9200"
+      - "version: 7.17.3"

kubebot-monitoring-otel/unpacked filebeat/filebeat/examples/deployment/Makefile

@@ -0,0 +1,13 @@
+default: test
+
+include ../../../helpers/examples.mk
+
+RELEASE := helm-filebeat-deployment
+
+install:
+	helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../
+
+test: install goss
+
+purge:
+	helm del $(RELEASE)

kubebot-monitoring-otel/unpacked filebeat/filebeat/examples/deployment/README.md

@@ -0,0 +1,27 @@
+# Default
+
+This example deploy Filebeat 7.17.3 using [default values][] as a Kubernetes Deployment.
+
+
+## Usage
+
+* Deploy [Elasticsearch Helm chart][].
+
+* Deploy Filebeat chart with the default values: `make install`
+
+* You can now setup a port forward to query Filebeat indices:
+
+  ```
+  kubectl port-forward svc/elasticsearch-master 9200
+  curl localhost:9200/_cat/indices
+  ```
+
+
+## Testing
+
+You can also run [goss integration tests][] using `make test`
+
+
+[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/master/elasticsearch/examples/default/
+[goss integration tests]: https://github.com/elastic/helm-charts/tree/master/filebeat/examples/deployment/test/goss.yaml
+[default values]: https://github.com/elastic/helm-charts/tree/master/filebeat/values.yaml

kubebot-monitoring-otel/unpacked filebeat/filebeat/examples/deployment/test/goss.yaml

@@ -0,0 +1,6 @@
+http:
+  http://elasticsearch-master:9200/_cat/indices:
+    status: 200
+    timeout: 2000
+    body:
+      - "filebeat-7.17.3"

kubebot-monitoring-otel/unpacked filebeat/filebeat/examples/deployment/values.yaml

@@ -0,0 +1,16 @@
+deployment:
+  enabled: true
+
+daemonset:
+  enabled: false
+
+filebeatConfig:
+  filebeat.yml: |
+    filebeat.inputs:
+      - type: log 
+        paths:
+        - /usr/share/filebeat/logs/filebeat
+
+    output.elasticsearch:
+      host: '${NODE_NAME}'
+      hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}'

kubebot-monitoring-otel/unpacked filebeat/filebeat/examples/oss/Makefile

@@ -0,0 +1,13 @@
+default: test
+
+include ../../../helpers/examples.mk
+
+RELEASE := helm-filebeat-oss
+
+install:
+	helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../
+
+test: install goss
+
+purge:
+	helm del $(RELEASE)

kubebot-monitoring-otel/unpacked filebeat/filebeat/examples/oss/README.md

@@ -0,0 +1,27 @@
+# OSS
+
+This example deploy Filebeat 7.17.3 using [Filebeat OSS][] version.
+
+
+## Usage
+
+* Deploy [Elasticsearch Helm chart][].
+
+* Deploy Filebeat chart with the default values: `make install`
+
+* You can now setup a port forward to query Filebeat indices:
+
+  ```
+  kubectl port-forward svc/oss-master 9200
+  curl localhost:9200/_cat/indices
+  ```
+
+
+## Testing
+
+You can also run [goss integration tests][] using `make test`
+
+
+[filebeat oss]: https://www.elastic.co/downloads/beats/filebeat-oss
+[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.17/elasticsearch/examples/oss/
+[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.17/filebeat/examples/oss/test/goss.yaml

kubebot-monitoring-otel/unpacked filebeat/filebeat/examples/oss/test/goss.yaml

@@ -0,0 +1,22 @@
+port:
+  tcp:5066:
+    listening: true
+    ip:
+      - "127.0.0.1"
+
+mount:
+  /usr/share/filebeat/data:
+    exists: true
+
+user:
+  filebeat:
+    exists: true
+    uid: 1000
+    gid: 1000
+
+http:
+  http://elasticsearch-master:9200/_cat/indices:
+    status: 200
+    timeout: 2000
+    body:
+      - "filebeat-oss-7.17.3"

kubebot-monitoring-otel/unpacked filebeat/filebeat/examples/oss/values.yaml

@@ -0,0 +1,22 @@
+image: docker.elastic.co/beats/filebeat-oss
+
+daemonset:
+  filebeatConfig:
+    filebeat.yml: |
+      filebeat.inputs:
+      - type: container
+        paths:
+          - /var/log/containers/*.log
+        processors:
+        - add_kubernetes_metadata:
+            host: ${NODE_NAME}
+            matchers:
+            - logs_path:
+                logs_path: "/var/log/containers/"
+      output.elasticsearch:
+        host: '${NODE_NAME}'
+        hosts: "elasticsearch-master:9200"
+        index: "filebeat-oss-%{[agent.version]}-%{+yyyy.MM.dd}"
+      setup.ilm.enabled: false
+      setup.template.name: "filebeat"
+      setup.template.pattern: "filebeat-oss-*"

kubebot-monitoring-otel/unpacked filebeat/filebeat/examples/security/Makefile

@@ -0,0 +1,13 @@
+default: test
+
+include ../../../helpers/examples.mk
+
+RELEASE := helm-filebeat-security
+
+install:
+	helm upgrade --wait --timeout=$(TIMEOUT) --install --values values.yaml $(RELEASE) ../../
+
+test: install goss
+
+purge:
+	helm del $(RELEASE)

kubebot-monitoring-otel/unpacked filebeat/filebeat/examples/security/README.md

@@ -0,0 +1,28 @@
+# Security
+
+This example deploy Filebeat 7.17.3 using authentication and TLS to connect to
+Elasticsearch (see [values][]).
+
+
+## Usage
+
+* Deploy [Elasticsearch Helm chart][].
+
+* Deploy Filebeat chart with security: `make install`
+
+* You can now setup a port forward to query Filebeat indices:
+
+  ```
+  kubectl port-forward svc/security-master 9200
+  curl -u elastic:changeme https://localhost:9200/_cat/indices
+  ```
+
+
+## Testing
+
+You can also run [goss integration tests][] using `make test`
+
+
+[elasticsearch helm chart]: https://github.com/elastic/helm-charts/tree/7.17/elasticsearch/examples/security/
+[goss integration tests]: https://github.com/elastic/helm-charts/tree/7.17/filebeat/examples/security/test/goss.yaml
+[values]: https://github.com/elastic/helm-charts/tree/7.17/filebeat/examples/security/values.yaml

kubebot-monitoring-otel/unpacked filebeat/filebeat/examples/security/test/goss.yaml

@@ -0,0 +1,9 @@
+http:
+  https://security-master:9200/_cat/indices:
+    status: 200
+    timeout: 2000
+    body:
+      - "filebeat-7.17.3"
+    allow-insecure: true
+    username: "{{ .Env.ELASTICSEARCH_USERNAME }}"
+    password: "{{ .Env.ELASTICSEARCH_PASSWORD }}"

kubebot-monitoring-otel/unpacked filebeat/filebeat/examples/security/values.yaml

@@ -0,0 +1,37 @@
+filebeatConfig:
+  filebeat.yml: |
+    filebeat.inputs:
+    - type: container
+      paths:
+        - /var/log/containers/*.log
+      processors:
+      - add_kubernetes_metadata: 
+          host: ${NODE_NAME}
+          matchers:
+          - logs_path:
+              logs_path: "/var/log/containers/"
+
+    output.elasticsearch:
+      username: '${ELASTICSEARCH_USERNAME}'
+      password: '${ELASTICSEARCH_PASSWORD}'
+      protocol: https
+      hosts: ["security-master:9200"]
+      ssl.certificate_authorities:
+        - /usr/share/filebeat/config/certs/elastic-certificate.pem
+
+secretMounts:
+  - name: elastic-certificate-pem
+    secretName: elastic-certificate-pem
+    path: /usr/share/filebeat/config/certs
+
+extraEnvs:
+  - name: 'ELASTICSEARCH_USERNAME'
+    valueFrom:
+      secretKeyRef:
+        name: elastic-credentials
+        key: username
+  - name: 'ELASTICSEARCH_PASSWORD'
+    valueFrom:
+      secretKeyRef:
+        name: elastic-credentials
+        key: password

kubebot-monitoring-otel/unpacked filebeat/filebeat/examples/upgrade/Makefile

@@ -0,0 +1,17 @@
+default: test
+
+include ../../../helpers/examples.mk
+
+CHART := filebeat
+RELEASE := helm-filebeat-upgrade
+# K8S 1.22 doesn't support anymore rbac.authorization.k8s.io/v1beta1 used in 7.9.0
+FROM := 7.10.0
+
+install:
+	../../../helpers/upgrade.sh --chart $(CHART) --release $(RELEASE) --from $(FROM)
+	kubectl rollout status daemonset $(RELEASE)-filebeat
+
+test: install goss
+
+purge:
+	helm del $(RELEASE)