Feat vector cloudwatch ingest

clusters/mycluster-0/flux-system/gotk-sync.yaml

@@ -0,0 +1,27 @@
+# This manifest was generated by flux. DO NOT EDIT.
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: GitRepository
+metadata:
+  name: flux-system
+  namespace: flux-system
+spec:
+  interval: 1m0s
+  ref:
+    branch: feat_vector_cloudwatch_ingest
+  secretRef:
+    name: flux-system
+  url: ssh://[email protected]/Smana/demo-secured-eks.git
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: flux-system
+  namespace: flux-system
+spec:
+  interval: 10m0s
+  path: ./clusters/mycluster-0
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system

clusters/mycluster-0/flux-system/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- gotk-components.yaml
+- gotk-sync.yaml

clusters/mycluster-0/observability.yaml

@@ -29,6 +29,10 @@ spec:
       kind: HelmRelease
       name: loki
       namespace: observability
+    - apiVersion: helm.toolkit.fluxcd.io/v1beta1
+      kind: HelmRelease
+      name: vector-cloudwatch
+      namespace: observability
     - apiVersion: helm.toolkit.fluxcd.io/v1beta1
       kind: HelmRelease
       name: vector-agent

infrastructure/base/crossplane/providers/kustomization.yaml

@@ -3,5 +3,8 @@ kind: Kustomization
 
 resources:
   - controller-config.yaml
+  - provider-cloudwatchlogs.yaml
+  - provider-firehose.yaml
   - provider-iam.yaml
+  - provider-kinesis.yaml
   - provider-s3.yaml

infrastructure/base/crossplane/providers/provider-cloudwatchlogs.yaml

@@ -0,0 +1,8 @@
+apiVersion: pkg.crossplane.io/v1
+kind: Provider
+metadata:
+  name: provider-aws-cloudwatchlogs
+spec:
+  package: xpkg.upbound.io/upbound/provider-aws-cloudwatchlogs:v0.43.1
+  controllerConfigRef:
+    name: aws-config

infrastructure/base/crossplane/providers/provider-firehose.yaml

@@ -0,0 +1,8 @@
+apiVersion: pkg.crossplane.io/v1
+kind: Provider
+metadata:
+  name: provider-aws-firehose
+spec:
+  package: xpkg.upbound.io/upbound/provider-aws-firehose:v0.43.1
+  controllerConfigRef:
+    name: aws-config

infrastructure/base/crossplane/providers/provider-kinesis.yaml

@@ -0,0 +1,8 @@
+apiVersion: pkg.crossplane.io/v1
+kind: Provider
+metadata:
+  name: provider-aws-kinesis
+spec:
+  package: xpkg.upbound.io/upbound/provider-aws-kinesis:v0.43.1
+  controllerConfigRef:
+    name: aws-config

observability/base/vector-cloudwatch/cloudwatch-destination.yaml

@@ -0,0 +1,15 @@
+apiVersion: cloudwatchlogs.aws.upbound.io/v1beta1
+kind: Destination
+metadata:
+  labels:
+    ogenki.io/id: "vector-stream"
+  name: xplane-vector
+spec:
+  forProvider:
+    region: ${region}
+    roleArnSelector:
+      matchLabels:
+        ogenki.io/id: "vector-stream-firehose"
+    targetArnSelector:
+      matchLabels:
+        ogenki.io/id: "vector-stream"

observability/base/vector-cloudwatch/cloudwatch-log-subscriptionfilter.yaml

@@ -0,0 +1,19 @@
+apiVersion: cloudwatchlogs.aws.upbound.io/v1beta1
+kind: SubscriptionFilter
+metadata:
+  labels:
+    ogenki.io/id: "vector-stream"
+  name: xplane-vector
+spec:
+  forProvider:
+    destinationArnSelector:
+      matchLabels:
+        ogenki.io/id: "vector-stream"
+    distribution: Random
+    filterPattern: ""
+    logGroupName: xplane-vector
+    name: Destination
+    region: ${region}
+    roleArnSelector:
+      matchLabels:
+        ogenki.io/id: "vector-stream-subscription"

observability/base/vector-cloudwatch/cloudwatch-loggroup.yaml

@@ -0,0 +1,11 @@
+apiVersion: cloudwatchlogs.aws.upbound.io/v1beta1
+kind: Group
+metadata:
+  name: "xplane-vector"
+  labels:
+    ogenki.io/id: "vector-stream"
+spec:
+  forProvider:
+    region: ${region}
+    tags:
+      Environment: ${environment}

observability/base/vector-cloudwatch/cloudwatch-logstream.yaml

@@ -0,0 +1,13 @@
+apiVersion: cloudwatchlogs.aws.upbound.io/v1beta1
+kind: Stream
+metadata:
+  labels:
+    ogenki.io/id: "vector-stream"
+  name: xplane-vector
+spec:
+  forProvider:
+    logGroupNameSelector:
+      matchLabels:
+        ogenki.io/id: "vector-stream"
+    name: HttpEndpointDelivery
+    region: ${region}

observability/base/vector-cloudwatch/firehose-deliverystream.yaml

@@ -0,0 +1,32 @@
+apiVersion: firehose.aws.upbound.io/v1beta1
+kind: DeliveryStream
+metadata:
+  labels:
+    ogenki.io/id: "vector-stream"
+  name: xplane-vector
+spec:
+  forProvider:
+    destination: extended_s3
+    extendedS3Configuration:
+      - bucketArnSelector:
+          matchLabels:
+            ogenki.io/id: "vector-stream"
+        roleArnSelector:
+          matchLabels:
+            ogenki.io/id: "vector-stream-firehose"
+    httpEndpointConfiguration:
+      - accessKeySecretRef:
+          key: oidc_issuer_host
+          name: eks-mycluster-0-vars
+          namespace: crossplane-system
+        url: https://vector-cloudwatch-${cluster_name}.${domain_name}
+        retryDuration: 300
+        cloudwatchLoggingOptions:
+          - enabled: true
+            logGroupName: "xplane-vector"
+            logStreamName: "xplane-vector"
+        requestConfiguration:
+          - contentEncoding: GZIP
+        s3BackupMode: FailedDataOnly
+    name: xplane-vector
+    region: ${region}

observability/base/vector-cloudwatch/firehose-iam-role.yaml

@@ -0,0 +1,55 @@
+apiVersion: iam.aws.upbound.io/v1beta1
+kind: Role
+metadata:
+  labels:
+    ogenki.io/id: "vector-stream-firehose"
+  name: xplane-firehose-vector
+spec:
+  forProvider:
+    assumeRolePolicy: |
+      {
+        "Statement": {
+          "Effect": "Allow",
+          "Principal": {
+            "Service": "firehose.amazonaws.com"
+          },
+          "Action": "sts:AssumeRole",
+          "Condition": {
+            "StringEquals": {
+              "sts:ExternalId": "${aws_account_id}"
+            }
+          }
+        }
+      }
+    inlinePolicy:
+      - name: xplane-firehose-vector-s3
+        policy: |
+          {
+              "Version": "2012-10-17",
+              "Statement": [
+                  {
+                      "Effect": "Allow",
+                      "Action": [
+                          "s3:AbortMultipartUpload",
+                          "s3:GetBucketLocation",
+                          "s3:GetObject",
+                          "s3:ListBucket",
+                          "s3:ListBucketMultipartUploads",
+                          "s3:PutObject"
+                      ],
+                      "Resource": [
+                          "arn:aws:s3:::eu-west-3-ogenki-vector-stream",
+                          "arn:aws:s3:::eu-west-3-ogenki-vector-stream/*"
+                      ]
+                  },
+                  {
+                      "Effect": "Allow",
+                      "Action": [
+                          "logs:PutLogEvents"
+                      ],
+                      "Resource": [
+                          "arn:aws:logs:eu-west-3:396740644681:log-group:vector:*"
+                      ]
+                  }
+              ]
+          }

observability/base/vector-cloudwatch/helmrelease.yaml

@@ -0,0 +1,68 @@
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: vector-cloudwatch
+spec:
+  releaseName: vector-cloudwatch
+  chart:
+    spec:
+      chart: vector
+      sourceRef:
+        kind: HelmRepository
+        name: vector
+        namespace: observability
+      version: "0.26.0"
+  interval: 10m0s
+  timeout: 30m
+  install:
+    remediation:
+      retries: 3
+  values:
+    fullnameOverride: "vector-cloudwatch"
+    role: "Aggregator"
+    customConfig:
+      data_dir: /vector-data-dir
+      api:
+        enabled: true
+        address: 127.0.0.1:8686
+        playground: false
+      sources:
+        firehose:
+          type: aws_kinesis_firehose
+          address: "0.0.0.0:8080"
+          access_keys: ["${oidc_issuer_host}"] # Using oidc issuer as string to share with the Firehose access key
+        internal_metrics:
+          type: internal_metrics
+      transforms:
+        parse:
+          type: remap
+          inputs: ["firehose"]
+          drop_on_error: false
+          source: |-
+            parsed = parse_aws_cloudwatch_log_subscription_message!(.message)
+            . = unnest(parsed.log_events)
+            . = map_values(.) -> |value| {
+              event = del(value.log_events)
+              value |= event
+              message = del(.message)
+              . |= object!(parse_json!(message))
+            }
+      sinks:
+        prom_exporter:
+          type: prometheus_exporter
+          inputs: [internal_metrics]
+          address: 0.0.0.0:9090
+        loki:
+          type: loki
+          inputs: [kubernetes_logs]
+          endpoint: http://loki-gateway
+          encoding:
+            codec: json
+          labels:
+            env: dev
+
+    # Configure a PodMonitor for Vector, requires the PodMonitor CRD to be installed.
+    podMonitor:
+      enabled: true
+      additionalLabels:
+        prometheus-instance: main

observability/base/vector-cloudwatch/httproute.yaml

@@ -0,0 +1,18 @@
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: HTTPRoute
+metadata:
+  name: vector-cloudwatch
+spec:
+  parentRefs:
+    - name: platform-public
+      namespace: infrastructure
+  hostnames:
+    - "vector-cloudwatch-${cluster_name}.${domain_name}"
+  rules:
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /
+      backendRefs:
+        - name: vector-cloudwatch
+          port: 8080

observability/base/vector-cloudwatch/kustomization.yaml

@@ -0,0 +1,15 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: observability
+
+resources:
+  - cloudwatch-destination.yaml
+  - cloudwatch-log-subscriptionfilter.yaml
+  - cloudwatch-loggroup.yaml
+  - cloudwatch-logstream.yaml
+  - firehose-deliverystream.yaml
+  - firehose-iam-role.yaml
+  - httproute.yaml
+  - helmrelease.yaml
+  - s3-bucket.yaml
+  - subscription-iam-role.yaml

observability/base/vector-cloudwatch/s3-bucket.yaml

@@ -0,0 +1,11 @@
+apiVersion: s3.aws.upbound.io/v1beta1
+kind: Bucket
+metadata:
+  name: vector-cloudwatch
+  labels:
+    ogenki.io/id: "vector-stream"
+  annotations:
+    crossplane.io/external-name: ${region}-ogenki-vector-stream
+spec:
+  forProvider:
+    region: ${region}

observability/base/vector-cloudwatch/subscription-iam-role.yaml

@@ -0,0 +1,28 @@
+apiVersion: iam.aws.upbound.io/v1beta1
+kind: Role
+metadata:
+  labels:
+    ogenki.io/id: "vector-stream-subscription"
+  name: xplane-subscription-vector
+spec:
+  forProvider:
+    assumeRolePolicy: |
+      {
+        "Statement": {
+          "Effect": "Allow",
+          "Principal": { "Service": "logs.${region}.amazonaws.com" },
+          "Action": "sts:AssumeRole"
+        }
+      }
+    inlinePolicy:
+      - name: xplane-cloudwatch-vector-kinesis
+        policy: |
+          {
+              "Statement":[
+                {
+                  "Effect":"Allow",
+                  "Action":["firehose:*"],
+                  "Resource":["arn:aws:firehose:${region}:${aws_account_id}:deliverystream/vector-stream"]
+                }
+              ]
+          }

observability/mycluster-0/kustomization.yaml

@@ -4,4 +4,5 @@ kind: Kustomization
 resources:
   - ../base/kube-prometheus-stack
   - ../base/loki
+  - ../base/vector-cloudwatch
   - ../base/vector-agent