diff --git a/README.md b/README.md
index f109c46..f7954bd 100644
--- a/README.md
+++ b/README.md
@@ -235,14 +235,14 @@ worker_groups = {
| [adot\_version](#input\_adot\_version) | The version of the AWS Distro for OpenTelemetry addon to use. | `string` | `"v0.78.0-eksbuild.1"` | no |
| [alb\_log\_bucket\_name](#input\_alb\_log\_bucket\_name) | n/a | `string` | `""` | no |
| [alb\_log\_bucket\_path](#input\_alb\_log\_bucket\_path) | ALB-INGRESS-CONTROLLER | `string` | `""` | no |
-| [api\_gateway\_resources](#input\_api\_gateway\_resources) | Nested map containing API, Stage, and VPC Link resources | list(object({
namespace = string
api = object({
name = string
protocolType = string
})
stages = optional(list(object({
name = string
namespace = string
apiRef_name = string
stageName = string
autoDeploy = bool
description = string
})))
vpc_links = optional(list(object({
name = string
namespace = string
})))
})) | n/a | yes |
+| [api\_gateway\_resources](#input\_api\_gateway\_resources) | Nested map containing API, Stage, and VPC Link resources | list(object({
namespace = string
api = object({
name = string
protocolType = string
})
stages = optional(list(object({
name = string
namespace = string
apiRef_name = string
stageName = string
autoDeploy = bool
description = string
})))
vpc_links = optional(list(object({
name = string
namespace = string
})))
})) | `[]` | no |
| [api\_gw\_deploy\_region](#input\_api\_gw\_deploy\_region) | Region in which API gatewat will be configured | `string` | `""` | no |
| [autoscaler\_image\_patch](#input\_autoscaler\_image\_patch) | The patch number of autoscaler image | `number` | `0` | no |
| [autoscaler\_limits](#input\_autoscaler\_limits) | n/a | object({
cpu = string
memory = string
}) | {
"cpu": "100m",
"memory": "600Mi"
} | no |
| [autoscaler\_requests](#input\_autoscaler\_requests) | n/a | object({
cpu = string
memory = string
}) | {
"cpu": "100m",
"memory": "600Mi"
} | no |
| [autoscaling](#input\_autoscaling) | Weather enable autoscaling or not in EKS | `bool` | `false` | no |
| [bindings](#input\_bindings) | Variable which describes group and role binding | list(object({
group = string
namespace = string
roles = list(string)
})) | `[]` | no |
-| [cluster\_enabled\_log\_types](#input\_cluster\_enabled\_log\_types) | A list of the desired control plane logs to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` | [
"audit"
]
| no |
+| [cluster\_enabled\_log\_types](#input\_cluster\_enabled\_log\_types) | A list of the desired control plane logs to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) | `list(string)` | `[]` | no |
| [cluster\_endpoint\_public\_access](#input\_cluster\_endpoint\_public\_access) | n/a | `bool` | `true` | no |
| [cluster\_name](#input\_cluster\_name) | Creating eks cluster name. | `string` | n/a | yes |
| [cluster\_version](#input\_cluster\_version) | Allows to set/change kubernetes cluster version, kubernetes version needs to be updated at leas once a year. Please check here for available versions https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html | `string` | `"1.27"` | no |
@@ -259,9 +259,7 @@ worker_groups = {
| [enable\_olm](#input\_enable\_olm) | To install OLM controller (experimental). | `bool` | `false` | no |
| [enable\_sso\_rbac](#input\_enable\_sso\_rbac) | Enable SSO RBAC integration or not | `bool` | `false` | no |
| [external\_secrets\_namespace](#input\_external\_secrets\_namespace) | The namespace of external-secret operator | `string` | `"kube-system"` | no |
-| [fluent\_bit\_name](#input\_fluent\_bit\_name) | FLUENT-BIT | `string` | `""` | no |
-| [log\_group\_name](#input\_log\_group\_name) | n/a | `string` | `""` | no |
-| [log\_retention\_days](#input\_log\_retention\_days) | n/a | `number` | `90` | no |
+| [fluent\_bit\_configs](#input\_fluent\_bit\_configs) | Fluent Bit configs | object({
fluent_bit_name = optional(string, "")
log_group_name = optional(string, "")
system_log_group_name = optional(string, "")
log_retention_days = optional(number, 90)
values_yaml = optional(string, "")
configs = optional(object({
inputs = optional(string, "")
filters = optional(string, "")
outputs = optional(string, "")
}), {})
drop_namespaces = optional(list(string), [])
log_filters = optional(list(string), [])
additional_log_filters = optional(list(string), [])
}) | {
"additional_log_filters": [
"ELB-HealthChecker",
"Amazon-Route53-Health-Check-Service"
],
"configs": {
"filters": "",
"inputs": "",
"outputs": ""
},
"drop_namespaces": [
"kube-system",
"opentelemetry-operator-system",
"adot",
"cert-manager"
],
"fluent_bit_name": "",
"log_filters": [
"kube-probe",
"health",
"prometheus",
"liveness"
],
"log_group_name": "",
"log_retention_days": 90,
"system_log_group_name": "",
"values_yaml": ""
} | no |
| [manage\_aws\_auth](#input\_manage\_aws\_auth) | n/a | `bool` | `true` | no |
| [map\_roles](#input\_map\_roles) | Additional IAM roles to add to the aws-auth configmap. | list(object({
rolearn = string
username = string
groups = list(string)
})) | `[]` | no |
| [metrics\_exporter](#input\_metrics\_exporter) | Metrics Exporter, can use cloudwatch or adot | `string` | `"cloudwatch"` | no |
diff --git a/examples/spot-instance/README.md b/examples/spot-instance/README.md
index 41252e4..0bb326b 100644
--- a/examples/spot-instance/README.md
+++ b/examples/spot-instance/README.md
@@ -12,7 +12,7 @@
| Name | Version |
|------|---------|
-| [aws](#provider\_aws) | >= 3.41 |
+| [aws](#provider\_aws) | 4.67.0 |
## Modules
diff --git a/fluent-bit.tf b/fluent-bit.tf
new file mode 100644
index 0000000..22909ac
--- /dev/null
+++ b/fluent-bit.tf
@@ -0,0 +1,43 @@
+module "fluent-bit" {
+ source = "./modules/fluent-bit"
+
+ count = var.create ? 1 : 0
+
+ account_id = local.account_id
+ region = local.region
+
+ cluster_name = module.eks-cluster[0].cluster_id
+ eks_oidc_root_ca_thumbprint = module.eks-cluster[0].eks_oidc_root_ca_thumbprint
+ oidc_provider_arn = module.eks-cluster[0].oidc_provider_arn
+
+ fluent_bit_name = try(var.fluent_bit_configs.fluent_bit_name, "") != "" ? var.fluent_bit_configs.fluent_bit_name : "${module.eks-cluster[0].cluster_id}-fluent-bit"
+ log_group_name = try(var.fluent_bit_configs.log_group_name, "") != "" ? var.fluent_bit_configs.log_group_name : "fluent-bit-cloudwatch-${module.eks-cluster[0].cluster_id}"
+ system_log_group_name = try(var.fluent_bit_configs.system_log_group_name, "")
+ log_retention_days = try(var.fluent_bit_configs.log_retention_days, 90)
+
+ values_yaml = try(var.fluent_bit_configs.values_yaml, "")
+
+ drop_namespaces = try(var.fluent_bit_configs.drop_namespaces, [
+ "kube-system",
+ "opentelemetry-operator-system",
+ "adot",
+ "cert-manager"
+ ])
+ log_filters = try(var.fluent_bit_configs.log_filters, [
+ "kube-probe",
+ "health",
+ "prometheus",
+ "liveness"
+ ])
+
+ additional_log_filters = try(var.fluent_bit_configs.additional_log_filters, [
+ "ELB-HealthChecker",
+ "Amazon-Route53-Health-Check-Service",
+ ])
+
+ fluent_bit_config = try(var.fluent_bit_configs.configs, {
+ inputs = ""
+ outputs = ""
+ filters = ""
+ })
+}
diff --git a/main.tf b/main.tf
index 454c28d..2f30f0e 100644
--- a/main.tf
+++ b/main.tf
@@ -251,22 +251,6 @@ module "alb-ingress-controller" {
# alb_log_bucket_path = var.alb_log_bucket_path != "" ? var.alb_log_bucket_path : module.eks-cluster[0].cluster_id
}
-module "fluent-bit" {
- source = "./modules/fluent-bit"
-
- count = var.create ? 1 : 0
-
- account_id = local.account_id
- region = local.region
-
- fluent_bit_name = var.fluent_bit_name != "" ? var.fluent_bit_name : "${module.eks-cluster[0].cluster_id}-fluent-bit"
- log_group_name = var.log_group_name != "" ? var.log_group_name : "fluent-bit-cloudwatch-${module.eks-cluster[0].cluster_id}"
- log_retention_days = var.log_retention_days
- cluster_name = module.eks-cluster[0].cluster_id
- eks_oidc_root_ca_thumbprint = module.eks-cluster[0].eks_oidc_root_ca_thumbprint
- oidc_provider_arn = module.eks-cluster[0].oidc_provider_arn
-}
-
module "metrics-server" {
source = "./modules/metrics-server"
diff --git a/modules/adot/tests/template_file/README.md b/modules/adot/tests/template_file/README.md
index a04381b..2935c08 100644
--- a/modules/adot/tests/template_file/README.md
+++ b/modules/adot/tests/template_file/README.md
@@ -12,7 +12,7 @@
| Name | Version |
|------|---------|
-| [helm](#provider\_helm) | n/a |
+| [helm](#provider\_helm) | 2.9.0 |
| [test](#provider\_test) | n/a |
## Modules
diff --git a/modules/fluent-bit/README.md b/modules/fluent-bit/README.md
index a61b69d..b8f9adc 100644
--- a/modules/fluent-bit/README.md
+++ b/modules/fluent-bit/README.md
@@ -47,18 +47,23 @@ No modules.
| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
| [account\_id](#input\_account\_id) | AWS Account Id to apply changes into | `string` | n/a | yes |
+| [additional\_log\_filters](#input\_additional\_log\_filters) | Fluent bit doesn't send logs if message consists of this values | `list(string)` | [
"ELB-HealthChecker",
"Amazon-Route53-Health-Check-Service"
]
| no |
| [cluster\_name](#input\_cluster\_name) | AWS EKS Cluster name. | `string` | n/a | yes |
| [create\_log\_group](#input\_create\_log\_group) | Wether or no to create log group. | `bool` | `true` | no |
| [create\_namespace](#input\_create\_namespace) | Wether or no to create namespace. | `bool` | `false` | no |
+| [drop\_namespaces](#input\_drop\_namespaces) | Flunt bit doesn't send logs for this namespaces | `list(string)` | [
"kube-system",
"opentelemetry-operator-system",
"adot",
"cert-manager"
]
| no |
| [eks\_oidc\_root\_ca\_thumbprint](#input\_eks\_oidc\_root\_ca\_thumbprint) | n/a | `string` | n/a | yes |
+| [fluent\_bit\_config](#input\_fluent\_bit\_config) | You can add other inputs,outputs and filters which module doesn't have by default | `any` | {
"filters": "",
"inputs": "",
"outputs": ""
} | no |
| [fluent\_bit\_name](#input\_fluent\_bit\_name) | Container resource name. | `string` | `"fluent-bit"` | no |
+| [log\_filters](#input\_log\_filters) | Fluent bit doesn't send logs if message consists of this values | `list(string)` | [
"kube-probe",
"health",
"prometheus",
"liveness"
]
| no |
| [log\_group\_name](#input\_log\_group\_name) | Log group name fluent-bit will be streaming logs into. | `string` | `"fluentbit-default-log-group"` | no |
| [log\_retention\_days](#input\_log\_retention\_days) | If set to a number greater than zero, and newly create log group's retention policy is set to this many days. Valid values are: [0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, 3653] | `number` | `90` | no |
| [namespace](#input\_namespace) | k8s namespace fluent-bit should be deployed into. | `string` | `"kube-system"` | no |
| [oidc\_provider\_arn](#input\_oidc\_provider\_arn) | n/a | `string` | n/a | yes |
| [region](#input\_region) | AWS Region name. | `string` | n/a | yes |
| [s3\_permission](#input\_s3\_permission) | If you want send logs to s3 you should enable s3 permission | `bool` | `false` | no |
-| [values\_yaml](#input\_values\_yaml) | Content of the values.yaml given to the helm chart. This disables the rendered values.yaml file from this module. | `string` | `null` | no |
+| [system\_log\_group\_name](#input\_system\_log\_group\_name) | Log group name fluent-bit will be streaming kube-system logs. | `string` | `""` | no |
+| [values\_yaml](#input\_values\_yaml) | Content of the values.yaml if you want override all default configs. | `string` | `""` | no |
## Outputs
diff --git a/modules/fluent-bit/locals.tf b/modules/fluent-bit/locals.tf
index 4d8beae..0c321e2 100644
--- a/modules/fluent-bit/locals.tf
+++ b/modules/fluent-bit/locals.tf
@@ -3,10 +3,18 @@ locals {
log_group_name = var.log_group_name != "" ? var.log_group_name : "fluent-bit-cloudwatch"
region = var.region
config_settings = {
- log_group_name = local.log_group_name,
- region = local.region,
- log_retention_days = var.log_retention_days
- auto_create_group = var.create_log_group ? "On" : "Off"
+ log_group_name = local.log_group_name
+ system_log_group_name = var.system_log_group_name == "" ? "${local.log_group_name}-kube" : "${var.system_log_group_name}"
+ region = local.region
+ log_retention_days = var.log_retention_days
+ auto_create_group = var.create_log_group ? "On" : "Off"
+ drop_namespaces = "(${join("|", var.drop_namespaces)})"
+ log_filters = "(${join("|", var.log_filters)})"
+ additional_log_filters = "(${join("|", var.additional_log_filters)})"
+ inputs = try(var.fluent_bit_config.inputs, "")
+ outputs = try(var.fluent_bit_config.outputs, "")
+ filters = try(var.fluent_bit_config.filters, "")
}
- values = var.values_yaml == null ? templatefile("${path.module}/values.yaml", local.config_settings) : var.values_yaml
+
+ values = var.values_yaml == "" ? templatefile("${path.module}/values.yaml.tpl", local.config_settings) : var.values_yaml
}
diff --git a/modules/fluent-bit/tests/advanced/0-setup.tf b/modules/fluent-bit/tests/advanced/0-setup.tf
new file mode 100644
index 0000000..72a3014
--- /dev/null
+++ b/modules/fluent-bit/tests/advanced/0-setup.tf
@@ -0,0 +1,21 @@
+terraform {
+ required_providers {
+ test = {
+ source = "terraform.io/builtin/test"
+ }
+
+ aws = {
+ source = "hashicorp/aws"
+ version = "~> 4.37"
+ }
+ kubernetes = {
+ source = "hashicorp/kubernetes"
+ version = "~>2.23"
+ }
+ helm = ">= 2.0"
+ }
+}
+
+provider "aws" {}
+provider "helm" {}
+provider "kubernetes" {}
diff --git a/modules/fluent-bit/tests/advanced/1-example.tf b/modules/fluent-bit/tests/advanced/1-example.tf
new file mode 100644
index 0000000..ed2d3ff
--- /dev/null
+++ b/modules/fluent-bit/tests/advanced/1-example.tf
@@ -0,0 +1,52 @@
+locals {
+ oidc_provider_arn = "arn:aws:iam::000000000000:oidc-provider/oidc.eks.eu-central-1.amazonaws.com/id/6F40EA94327Dh8956DDB9S0AE7907CFD"
+}
+
+module "fluent-bit" {
+ source = "../../"
+
+ account_id = 000000000000
+ region = "eu-central-1"
+
+ cluster_name = "Test"
+ oidc_provider_arn = local.oidc_provider_arn
+ eks_oidc_root_ca_thumbprint = replace(local.oidc_provider_arn, "/.*id//", "")
+
+
+ log_group_name = "fluent-bit"
+ system_log_group_name = "fluent-bit-kube"
+ create_log_group = true
+ log_retention_days = 7
+
+ drop_namespaces = [
+ "kube-system",
+ "opentelemetry-operator-system",
+ "adot",
+ "cert-manager"
+ ]
+
+ additional_log_filters = [
+ "ELB-HealthChecker",
+ "Amazon-Route53-Health-Check-Service",
+ ]
+
+ log_filters = [
+ "ELB-HealthChecker",
+ "Amazon-Route53-Health-Check-Service",
+ "kube-probe",
+ "health",
+ "prometheus",
+ "liveness"
+ ]
+
+ fluent_bit_config = {
+ inputs = templatefile("${path.module}/templates/inputs.yaml.tpl", {})
+ outputs = templatefile("${path.module}/templates/outputs.yaml.tpl", {})
+ filters = templatefile("${path.module}/templates/filters.yaml.tpl", {})
+ }
+
+}
+
+output "merged_inputs" {
+ value = module.fluent-bit
+}
diff --git a/modules/fluent-bit/tests/advanced/2-assert.tf b/modules/fluent-bit/tests/advanced/2-assert.tf
new file mode 100644
index 0000000..414ed51
--- /dev/null
+++ b/modules/fluent-bit/tests/advanced/2-assert.tf
@@ -0,0 +1,9 @@
+resource "test_assertions" "api_url" {
+ component = "Basic-Setup"
+
+ equal "scheme" {
+ description = "As module does not have any output and data just make sure the case runs. Probably can be thrown away."
+ got = "all good"
+ want = "all good"
+ }
+}
diff --git a/modules/fluent-bit/tests/advanced/README.md b/modules/fluent-bit/tests/advanced/README.md
new file mode 100644
index 0000000..b9d2940
--- /dev/null
+++ b/modules/fluent-bit/tests/advanced/README.md
@@ -0,0 +1,39 @@
+# basic
+
+
+## Requirements
+
+| Name | Version |
+|------|---------|
+| [aws](#requirement\_aws) | ~> 4.37 |
+| [helm](#requirement\_helm) | >= 2.0 |
+| [kubernetes](#requirement\_kubernetes) | ~>2.23 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| [test](#provider\_test) | n/a |
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| [fluent-bit](#module\_fluent-bit) | ../../ | n/a |
+
+## Resources
+
+| Name | Type |
+|------|------|
+| test_assertions.api_url | resource |
+
+## Inputs
+
+No inputs.
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| [merged\_inputs](#output\_merged\_inputs) | n/a |
+
diff --git a/modules/fluent-bit/tests/advanced/templates/filters.yaml.tpl b/modules/fluent-bit/tests/advanced/templates/filters.yaml.tpl
new file mode 100644
index 0000000..f219dff
--- /dev/null
+++ b/modules/fluent-bit/tests/advanced/templates/filters.yaml.tpl
@@ -0,0 +1,9 @@
+[FILTER]
+ Name grep
+ Match kube.*
+ Exclude $log (test)
+
+[FILTER]
+ Name grep
+ Match audit.*
+ regex $log (test)
diff --git a/modules/fluent-bit/tests/advanced/templates/inputs.yaml.tpl b/modules/fluent-bit/tests/advanced/templates/inputs.yaml.tpl
new file mode 100644
index 0000000..f61cf38
--- /dev/null
+++ b/modules/fluent-bit/tests/advanced/templates/inputs.yaml.tpl
@@ -0,0 +1,9 @@
+[INPUT]
+ Name tail
+ Tag test.*
+ Path /var/log/containers/*.log
+ Read_from_head true
+ multiline.parser docker, cri
+ Docker_Mode On
+ Parser docker
+ Mem_Buf_Limit 50MB
diff --git a/modules/fluent-bit/tests/advanced/templates/outputs.yaml.tpl b/modules/fluent-bit/tests/advanced/templates/outputs.yaml.tpl
new file mode 100644
index 0000000..4c1378a
--- /dev/null
+++ b/modules/fluent-bit/tests/advanced/templates/outputs.yaml.tpl
@@ -0,0 +1,7 @@
+[OUTPUT]
+ Name s3
+ Match test.*
+ bucket s3-bucket
+ region eu-central-1
+ total_file_size 250M
+ s3_key_format /%Y/%m/%d/%H_%M_%S.gz
diff --git a/modules/fluent-bit/values.yaml b/modules/fluent-bit/values.yaml
deleted file mode 100644
index 0eab1f5..0000000
--- a/modules/fluent-bit/values.yaml
+++ /dev/null
@@ -1,26 +0,0 @@
-config:
- ## https://docs.fluentbit.io/manual/pipeline/inputs
- inputs: |
- [INPUT]
- Name tail
- Path /var/log/containers/*.log
- multiline.parser docker, cri
- Tag kube.*
- Mem_Buf_Limit 5MB
- Skip_Long_Lines On
-
- [INPUT]
- Name systemd
- Tag host.*
- Systemd_Filter _SYSTEMD_UNIT=kubelet.service
- Read_From_Tail On
-
- outputs: |
- [OUTPUT]
- Name cloudwatch_logs
- Match *
- region ${region}
- log_group_name ${log_group_name}
- log_stream_prefix from-fluent-bit-
- auto_create_group ${auto_create_group}
- log_retention_days ${log_retention_days}
diff --git a/modules/fluent-bit/values.yaml.tpl b/modules/fluent-bit/values.yaml.tpl
new file mode 100644
index 0000000..29a1524
--- /dev/null
+++ b/modules/fluent-bit/values.yaml.tpl
@@ -0,0 +1,67 @@
+config:
+ ## https://docs.fluentbit.io/manual/pipeline/inputs
+ inputs: |
+ [INPUT]
+ Name tail
+ Tag kube.*
+ Path /var/log/containers/*.log
+ Read_from_head true
+ multiline.parser docker, cri
+ Docker_Mode On
+ Parser docker
+ Mem_Buf_Limit 50MB
+
+ [INPUT]
+ Name systemd
+ Tag host.*
+ Systemd_Filter _SYSTEMD_UNIT=kubelet.service
+ Read_From_Tail On
+
+ ${indent(4, inputs)}
+
+ ## https://docs.fluentbit.io/manual/pipeline/filters
+ filters: |
+ [FILTER]
+ Name kubernetes
+ Match kube.*
+ Merge_Log On
+ Keep_Log Off
+ K8S-Logging.Parser On
+ K8S-Logging.Exclude On
+
+ [FILTER]
+ Name grep
+ Match app.*
+ Exclude $message ${log_filters}
+
+ [FILTER]
+ Name grep
+ Match app.*
+ Exclude $message ${additional_log_filters}
+
+ [FILTER]
+ Name grep
+ Match kube.*
+ Exclude $kubernetes['namespace_name'] ${drop_namespaces}
+
+ ${indent(4, filters)}
+ outputs: |
+ [OUTPUT]
+ Name cloudwatch_logs
+ Match *
+ region ${region}
+ log_group_name ${log_group_name}
+ log_stream_prefix from-fluent-bit-
+ auto_create_group ${auto_create_group}
+ log_retention_days ${log_retention_days}
+
+ [OUTPUT]
+ Name cloudwatch_logs
+ Match kube.*
+ region ${region}
+ log_group_name ${system_log_group_name}
+ log_stream_prefix from-fluent-bit-
+ auto_create_group ${auto_create_group}
+ log_retention_days ${log_retention_days}
+
+ ${indent(4, outputs)}
diff --git a/modules/fluent-bit/variables.tf b/modules/fluent-bit/variables.tf
index 00044a3..d57ff68 100644
--- a/modules/fluent-bit/variables.tf
+++ b/modules/fluent-bit/variables.tf
@@ -45,6 +45,12 @@ variable "log_group_name" {
description = "Log group name fluent-bit will be streaming logs into."
}
+variable "system_log_group_name" {
+ type = string
+ default = ""
+ description = "Log group name fluent-bit will be streaming kube-system logs."
+}
+
variable "create_log_group" {
type = bool
default = true
@@ -62,13 +68,54 @@ variable "log_retention_days" {
}
variable "values_yaml" {
- description = "Content of the values.yaml given to the helm chart. This disables the rendered values.yaml file from this module."
- default = null
+ description = "Content of the values.yaml if you want override all default configs."
+ default = ""
type = string
}
+variable "fluent_bit_config" {
+ description = "You can add other inputs,outputs and filters which module doesn't have by default"
+ default = {
+ inputs = ""
+ outputs = ""
+ filters = ""
+ }
+ type = any
+}
+
variable "s3_permission" {
description = "If you want send logs to s3 you should enable s3 permission"
default = false
type = bool
}
+
+variable "drop_namespaces" {
+ type = list(string)
+ default = [
+ "kube-system",
+ "opentelemetry-operator-system",
+ "adot",
+ "cert-manager"
+ ]
+ description = "Flunt bit doesn't send logs for this namespaces"
+}
+
+variable "log_filters" {
+ type = list(string)
+ default = [
+ "kube-probe",
+ "health",
+ "prometheus",
+ "liveness"
+ ]
+ description = "Fluent bit doesn't send logs if message consists of this values"
+}
+
+variable "additional_log_filters" {
+ type = list(string)
+ default = [
+ "ELB-HealthChecker",
+ "Amazon-Route53-Health-Check-Service",
+ ]
+ description = "Fluent bit doesn't send logs if message consists of this values"
+}
diff --git a/tests/eks-fluent-bit/0-setup.tf b/tests/eks-fluent-bit/0-setup.tf
new file mode 100644
index 0000000..9c76587
--- /dev/null
+++ b/tests/eks-fluent-bit/0-setup.tf
@@ -0,0 +1,24 @@
+terraform {
+ required_providers {
+ test = {
+ source = "terraform.io/builtin/test"
+ }
+
+ aws = {
+ source = "hashicorp/aws"
+ version = ">= 3.41"
+ }
+ }
+
+ required_version = ">= 1.3.0"
+}
+
+/**
+ * set the following env vars so that aws provider will get authenticated before apply:
+
+ export AWS_ACCESS_KEY_ID=xxxxxxxxxxxxxxxxxxxxxxxx
+ export AWS_SECRET_ACCESS_KEY=xxxxxxxxxxxxxxxxxxxxxxxx
+*/
+provider "aws" {
+ region = "eu-central-1"
+}
diff --git a/tests/eks-fluent-bit/1-example.tf b/tests/eks-fluent-bit/1-example.tf
new file mode 100644
index 0000000..9243f26
--- /dev/null
+++ b/tests/eks-fluent-bit/1-example.tf
@@ -0,0 +1,74 @@
+# Prepare for test
+data "aws_availability_zones" "available" {}
+data "aws_vpcs" "ids" {
+ tags = {
+ Name = "default"
+ }
+}
+data "aws_subnet_ids" "subnets" {
+ vpc_id = data.aws_vpcs.ids.ids[0]
+}
+
+module "this" {
+ source = "../.."
+
+ account_id = "0000000000"
+ adot_config = {
+ "accept_namespace_regex" : "(default|kube-system)",
+ "additional_metrics" : [],
+ "log_group_name" : "adot-logs"
+ }
+ cluster_enabled_log_types = ["audit"]
+ cluster_name = "eks-dev"
+ cluster_version = "1.27"
+ metrics_exporter = "adot"
+ node_groups = {
+ "dev_nodes" : {
+ "desired_size" : 2,
+ "max_capacity" : 5,
+ "max_size" : 5,
+ "min_size" : 2
+ }
+ }
+ node_groups_default = {
+ "capacity_type" : "SPOT",
+ "instance_types" : ["t3.medium"]
+ }
+ send_alb_logs_to_cloudwatch = false
+ users = [
+ { "username" : "dasmeta" },
+ ]
+
+ vpc = {
+ link = {
+ id = data.aws_vpcs.ids.ids[0]
+ private_subnet_ids = data.aws_subnet_ids.subnets.ids
+ }
+ }
+
+ fluent_bit_configs = {
+ config = {
+ inputs = templatefile("${path.module}/templates/inputs.yaml.tpl", {})
+ outputs = templatefile("${path.module}/templates/outputs.yaml.tpl", {})
+ filters = templatefile("${path.module}/templates/filters.yaml.tpl", {})
+ }
+ drop_namespaces = [
+ "kube-system",
+ "opentelemetry-operator-system",
+ "adot",
+ "cert-manager"
+ ]
+ additional_log_filters = [
+ "ELB-HealthChecker",
+ "Amazon-Route53-Health-Check-Service",
+ ]
+ log_filters = [
+ "ELB-HealthChecker",
+ "Amazon-Route53-Health-Check-Service",
+ "kube-probe",
+ "health",
+ "prometheus",
+ "liveness"
+ ]
+ }
+}
diff --git a/tests/eks-fluent-bit/2-assert.tf b/tests/eks-fluent-bit/2-assert.tf
new file mode 100644
index 0000000..99458ca
--- /dev/null
+++ b/tests/eks-fluent-bit/2-assert.tf
@@ -0,0 +1,9 @@
+resource "test_assertions" "dummy" {
+ component = "this"
+
+ equal "scheme" {
+ description = "As module does not have any output and data just make sure the case runs. Probably can be thrown away."
+ got = "all good"
+ want = "all good"
+ }
+}
diff --git a/tests/eks-fluent-bit/README.md b/tests/eks-fluent-bit/README.md
new file mode 100644
index 0000000..5db0b33
--- /dev/null
+++ b/tests/eks-fluent-bit/README.md
@@ -0,0 +1,40 @@
+# eks-fluent-bit
+
+
+## Requirements
+
+| Name | Version |
+|------|---------|
+| [terraform](#requirement\_terraform) | >= 1.3.0 |
+| [aws](#requirement\_aws) | >= 3.41 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| [aws](#provider\_aws) | >= 3.41 |
+| [test](#provider\_test) | n/a |
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| [this](#module\_this) | ../.. | n/a |
+
+## Resources
+
+| Name | Type |
+|------|------|
+| test_assertions.dummy | resource |
+| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source |
+| [aws_subnet_ids.subnets](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet_ids) | data source |
+| [aws_vpcs.ids](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/vpcs) | data source |
+
+## Inputs
+
+No inputs.
+
+## Outputs
+
+No outputs.
+
diff --git a/tests/eks-fluent-bit/templates/filters.yaml.tpl b/tests/eks-fluent-bit/templates/filters.yaml.tpl
new file mode 100644
index 0000000..f219dff
--- /dev/null
+++ b/tests/eks-fluent-bit/templates/filters.yaml.tpl
@@ -0,0 +1,9 @@
+[FILTER]
+ Name grep
+ Match kube.*
+ Exclude $log (test)
+
+[FILTER]
+ Name grep
+ Match audit.*
+ regex $log (test)
diff --git a/tests/eks-fluent-bit/templates/inputs.yaml.tpl b/tests/eks-fluent-bit/templates/inputs.yaml.tpl
new file mode 100644
index 0000000..f61cf38
--- /dev/null
+++ b/tests/eks-fluent-bit/templates/inputs.yaml.tpl
@@ -0,0 +1,9 @@
+[INPUT]
+ Name tail
+ Tag test.*
+ Path /var/log/containers/*.log
+ Read_from_head true
+ multiline.parser docker, cri
+ Docker_Mode On
+ Parser docker
+ Mem_Buf_Limit 50MB
diff --git a/tests/eks-fluent-bit/templates/outputs.yaml.tpl b/tests/eks-fluent-bit/templates/outputs.yaml.tpl
new file mode 100644
index 0000000..4c1378a
--- /dev/null
+++ b/tests/eks-fluent-bit/templates/outputs.yaml.tpl
@@ -0,0 +1,7 @@
+[OUTPUT]
+ Name s3
+ Match test.*
+ bucket s3-bucket
+ region eu-central-1
+ total_file_size 250M
+ s3_key_format /%Y/%m/%d/%H_%M_%S.gz
diff --git a/variables.tf b/variables.tf
index 42bdc52..9108ca8 100644
--- a/variables.tf
+++ b/variables.tf
@@ -103,19 +103,52 @@ variable "alb_log_bucket_name" {
}
# FLUENT-BIT
-variable "fluent_bit_name" {
- type = string
- default = ""
-}
-
-variable "log_group_name" {
- type = string
- default = ""
-}
-variable "log_retention_days" {
- type = number
- default = 90
+variable "fluent_bit_configs" {
+ type = object({
+ fluent_bit_name = optional(string, "")
+ log_group_name = optional(string, "")
+ system_log_group_name = optional(string, "")
+ log_retention_days = optional(number, 90)
+ values_yaml = optional(string, "")
+ configs = optional(object({
+ inputs = optional(string, "")
+ filters = optional(string, "")
+ outputs = optional(string, "")
+ }), {})
+ drop_namespaces = optional(list(string), [])
+ log_filters = optional(list(string), [])
+ additional_log_filters = optional(list(string), [])
+ })
+ default = {
+ fluent_bit_name = ""
+ log_group_name = ""
+ system_log_group_name = ""
+ log_retention_days = 90
+ values_yaml = ""
+ configs = {
+ inputs = ""
+ outputs = ""
+ filters = ""
+ }
+ drop_namespaces = [
+ "kube-system",
+ "opentelemetry-operator-system",
+ "adot",
+ "cert-manager"
+ ]
+ log_filters = [
+ "kube-probe",
+ "health",
+ "prometheus",
+ "liveness"
+ ]
+ additional_log_filters = [
+ "ELB-HealthChecker",
+ "Amazon-Route53-Health-Check-Service",
+ ]
+ }
+ description = "Fluent Bit configs"
}
# METRICS-SERVER
@@ -142,7 +175,7 @@ variable "external_secrets_namespace" {
variable "cluster_enabled_log_types" {
description = "A list of the desired control plane logs to enable. For more information, see Amazon EKS Control Plane Logging documentation (https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html)"
type = list(string)
- default = ["audit"]
+ default = []
}
variable "cluster_version" {
@@ -370,6 +403,7 @@ variable "api_gw_deploy_region" {
variable "api_gateway_resources" {
description = "Nested map containing API, Stage, and VPC Link resources"
+ default = []
type = list(object({
namespace = string
api = object({