diff --git a/dev/local/ch-full/docker-compose.yml b/dev/local/ch-full/docker-compose.yml index a188342d..8bb546ff 100644 --- a/dev/local/ch-full/docker-compose.yml +++ b/dev/local/ch-full/docker-compose.yml @@ -106,7 +106,7 @@ services: - "9091:9090" restart: unless-stopped volumes: - - ../prometheus:/etc/prometheus + - ../prometheus/prometheus.yml:/etc/prometheus/prometheus.yml - prom:/prometheus # For compliance testing. diff --git a/dev/local/ch-remote-write/docker-compose.yml b/dev/local/ch-remote-write/docker-compose.yml new file mode 100644 index 00000000..13e0598f --- /dev/null +++ b/dev/local/ch-remote-write/docker-compose.yml @@ -0,0 +1,146 @@ +version: "3" + +volumes: + prom: + +services: + clickhouse: + image: clickhouse/clickhouse-server:23.10 + ports: + - "9000:9000" + - "8123:8123" + volumes: + - ../clickhouse.xml:/etc/clickhouse-server/config.d/monitoring.xml + healthcheck: + test: ['CMD', 'wget', '--spider', '-q', '127.0.0.1:8123/ping'] + interval: 1s + timeout: 1s + retries: 30 + + # Exports traces from clickhouse internal table to otel. + chotel: + restart: always + build: + context: ../../../ + dockerfile: chotel.Dockerfile + environment: + - OTEL_EXPORTER_OTLP_ENDPOINT=http://otelcol:4317 + - OTEL_EXPORTER_OTLP_PROTOCOL=grpc + - OTEL_EXPORTER_OTLP_INSECURE=true + - OTEL_RESOURCE_ATTRIBUTES=service.name=go-faster.oteldb.chotel + depends_on: + - clickhouse + + faker-api: + restart: always + build: + context: ../../../ + dockerfile: otelfaker.Dockerfile + environment: + - OTEL_METRICS_EXPORTER=prometheus + - OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc + - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://otelcol:4317 + - OTEL_EXPORTER_PROMETHEUS_HOST=0.0.0.0 + - OTEL_RESOURCE_ATTRIBUTES=service.name=fuck,service.namespace=demo,oteldb.faker=true + + oteldb: + build: + context: ../../../ + dockerfile: Dockerfile + environment: + - OTELDB_STORAGE=ch + - CH_DSN=clickhouse://clickhouse:9000 + - OTEL_LOG_LEVEL=debug + - METRICS_ADDR=:3201 + - OTEL_METRICS_EXPORTER=otlp + - OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc + - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://otelcol:4317 + - OTEL_LOGS_EXPORTER=otlp + - OTEL_EXPORTER_OTLP_LOGS_PROTOCOL=grpc + - OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=http://otelcol:4317 + - OTEL_RESOURCE_ATTRIBUTES=service.name=go-faster.oteldb + ports: + - "9090:9090" # prometheus API + depends_on: + - clickhouse + + # to scrape node-exporter + # also can remote-write to otelcol + prometheus: + image: prom/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--web.enable-remote-write-receiver' + ports: + - "9091:9090" + restart: unless-stopped + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + - prom:/prometheus + + # For compliance testing. + demo-1: + image: julius/prometheus-demo-service:latest + demo-2: + image: julius/prometheus-demo-service:latest + demo-3: + image: julius/prometheus-demo-service:latest + + # https://opentelemetry.io/docs/collector/installation/#docker-compose + otelcol: + image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.89.0 + volumes: + - ./otelcol.yml:/etc/otelcol-contrib/config.yaml + depends_on: + - prometheus + - oteldb + - node-exporter + environment: + # probably excessive + - OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc + - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://otelcol:4317 + command: + - '--config' + - '/etc/otelcol-contrib/config.yaml' + # https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/observability.md#how-we-expose-telemetry + - '--feature-gates=telemetry.useOtelForInternalMetrics' + - '--feature-gates=telemetry.useOtelWithSDKConfigurationForInternalTelemetry' + + # to produce some metrics + node-exporter: + image: prom/node-exporter + restart: unless-stopped + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - '--path.procfs=/host/proc' + - '--path.rootfs=/rootfs' + - '--path.sysfs=/host/sys' + - '--collector.processes' + - '--web.max-requests=40' + - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' + + grafana: + image: "grafana/grafana:10.0.0" + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_AUTH_DISABLE_LOGIN_FORM=true + - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor,traceToMetrics,publicDashboards,tempoApmTable + - GF_ALERTING_ENABLED=false + - GF_USERS_ALLOW_SIGN_UP=false + - GF_LOG_LEVEL=debug + - GF_INSTALL_PLUGINS=grafana-clickhouse-datasource + ports: + - "3000:3000" + volumes: + - ../grafana/grafana.ini:/etc/grafana/grafana.ini + - ../grafana/datasources.yml:/etc/grafana/provisioning/datasources/datasources.yml + - ../grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/default.yml + - ../grafana/dashboards:/etc/grafana/dashboards:ro + depends_on: + - oteldb + - prometheus + - otelcol diff --git a/dev/local/ch-remote-write/otelcol.yml b/dev/local/ch-remote-write/otelcol.yml new file mode 100644 index 00000000..c8cb0553 --- /dev/null +++ b/dev/local/ch-remote-write/otelcol.yml @@ -0,0 +1,61 @@ +receivers: + otlp: + protocols: + grpc: + http: + +processors: + batch: + timeout: 1s + send_batch_max_size: 1024 + send_batch_size: 500 + batch/logs: + timeout: 10s + send_batch_max_size: 10000 + send_batch_size: 5000 + memory_limiter: + check_interval: 1s + limit_mib: 256 + spike_limit_mib: 500 +exporters: + otlp: + endpoint: oteldb:4317 + tls: + insecure: true +extensions: + health_check: + pprof: + zpages: + +service: + extensions: [health_check, pprof, zpages] + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [otlp] + metrics: + receivers: [otlp] + processors: [batch] + exporters: [otlp] + logs: + receivers: [otlp] + processors: [batch/logs] + exporters: [otlp] + # https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/observability.md#how-we-expose-telemetry + telemetry: + metrics: + readers: + - periodic: + interval: 5000 + exporter: + otlp: + protocol: grpc/protobuf + endpoint: http://localhost:4317 + traces: + processors: + - batch: + exporter: + otlp: + protocol: grpc/protobuf + endpoint: http://localhost:4317 \ No newline at end of file diff --git a/dev/local/ch-remote-write/prometheus.yml b/dev/local/ch-remote-write/prometheus.yml new file mode 100644 index 00000000..48ea1828 --- /dev/null +++ b/dev/local/ch-remote-write/prometheus.yml @@ -0,0 +1,36 @@ +global: + scrape_interval: 1s + +scrape_configs: + # For PromQL compliance tester + # https://github.com/prometheus/compliance/tree/main/promql#promql-compliance-tester + - job_name: 'demo' + scrape_interval: 1s + static_configs: + - targets: + - 'demo-1:8080' + - 'demo-2:8080' + - 'demo-3:8080' + - job_name: otel-collector + scrape_interval: 5s + static_configs: + - targets: [localhost:8888] + - job_name: node-exporter + scrape_interval: 5s + static_configs: + - targets: [node-exporter:9100] + - job_name: prometheus + scrape_interval: 5s + static_configs: + - targets: [prometheus:9090] + - job_name: clickhouse + scrape_interval: 5s + static_configs: + - targets: [clickhouse:8080] + - job_name: faker + scrape_interval: 5s + static_configs: + - targets: + - 'faker-api:9464' +remote_write: + - url: http://oteldb:19291 diff --git a/dev/local/ch/docker-compose.yml b/dev/local/ch/docker-compose.yml index 13e0598f..9baa8a3c 100644 --- a/dev/local/ch/docker-compose.yml +++ b/dev/local/ch/docker-compose.yml @@ -31,18 +31,6 @@ services: depends_on: - clickhouse - faker-api: - restart: always - build: - context: ../../../ - dockerfile: otelfaker.Dockerfile - environment: - - OTEL_METRICS_EXPORTER=prometheus - - OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=grpc - - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://otelcol:4317 - - OTEL_EXPORTER_PROMETHEUS_HOST=0.0.0.0 - - OTEL_RESOURCE_ATTRIBUTES=service.name=fuck,service.namespace=demo,oteldb.faker=true - oteldb: build: context: ../../../ @@ -75,7 +63,7 @@ services: - "9091:9090" restart: unless-stopped volumes: - - ./prometheus.yml:/etc/prometheus/prometheus.yml + - prometheus.yml:/etc/prometheus/prometheus.yml - prom:/prometheus # For compliance testing. @@ -143,4 +131,4 @@ services: depends_on: - oteldb - prometheus - - otelcol + - otelcol \ No newline at end of file diff --git a/dev/local/ch/otelcol.yml b/dev/local/ch/otelcol.yml index c8cb0553..007d5605 100644 --- a/dev/local/ch/otelcol.yml +++ b/dev/local/ch/otelcol.yml @@ -3,25 +3,67 @@ receivers: protocols: grpc: http: - + prometheus: + config: + scrape_configs: + # For PromQL compliance tester + # https://github.com/prometheus/compliance/tree/main/promql#promql-compliance-tester + - job_name: 'demo' + scrape_interval: 5s + static_configs: + - targets: + - 'demo-1:8080' + - 'demo-2:8080' + - 'demo-3:8080' + - job_name: otel-collector + scrape_interval: 5s + static_configs: + - targets: [localhost:8888] + - job_name: node-exporter + scrape_interval: 5s + static_configs: + - targets: [node-exporter:9100] + - job_name: prometheus + scrape_interval: 5s + static_configs: + - targets: [prometheus:9090] + - job_name: clickhouse + scrape_interval: 5s + static_configs: + - targets: [clickhouse:8080] processors: batch: timeout: 1s send_batch_max_size: 1024 send_batch_size: 500 - batch/logs: - timeout: 10s - send_batch_max_size: 10000 - send_batch_size: 5000 memory_limiter: check_interval: 1s limit_mib: 256 spike_limit_mib: 500 + # normalize between prometheus and otel + resource/normalize: + attributes: + - key: instance + from_attribute: service.instance.id + action: insert + - key: job + from_attribute: service.name + action: insert exporters: otlp: endpoint: oteldb:4317 tls: insecure: true + prometheusremotewrite: + endpoint: "http://prometheus:9090/api/v1/write" + resource_to_telemetry_conversion: + enabled: true # Convert resource attributes to metric labels + tls: + insecure: true + debug: + sampling_initial: 5 + sampling_thereafter: 200 + extensions: health_check: pprof: @@ -37,10 +79,14 @@ service: metrics: receivers: [otlp] processors: [batch] - exporters: [otlp] + exporters: [otlp, prometheusremotewrite] + metrics/scrape: + receivers: [prometheus] + processors: [resource/normalize, batch] + exporters: [otlp, prometheusremotewrite] logs: receivers: [otlp] - processors: [batch/logs] + processors: [batch] exporters: [otlp] # https://github.com/open-telemetry/opentelemetry-collector/blob/main/docs/observability.md#how-we-expose-telemetry telemetry: diff --git a/dev/local/ch/prometheus.yml b/dev/local/ch/prometheus.yml index 48ea1828..99096624 100644 --- a/dev/local/ch/prometheus.yml +++ b/dev/local/ch/prometheus.yml @@ -1,36 +1,10 @@ global: - scrape_interval: 1s + scrape_interval: 5s + scrape_timeout: 2s + evaluation_interval: 5s -scrape_configs: - # For PromQL compliance tester - # https://github.com/prometheus/compliance/tree/main/promql#promql-compliance-tester - - job_name: 'demo' - scrape_interval: 1s - static_configs: - - targets: - - 'demo-1:8080' - - 'demo-2:8080' - - 'demo-3:8080' - - job_name: otel-collector - scrape_interval: 5s - static_configs: - - targets: [localhost:8888] - - job_name: node-exporter - scrape_interval: 5s - static_configs: - - targets: [node-exporter:9100] - - job_name: prometheus - scrape_interval: 5s - static_configs: - - targets: [prometheus:9090] - - job_name: clickhouse - scrape_interval: 5s - static_configs: - - targets: [clickhouse:8080] - - job_name: faker - scrape_interval: 5s - static_configs: - - targets: - - 'faker-api:9464' -remote_write: - - url: http://oteldb:19291 +alerting: + +# scraping is done in otelcol +# metrics are ingested by prometheus remote write +scrape_configs: []