Kerem/metrics discovery without volumes (#382)

- This PR removes persistent volume requirements in order to simplify
the metrics aggregator deployment. This way we won't need to maintain
additional cloud components like EFS or EBS.
- We're also setting up docker builds in this same PR.

.github/workflows/Metrics_discovery_docker.yml

@@ -0,0 +1,64 @@
+name: 'Build River Metrics Discovery Docker Image'
+
+on:
+    push:
+        branches:
+            - main
+        paths:
+            - 'packages/**'
+    workflow_dispatch:
+
+env:
+    SLACK_WEBHOOK_URL: ${{ secrets.SLACK_CD_WORKFLOW_WEBHOOK_URL }}
+
+jobs:
+    build:
+        name: Build docker image
+
+        runs-on: ubuntu-latest
+
+        permissions:
+            contents: write
+            packages: write
+
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+
+            - name: Setup AWS Credentials
+              uses: aws-actions/configure-aws-credentials@v4
+              with:
+                  aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+                  aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+                  aws-region: us-east-1
+
+            - name: Login to Amazon ECR
+              id: login-aws-ecr
+              uses: aws-actions/amazon-ecr-login@v2
+              with:
+                  registry-type: 'public'
+
+            - name: Build and push docker image to Amazon ECR
+              env:
+                  ECR_REGISTRY: ${{ steps.login-aws-ecr.outputs.registry }}
+                  #This can be custom alias once requested to aws and approved for public repo
+                  REGISTRY_ALIAS: h5v6m2x1
+                  ECR_REPOSITORY: river-metrics-discovery
+              run: |
+                  docker build -t $ECR_REGISTRY/$REGISTRY_ALIAS/$ECR_REPOSITORY:latest . -f ./packages/metrics-discovery/Dockerfile
+                  docker push $ECR_REGISTRY/$REGISTRY_ALIAS/$ECR_REPOSITORY:latest
+
+            # If action failed, we send a slack notification
+            - name: Slack notification
+              if: failure()
+              uses: slackapi/[email protected]
+              with:
+                  payload: |
+                      {
+                          "step": "Build Metrics Discovery Docker Image",
+                          "environment": "N/",
+                          "branch": "${{ github.ref }}",
+                          "url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}",
+                          "commit": "${{ github.sha }}",
+                          "actor": "${{ github.actor }}"
+                      }

packages/metrics-discovery/.gitignore

@@ -1,2 +1,2 @@
-prometheus/etc/config/targets.json
+prometheus/etc/config
 .env

packages/metrics-discovery/docker-compose.yml

@@ -15,19 +15,30 @@ services:
             - ./.env
 
     prometheus:
+        depends_on:
+            - river-metrics-discovery
         container_name: prometheus
+        restart: always
         image: prom/prometheus:latest
         user: root
+        labels:
+            com.datadoghq.ad.check_names: '["prometheus"]'
+            com.datadoghq.ad.init_configs: '[{}]'
+            com.datadoghq.ad.instances: '[{"prometheus_url": "http://prometheus:9090/federate?match%5B%5D=%7Bjob%3D%22river-node%22%7D", "namespace": "river-node", "metrics": ["river*"], "type_overrides": {"*": "gauge"}, "max_returned_metrics": 999999999999}]'
+            # collect_counters_with_distributions
         volumes:
             - ./prometheus/etc:/prometheus/etc
             - prometheus_data:/prometheus
         command:
-            - --config.file=/prometheus/etc/config/prometheus-config.yaml
+            - --config.file=/prometheus/etc/config/prometheus-config.yml
             - --log.level=debug
         ports:
             - '9090:9090'
 
     datadog:
+        depends_on:
+            - prometheus
+            - river-metrics-discovery
         container_name: datadog
         image: datadog/agent:7
         ports:
@@ -37,6 +48,5 @@ services:
             - /var/run/docker.sock:/var/run/docker.sock
             - /proc/:/host/proc/:ro
             - /sys/fs/cgroup:/host/sys/fs/cgroup:ro
-            - ./datadog/prometheus.d:/conf.d/prometheus.d:ro
         env_file:
             - ./.env

packages/metrics-discovery/src/create-prometheus-config.ts

@@ -0,0 +1,15 @@
+import fs from 'fs'
+
+const PROMETHEUS_CONFIG_FILE_SOURCE = './prometheus-config.yml'
+const PROMETHEUS_CONFIG_FILE_DESTINATION = './prometheus/etc/config/prometheus-config.yml'
+
+export const createPrometheusConfig = async () => {
+    console.info('Creating prometheus config...')
+    const prometheusConfig = await fs.promises.readFile(PROMETHEUS_CONFIG_FILE_SOURCE, {
+        encoding: 'utf8',
+    })
+    await fs.promises.writeFile(PROMETHEUS_CONFIG_FILE_DESTINATION, prometheusConfig, {
+        encoding: 'utf8',
+    })
+    console.info(`Prometheus config written to: ${PROMETHEUS_CONFIG_FILE_DESTINATION}`)
+}

packages/metrics-discovery/src/index.ts

@@ -1,11 +1,17 @@
+import fs from 'fs'
 import { MetricsDiscovery } from './metrics-discovery'
 import { envVarsSchema } from './env-vars'
-import { sleep, writeFile } from './utils'
+import { sleep } from './utils'
+import { createPrometheusConfig } from './create-prometheus-config'
 
 const PROMETHEUS_TARGETS_FILE = './prometheus/etc/config/targets.json'
 const SLEEP_DURATION_MS = 1000 * 60 * 5 // 5 minutes
 
 const run = async () => {
+    console.info('Creating prometheus config...')
+    await createPrometheusConfig()
+    console.info('Prometheus config created')
+
     const envVars = envVarsSchema.parse(process.env)
     const metricsDiscovery = MetricsDiscovery.init({
         riverRpcURL: envVars.RIVER_RPC_URL,
@@ -16,7 +22,7 @@ const run = async () => {
         console.info('Getting prometheus targets...')
         const targets = await metricsDiscovery.getPrometheusTargets()
         console.info('Writing prometheus targets...', targets)
-        await writeFile(PROMETHEUS_TARGETS_FILE, targets, {
+        await fs.promises.writeFile(PROMETHEUS_TARGETS_FILE, targets, {
             encoding: 'utf8',
         })
         console.info(`Prometheus targets written to: ${PROMETHEUS_TARGETS_FILE}`)

packages/metrics-discovery/src/utils.ts

@@ -1,21 +1,3 @@
-import fs, { WriteFileOptions } from 'fs'
-
 export function sleep(ms: number) {
     return new Promise((resolve) => setTimeout(resolve, ms))
 }
-
-export function writeFile(
-    file: string,
-    data: string | NodeJS.ArrayBufferView,
-    options: WriteFileOptions,
-) {
-    return new Promise<void>((resolve, reject) => {
-        fs.writeFile(file, data, options, (err) => {
-            if (err) {
-                reject(err)
-            } else {
-                resolve()
-            }
-        })
-    })
-}