Kerem/metrics discovery dockerfile pt2 (#406)

packages/metrics-discovery/Dockerfile

@@ -28,11 +28,13 @@ COPY ./packages/proto /river/packages/proto
 COPY ./packages/metrics-discovery /river/packages/metrics-discovery
 
 # install dependencies and build
-RUN apk add --no-cache git
+RUN apk add --no-cache git curl
 RUN corepack enable
 RUN yarn install
 RUN yarn build:metrics-discovery
 
 WORKDIR /river/packages/metrics-discovery
 
+HEALTHCHECK --interval=30s --timeout=10s --retries=3 CMD curl -f http://localhost:8080 || exit 1
+
 CMD [ "yarn", "start"]

packages/metrics-discovery/docker-compose.yml

@@ -1,6 +1,3 @@
-volumes:
-    prometheus_data: {}
-
 services:
     river-metrics-discovery:
         container_name: river-metrics-discovery
@@ -11,12 +8,14 @@ services:
             - ./prometheus/etc/:/river/packages/metrics-discovery/prometheus/etc:rw
         env_file:
             - ./.env
+        ports:
+            - '8080:8080'
 
     prometheus:
         depends_on:
-            - river-metrics-discovery
+            river-metrics-discovery:
+                condition: service_healthy
         container_name: prometheus
-        restart: always
         image: prom/prometheus:latest
         user: root
         labels:
@@ -26,7 +25,6 @@ services:
             # collect_counters_with_distributions
         volumes:
             - ./prometheus/etc:/prometheus/etc
-            - prometheus_data:/prometheus
         command:
             - --config.file=/prometheus/etc/prometheus.yml
             - --log.level=debug
@@ -35,8 +33,8 @@ services:
 
     datadog:
         depends_on:
-            - prometheus
-            - river-metrics-discovery
+            river-metrics-discovery:
+                condition: service_healthy
         container_name: datadog
         image: datadog/agent:7
         ports:

packages/metrics-discovery/src/index.ts

@@ -3,9 +3,13 @@ import { MetricsDiscovery } from './metrics-discovery'
 import { envVarsSchema } from './env-vars'
 import { sleep } from './utils'
 import { createPrometheusConfig } from './create-prometheus-config'
+import http from 'node:http'
 
 const PROMETHEUS_TARGETS_FILE = './prometheus/etc/targets.json'
 const SLEEP_DURATION_MS = 1000 * 60 * 5 // 5 minutes
+const PORT = 8080
+
+let numWrites = 0
 
 const run = async () => {
     console.info('Creating prometheus config...')
@@ -18,13 +22,29 @@ const run = async () => {
         env: envVars.ENV,
     })
 
+    const server = http.createServer((req, res) => {
+        if (numWrites === 0) {
+            res.writeHead(500, { 'Content-Type': 'text/plain' })
+            res.end('No prometheus targets written yet\n')
+            return
+        } else {
+            res.writeHead(200, { 'Content-Type': 'text/plain' })
+            res.end('Healthy\n')
+        }
+    })
+
+    server.listen(PORT, () => {
+        console.log(`Server running at http://localhost:${PORT}/`)
+    })
+
     for (;;) {
         console.info('Getting prometheus targets...')
         const targets = await metricsDiscovery.getPrometheusTargets()
         console.info('Writing prometheus targets...', targets)
         await fs.promises.writeFile(PROMETHEUS_TARGETS_FILE, targets, {
             encoding: 'utf8',
         })
+        numWrites++
         console.info(`Prometheus targets written to: ${PROMETHEUS_TARGETS_FILE}`)
         console.info(`Sleeping for ${SLEEP_DURATION_MS} ms...`)
         await sleep(SLEEP_DURATION_MS)