Merge pull request #234 from deNBI/staging

Staging

.shared.default.env

@@ -28,6 +28,7 @@ DEV_OS_CREDITS=denbicloud/os_credits:dev
 # 'site' instance
 # Required by HAProxy at the site and by Prometheus in the portal
 PORTAL_AUTH_TOKEN=DontUseThisInProduction
+DEVSTACK_AUTH_TOKEN=DontUseThisInProduction
 # Grafana
 GF_SECURITY_ADMIN_USER=admin
 GF_SECURITY_ADMIN_PASSWORD=secret

.site.default.env

@@ -11,7 +11,7 @@
 # USAGE_EXPORTER_START_DATE=2018-12-01
 
 # Interval [seconds] used by the exporter to query OpenStack for usage values
-USAGE_EXPORTER_UPDATE_INTERVAL=5
+USAGE_EXPORTER_UPDATE_INTERVAL=60
 
 # The frequency of how often the weights should be updated, denotes a multiple of the update interval variable
 USAGE_EXPORTER_WEIGHT_UPDATE_FREQUENCY=10
@@ -44,5 +44,5 @@ USAGE_EXPORTER_VERBOSE_MODE=False
 # If your OpenStack user is not allowed to list all domains (necessary to
 # determine the `domain_id` of `elixir`) enter the correct id here
 # PROJECT_DOMAINS will be ignored if set
-# USAGE_EXPORTER_PROJECT_ID=xxxx
+# USAGE_EXPORTER_PROJECT_DOMAIN_ID=xxxx
 #

.staging.default.env

@@ -6,11 +6,15 @@ GF_SERVER_ROOT_URL=%(protocol)s://%(domain)s/grafana
 # Port which must be free on the host machine
 # `site_prometheus_proxy` will bind to it
 SITE_PROXY_PUBLIC_PORT=8080
+SITE_PROXY_DEVSTACK_PORT=80
 # Also needed for `portal_prometheus` as scrape target
 STAGING_SITE_PUBLIC_URL=portal-dev.denbi.de
+STAGING_SITE_DEVSTACK_URL=192.168.1.X
 # Values for the portal_prometheus.yml, needed for credits service and grafana
 OPENSTACK_DEV_LOCATION=
 OPENSTACK_DEV_RESOURCE_ID=
+DEVSTACK_LOCATION=bielefeld-devstack
+DEVSTACK_RESOURCE_ID=0000
 # Subpath used by the reverse proxy to redirect to `portal_prometheus`
 PORTAL_PROMETHEUS_SUBPATH=prom
 

bin/project_usage-compose.py

@@ -9,7 +9,7 @@
 import logging
 import sys
 from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
-from os import environ, execve, path, makedirs, chown
+from os import environ, makedirs, chown, execve
 from pathlib import Path
 from shutil import which
 from string import Template
@@ -20,7 +20,7 @@
 __license__ = "MIT"
 
 
-docker_compose_path = which("docker-compose")
+docker_path = which("docker")
 project_dir = Path(__file__).parent.parent
 dev_dir = project_dir / "dev"
 prod_dir = project_dir / "prod"
@@ -227,10 +227,10 @@ def main() -> int:
     logging.debug(
         "Known command line arguments: %s, Remainder: %s", args, remaining_args
     )
-    if not docker_compose_path:
-        logging.critical("Could not detect `docker-compose` in $PATH. Exiting")
+    if not docker_path:
+        logging.critical("Could not detect `docker` in $PATH. Exiting")
         return 1
-    call = [docker_compose_path]
+    call = [docker_path, "compose"]
 
     if "dev" == args.subcommand:
         needed_compose_files = development_files
@@ -264,7 +264,10 @@ def main() -> int:
     if args.dry_run:
         print(" ".join(call))
         return 0
-    create_postgres_data_folder(env)
+    if "dev" == args.subcommand:
+        create_postgres_data_folder(env)
+    elif args.stack == "portal":
+        create_postgres_data_folder(env)
     if args.subcommand == "prod" and (args.staging or args.staging_dev):
         for file in staging_template_files[args.stack]:
             template_file = template_files_dir / file
@@ -285,7 +288,7 @@ def main() -> int:
             out_file.write_text(out_file_str)
 
     logging.debug("Executing %s with environment %s", call, env)
-    execve(docker_compose_path, call, env)
+    execve(docker_path, call, env)
 
     return 0
 

dev/portal_prometheus/Dockerfile

@@ -1,2 +1,2 @@
-FROM prom/prometheus:v2.39.0
+FROM prom/prometheus:v2.39.1
 ADD prometheus.yml /etc/prometheus

dev/site_prometheus/Dockerfile

@@ -1,2 +1,2 @@
-FROM prom/prometheus:v2.39.0
+FROM prom/prometheus:v2.39.1
 ADD prometheus.yml /etc/prometheus

docker-compose.portal.yml

@@ -11,7 +11,7 @@ x-logging:
 services:
   # global prometheus instance scraping data from the local instances
   portal_prometheus:
-    image: "prom/prometheus:v2.39.0"
+    image: "prom/prometheus:v2.39.1"
     container_name: portal_prometheus
     networks:
       portal:
@@ -37,7 +37,7 @@ services:
       - ${POSTGRES_DATA_FOLDER}:/home/postgres/pgdata
   portal_promscale:
     container_name: portal_promscale
-    image: "timescale/promscale:0.15.0"
+    image: "timescale/promscale:0.16.0"
     ports:
       - 9201:9201/tcp
     restart: on-failure
@@ -52,7 +52,7 @@ services:
       PROMSCALE_DB_URI: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}?sslmode=allow
 
   portal_grafana:
-    image: "grafana/grafana:9.2.0"
+    image: "grafana/grafana:9.2.3"
     container_name: portal_grafana
     depends_on:
       - portal_prometheus

docker-compose.site.yml

@@ -35,7 +35,7 @@ services:
       - "${EXPORTER_PORT}:8080"
   # prometheus instance responsible for monitoring
   site_prometheus:
-    image: "prom/prometheus:v2.39.0"
+    image: "prom/prometheus:v2.39.1"
     # use self build Dockerfile as alternative
     # build: prod/site_prometheus
     container_name: project_usage_prometheus
@@ -51,7 +51,7 @@ services:
     # use the one below to have haproxy debug log directly to stdout including
     # every request with auth headers
     # image: mminks/haproxy-docker-logging:latest
-    image: "haproxy:2.6.4-alpine"
+    image: "haproxy:2.6.6-alpine"
     # use self build Dockerfile as alternative
     # build: prod/haproxy
     container_name: project_usage_proxy
@@ -71,7 +71,7 @@ services:
     environment:
       - PORTAL_AUTH_TOKEN
   site_grafana:
-    image: "grafana/grafana:9.2.0"
+    image: "grafana/grafana:9.2.3"
     # use self build Dockerfile as alternative
     # build: prod/grafana
     container_name: project_usage_grafana

prod/portal_prometheus/Dockerfile

@@ -1,2 +1,2 @@
-FROM prom/prometheus:v2.39.0
+FROM prom/prometheus:v2.39.1
 ADD prometheus.yml /etc/prometheus

prod/site_prometheus/Dockerfile

@@ -1,4 +1,4 @@
-FROM prom/prometheus:v2.39.0
+FROM prom/prometheus:v2.39.1
 # relative to compose file
 ADD prometheus.yml /etc/prometheus/prometheus.yml
 

shared/grafana/Dockerfile

@@ -1,4 +1,4 @@
-FROM grafana/grafana:9.2.0
+FROM grafana/grafana:9.2.3
 COPY provisioning/* /etc/grafana/provisioning/
 
 EXPOSE 3000

staging/portal_prometheus/prometheus.yml.in

@@ -32,6 +32,24 @@ scrape_configs:
 
     bearer_token: "${PORTAL_AUTH_TOKEN}"
 
+  - job_name: 'denbi_cloud_project_devstack_usage'
+    honor_labels: true
+    metrics_path: '/federate'
+    scheme: 'http'
+
+    params:
+      'match[]':
+        # do not collect any metrics besides the one we are interested in
+        - '{job="project_usages",__name__=~"project_.*_usage"}'
+
+    static_configs:
+      - targets: ['${STAGING_SITE_DEVSTACK_URL}:${SITE_PROXY_DEVSTACK_PORT}']
+        labels:
+          location: ${DEVSTACK_LOCATION}
+          location_id: ${DEVSTACK_RESOURCE_ID}
+
+    bearer_token: "${DEVSTACK_AUTH_TOKEN}"
+
   - job_name: 'grafana'
     honor_labels: true
     static_configs: