diff --git a/.shared.default.env b/.shared.default.env index 0bab6e3..d6a2b1a 100644 --- a/.shared.default.env +++ b/.shared.default.env @@ -28,6 +28,7 @@ DEV_OS_CREDITS=denbicloud/os_credits:dev # 'site' instance # Required by HAProxy at the site and by Prometheus in the portal PORTAL_AUTH_TOKEN=DontUseThisInProduction +DEVSTACK_AUTH_TOKEN=DontUseThisInProduction # Grafana GF_SECURITY_ADMIN_USER=admin GF_SECURITY_ADMIN_PASSWORD=secret diff --git a/.site.default.env b/.site.default.env index b9e1556..2760c5b 100644 --- a/.site.default.env +++ b/.site.default.env @@ -11,7 +11,7 @@ # USAGE_EXPORTER_START_DATE=2018-12-01 # Interval [seconds] used by the exporter to query OpenStack for usage values -USAGE_EXPORTER_UPDATE_INTERVAL=5 +USAGE_EXPORTER_UPDATE_INTERVAL=60 # The frequency of how often the weights should be updated, denotes a multiple of the update interval variable USAGE_EXPORTER_WEIGHT_UPDATE_FREQUENCY=10 @@ -44,5 +44,5 @@ USAGE_EXPORTER_VERBOSE_MODE=False # If your OpenStack user is not allowed to list all domains (necessary to # determine the `domain_id` of `elixir`) enter the correct id here # PROJECT_DOMAINS will be ignored if set -# USAGE_EXPORTER_PROJECT_ID=xxxx +# USAGE_EXPORTER_PROJECT_DOMAIN_ID=xxxx # diff --git a/.staging.default.env b/.staging.default.env index 8725740..4524496 100644 --- a/.staging.default.env +++ b/.staging.default.env @@ -6,11 +6,15 @@ GF_SERVER_ROOT_URL=%(protocol)s://%(domain)s/grafana # Port which must be free on the host machine # `site_prometheus_proxy` will bind to it SITE_PROXY_PUBLIC_PORT=8080 +SITE_PROXY_DEVSTACK_PORT=80 # Also needed for `portal_prometheus` as scrape target STAGING_SITE_PUBLIC_URL=portal-dev.denbi.de +STAGING_SITE_DEVSTACK_URL=192.168.1.X # Values for the portal_prometheus.yml, needed for credits service and grafana OPENSTACK_DEV_LOCATION= OPENSTACK_DEV_RESOURCE_ID= +DEVSTACK_LOCATION=bielefeld-devstack +DEVSTACK_RESOURCE_ID=0000 # Subpath used by the reverse proxy to redirect to `portal_prometheus` PORTAL_PROMETHEUS_SUBPATH=prom diff --git a/bin/project_usage-compose.py b/bin/project_usage-compose.py index c382b1b..ee7ed0b 100755 --- a/bin/project_usage-compose.py +++ b/bin/project_usage-compose.py @@ -9,7 +9,7 @@ import logging import sys from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser -from os import environ, execve, path, makedirs, chown +from os import environ, makedirs, chown, execve from pathlib import Path from shutil import which from string import Template @@ -20,7 +20,7 @@ __license__ = "MIT" -docker_compose_path = which("docker-compose") +docker_path = which("docker") project_dir = Path(__file__).parent.parent dev_dir = project_dir / "dev" prod_dir = project_dir / "prod" @@ -227,10 +227,10 @@ def main() -> int: logging.debug( "Known command line arguments: %s, Remainder: %s", args, remaining_args ) - if not docker_compose_path: - logging.critical("Could not detect `docker-compose` in $PATH. Exiting") + if not docker_path: + logging.critical("Could not detect `docker` in $PATH. Exiting") return 1 - call = [docker_compose_path] + call = [docker_path, "compose"] if "dev" == args.subcommand: needed_compose_files = development_files @@ -264,7 +264,10 @@ def main() -> int: if args.dry_run: print(" ".join(call)) return 0 - create_postgres_data_folder(env) + if "dev" == args.subcommand: + create_postgres_data_folder(env) + elif args.stack == "portal": + create_postgres_data_folder(env) if args.subcommand == "prod" and (args.staging or args.staging_dev): for file in staging_template_files[args.stack]: template_file = template_files_dir / file @@ -285,7 +288,7 @@ def main() -> int: out_file.write_text(out_file_str) logging.debug("Executing %s with environment %s", call, env) - execve(docker_compose_path, call, env) + execve(docker_path, call, env) return 0 diff --git a/dev/portal_prometheus/Dockerfile b/dev/portal_prometheus/Dockerfile index 5a0c061..c3a19f4 100644 --- a/dev/portal_prometheus/Dockerfile +++ b/dev/portal_prometheus/Dockerfile @@ -1,2 +1,2 @@ -FROM prom/prometheus:v2.39.0 +FROM prom/prometheus:v2.39.1 ADD prometheus.yml /etc/prometheus diff --git a/dev/site_prometheus/Dockerfile b/dev/site_prometheus/Dockerfile index 5a0c061..c3a19f4 100644 --- a/dev/site_prometheus/Dockerfile +++ b/dev/site_prometheus/Dockerfile @@ -1,2 +1,2 @@ -FROM prom/prometheus:v2.39.0 +FROM prom/prometheus:v2.39.1 ADD prometheus.yml /etc/prometheus diff --git a/docker-compose.portal.yml b/docker-compose.portal.yml index ec985fa..afc846f 100644 --- a/docker-compose.portal.yml +++ b/docker-compose.portal.yml @@ -11,7 +11,7 @@ x-logging: services: # global prometheus instance scraping data from the local instances portal_prometheus: - image: "prom/prometheus:v2.39.0" + image: "prom/prometheus:v2.39.1" container_name: portal_prometheus networks: portal: @@ -37,7 +37,7 @@ services: - ${POSTGRES_DATA_FOLDER}:/home/postgres/pgdata portal_promscale: container_name: portal_promscale - image: "timescale/promscale:0.15.0" + image: "timescale/promscale:0.16.0" ports: - 9201:9201/tcp restart: on-failure @@ -52,7 +52,7 @@ services: PROMSCALE_DB_URI: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}?sslmode=allow portal_grafana: - image: "grafana/grafana:9.2.0" + image: "grafana/grafana:9.2.3" container_name: portal_grafana depends_on: - portal_prometheus diff --git a/docker-compose.site.yml b/docker-compose.site.yml index ff51852..70e05ab 100644 --- a/docker-compose.site.yml +++ b/docker-compose.site.yml @@ -35,7 +35,7 @@ services: - "${EXPORTER_PORT}:8080" # prometheus instance responsible for monitoring site_prometheus: - image: "prom/prometheus:v2.39.0" + image: "prom/prometheus:v2.39.1" # use self build Dockerfile as alternative # build: prod/site_prometheus container_name: project_usage_prometheus @@ -51,7 +51,7 @@ services: # use the one below to have haproxy debug log directly to stdout including # every request with auth headers # image: mminks/haproxy-docker-logging:latest - image: "haproxy:2.6.4-alpine" + image: "haproxy:2.6.6-alpine" # use self build Dockerfile as alternative # build: prod/haproxy container_name: project_usage_proxy @@ -71,7 +71,7 @@ services: environment: - PORTAL_AUTH_TOKEN site_grafana: - image: "grafana/grafana:9.2.0" + image: "grafana/grafana:9.2.3" # use self build Dockerfile as alternative # build: prod/grafana container_name: project_usage_grafana diff --git a/prod/portal_prometheus/Dockerfile b/prod/portal_prometheus/Dockerfile index 5a0c061..c3a19f4 100644 --- a/prod/portal_prometheus/Dockerfile +++ b/prod/portal_prometheus/Dockerfile @@ -1,2 +1,2 @@ -FROM prom/prometheus:v2.39.0 +FROM prom/prometheus:v2.39.1 ADD prometheus.yml /etc/prometheus diff --git a/prod/site_prometheus/Dockerfile b/prod/site_prometheus/Dockerfile index 52b657a..3d3d88f 100644 --- a/prod/site_prometheus/Dockerfile +++ b/prod/site_prometheus/Dockerfile @@ -1,4 +1,4 @@ -FROM prom/prometheus:v2.39.0 +FROM prom/prometheus:v2.39.1 # relative to compose file ADD prometheus.yml /etc/prometheus/prometheus.yml diff --git a/shared/grafana/Dockerfile b/shared/grafana/Dockerfile index 64f3c17..1357f2e 100644 --- a/shared/grafana/Dockerfile +++ b/shared/grafana/Dockerfile @@ -1,4 +1,4 @@ -FROM grafana/grafana:9.2.0 +FROM grafana/grafana:9.2.3 COPY provisioning/* /etc/grafana/provisioning/ EXPOSE 3000 diff --git a/staging/portal_prometheus/prometheus.yml.in b/staging/portal_prometheus/prometheus.yml.in index 1b5c2c0..270b875 100644 --- a/staging/portal_prometheus/prometheus.yml.in +++ b/staging/portal_prometheus/prometheus.yml.in @@ -32,6 +32,24 @@ scrape_configs: bearer_token: "${PORTAL_AUTH_TOKEN}" + - job_name: 'denbi_cloud_project_devstack_usage' + honor_labels: true + metrics_path: '/federate' + scheme: 'http' + + params: + 'match[]': + # do not collect any metrics besides the one we are interested in + - '{job="project_usages",__name__=~"project_.*_usage"}' + + static_configs: + - targets: ['${STAGING_SITE_DEVSTACK_URL}:${SITE_PROXY_DEVSTACK_PORT}'] + labels: + location: ${DEVSTACK_LOCATION} + location_id: ${DEVSTACK_RESOURCE_ID} + + bearer_token: "${DEVSTACK_AUTH_TOKEN}" + - job_name: 'grafana' honor_labels: true static_configs: