From 8c4606331b44f81431196c188fb0d76af9d952f7 Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Thu, 5 Dec 2024 15:33:15 +0000 Subject: [PATCH] Add COS module --- .module_defaults/cos | 1 + common/generate_bundle_base | 2 +- cos/common | 1 + cos/configure | 48 ++++++++++++++ cos/cos.yaml.template | 17 +++++ cos/generate-bundle.sh | 1 + cos/module_defaults | 18 ++++++ cos/overlays | 1 + cos/pipeline/00setup | 21 ++++++ cos/pipeline/01import-config-defaults | 5 ++ cos/pipeline/02configure | 64 +++++++++++++++++++ cos/pipeline/03build | 5 ++ cos/resources/README_resources.md | 5 ++ overlays/cos/ceph-csi.yaml | 11 ++++ overlays/cos/charmed-ceph-lxd.yaml | 34 ++++++++++ overlays/cos/charmed-ceph.yaml | 34 ++++++++++ overlays/cos/cos-lite-offers.yaml | 24 +++++++ overlays/cos/cos-proxy-filebeat.yaml | 10 +++ overlays/cos/cos-proxy.yaml | 18 ++++++ overlays/cos/microceph.yaml | 20 ++++++ .../unit_placement/cos-proxy.yaml.template | 3 + .../unit_placement/microk8s.yaml.template | 3 + tox.ini | 3 + 23 files changed, 348 insertions(+), 1 deletion(-) create mode 120000 .module_defaults/cos create mode 120000 cos/common create mode 100755 cos/configure create mode 100644 cos/cos.yaml.template create mode 120000 cos/generate-bundle.sh create mode 100644 cos/module_defaults create mode 120000 cos/overlays create mode 100644 cos/pipeline/00setup create mode 100644 cos/pipeline/01import-config-defaults create mode 100644 cos/pipeline/02configure create mode 100644 cos/pipeline/03build create mode 100644 cos/resources/README_resources.md create mode 100644 overlays/cos/ceph-csi.yaml create mode 100644 overlays/cos/charmed-ceph-lxd.yaml create mode 100644 overlays/cos/charmed-ceph.yaml create mode 100644 overlays/cos/cos-lite-offers.yaml create mode 100644 overlays/cos/cos-proxy-filebeat.yaml create mode 100644 overlays/cos/cos-proxy.yaml create mode 100644 overlays/cos/microceph.yaml create mode 100644 overlays/unit_placement/cos-proxy.yaml.template create mode 100644 overlays/unit_placement/microk8s.yaml.template diff --git a/.module_defaults/cos b/.module_defaults/cos new file mode 120000 index 00000000..6377437c --- /dev/null +++ b/.module_defaults/cos @@ -0,0 +1 @@ +../cos/module_defaults \ No newline at end of file diff --git a/common/generate_bundle_base b/common/generate_bundle_base index f258361a..e2e6e633 100644 --- a/common/generate_bundle_base +++ b/common/generate_bundle_base @@ -185,7 +185,7 @@ if ${MASTER_OPTS[HYPERCONVERGED_DEPLOYMENT]}; then num_placement_machines=\ $((${MOD_PARAMS[__NUM_K8S_CONTROL_PLANE_UNITS__]}+ ${MOD_PARAMS[__NUM_K8S_WORKER_UNITS__]})) - elif [[ $MOD_NAME = ceph ]]; then + elif [[ $MOD_NAME = ceph ]] || [[ $MOD_NAME = cos ]]; then num_placement_machines=$((${MOD_PARAMS[__NUM_CEPH_OSD_UNITS__]})) else echo -n "ERROR: module '$MOD_NAME'" 1>&2 diff --git a/cos/common b/cos/common new file mode 120000 index 00000000..60d3b0a6 --- /dev/null +++ b/cos/common @@ -0,0 +1 @@ +../common \ No newline at end of file diff --git a/cos/configure b/cos/configure new file mode 100755 index 00000000..6266d2e3 --- /dev/null +++ b/cos/configure @@ -0,0 +1,48 @@ +#!/bin/bash -x +COS_MODEL=cos +SCRIPT_DIR=$(realpath $(dirname $0)) + +juju_run_cmd="juju run" +if (( $(juju --version | awk -F. {'print $1'}) > 2 )); then + juju_run_cmd="juju exec" +fi + +which kubectl || sudo snap install kubectl --classic +mkdir -p ~/.kube + +if $(juju list-models| egrep -q "^${COS_MODEL}\*"); then + echo "WARNING: currently in '$COS_MODEL' context - switch to microk8s model to re-run microk8s config" +else + mk8s_unit=$(juju status| sed -nr 's,(^microk8s/[[:digit:]]+)\*.*,\1,p') + $juju_run_cmd --unit $mk8s_unit microk8s.config > ~/.kube/config + $juju_run_cmd --unit $mk8s_unit -- 'IPADDR=$( ip r get 2.2.2.2| sed -rn "s/.+src ([0-9\.]+) .+/\1/p"); microk8s enable metallb:$IPADDR-$IPADDR' +fi + +kubectl get pods -A + +if ! $(juju list-clouds| egrep -q "^microk8s-cos"); then + KUBECONFIG=~/.kube/config juju add-k8s microk8s-cos --cluster-name=microk8s-cluster --client --controller ${OS_PROJECT_NAME/_/-} --storage=ceph-xfs +fi +if ! $(juju list-models| egrep -q "^${COS_MODEL}"); then + juju add-model $COS_MODEL microk8s-cos + juju deploy cos-lite --overlay ${SCRIPT_DIR}/overlays/cos/cos-lite-offers.yaml --trust +else + echo "INFO: model '$COS_MODEL' already exists - skipping deploy" + juju switch $COS_MODEL +fi + +juju wait-for application grafana + +set +x +echo "INFO: COS should now be reachable at the following endpoints:" +juju run traefik/0 show-proxied-endpoints --format=yaml| yq '."traefik/0".results."proxied-endpoints"' | jq + +GRAFANA_PASSWORD=$(juju run grafana/leader get-admin-password --model cos 2>/dev/null| sed -rn 's/admin-password:\s+(.+)/\1/p') +GRAFANA_USER=$(juju config grafana admin_user) +echo "Grafana login info: ${GRAFANA_USER}/$GRAFANA_PASSWORD" + +echo "INFO: run the following to consume COS from your microk8s model:" +echo "juju switch " +for offer in $(juju list-offers| tail -n+2| awk '{print $1}'); do + echo "juju consume ${COS_MODEL}.$offer" +done diff --git a/cos/cos.yaml.template b/cos/cos.yaml.template new file mode 100644 index 00000000..ca4d00ea --- /dev/null +++ b/cos/cos.yaml.template @@ -0,0 +1,17 @@ +series: __SERIES__ +machines: + '0': + constraints: __MACHINE1_CONSTRAINTS__ + series: __SERIES__ +applications: + microk8s: + charm: __CHARM_STORE____CHARM_CS_NS____CHARM_CH_PREFIX__microk8s + num_units: 1 + expose: true + options: + containerd_http_proxy: __CONTAINERD_PROXY__ + containerd_https_proxy: __CONTAINERD_PROXY__ + containerd_no_proxy: __CONTAINERD_NO_PROXY__ + to: + - 0 + diff --git a/cos/generate-bundle.sh b/cos/generate-bundle.sh new file mode 120000 index 00000000..394558ee --- /dev/null +++ b/cos/generate-bundle.sh @@ -0,0 +1 @@ +common/generate-bundle.sh \ No newline at end of file diff --git a/cos/module_defaults b/cos/module_defaults new file mode 100644 index 00000000..8a37d73f --- /dev/null +++ b/cos/module_defaults @@ -0,0 +1,18 @@ +# This file must contain defaults for all variables used in bundles/overlays. +# They are used to render to final product in the event they are not provided +# elsewhere. It is inserted into the global context at the start of the +# pipeline. +# +# You can check that none are missing by running lint/check_var_defaults.sh +# + +MOD_PARAMS[__MICROK8S_CHANNEL__]="latest/edge" +MOD_PARAMS[__CONTAINERD_PROXY__]='' +MOD_PARAMS[__CONTAINERD_NO_PROXY__]='127.0.0.1,localhost,::1,10.149.0.0/16,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16' +MOD_PARAMS[__NUM_MICROK8S_UNITS__]=1 +MOD_PARAMS[__NUM_COS_PROXY_UNITS__]=1 +MOD_PARAMS[__NUM_CEPH_MON_UNITS__]=3 +MOD_PARAMS[__NUM_CEPH_OSD_UNITS__]=3 +MOD_PARAMS[__MACHINE1_CONSTRAINTS__]='mem=8G' +MOD_PARAMS[__MACHINE2_CONSTRAINTS__]='mem=8G' +MOD_PARAMS[__MACHINE3_CONSTRAINTS__]='mem=8G' diff --git a/cos/overlays b/cos/overlays new file mode 120000 index 00000000..0d44a21c --- /dev/null +++ b/cos/overlays @@ -0,0 +1 @@ +../overlays \ No newline at end of file diff --git a/cos/pipeline/00setup b/cos/pipeline/00setup new file mode 100644 index 00000000..fecb3e76 --- /dev/null +++ b/cos/pipeline/00setup @@ -0,0 +1,21 @@ +#!/bin/bash + +# Globals +export MOD_NAME=cos +export MOD_BASE_TEMPLATE=cos.yaml.template +export MOD_SSL_STATE_DIR=${MOD_NAME} +[ -n "${MASTER_OPTS[BUNDLE_NAME]}" ] && \ + MOD_SSL_STATE_DIR="${MOD_SSL_STATE_DIR}-${MASTER_OPTS[BUNDLE_NAME]}" + +# opts that 02configure does not recognise that get passed to the generator +export -a MOD_PASSTHROUGH_OPTS=() + +# Collection of messages to display at the end +export -A MOD_MSGS=() +# Use order 0 to ensure this is first displayed +MOD_MSGS[0_common.0]="run ./configure to initialise your deployment" + +# Array list of overlays to use with this deployment. +export -a MOD_OVERLAYS=() + +export -A MOD_PARAMS=() diff --git a/cos/pipeline/01import-config-defaults b/cos/pipeline/01import-config-defaults new file mode 100644 index 00000000..3dd4b529 --- /dev/null +++ b/cos/pipeline/01import-config-defaults @@ -0,0 +1,5 @@ +# Start with dependency defaults in case we want to override any locally +. $MOD_DIR/../.module_defaults/ceph + +# Current module imports +. $MOD_DIR/module_defaults diff --git a/cos/pipeline/02configure b/cos/pipeline/02configure new file mode 100644 index 00000000..9c874349 --- /dev/null +++ b/cos/pipeline/02configure @@ -0,0 +1,64 @@ +#!/bin/bash +# Global variables are first defined in 00setup and module +# dependencies are defined in 01import-config-defaults +# +# All overlay/bundle variables (MOD_PARAMS) defaults must go into +# the /module_defaults file. + +target=$series +[ -z "$pocket" ] || target=${target}-$pocket +target=${target}:${MOD_PARAMS[__MICROK8S_CHANNEL__]} +MOD_PASSTHROUGH_OPTS+=( --release-name $target ) + +# Automatically use proxy if in prodstack only +if $(timeout 1s getent hosts squid.internal &> /dev/null) && [ -z "${MOD_PARAMS[__CONTAINERD_PROXY__]}" ]; then + MOD_MSGS[1_proxy.0]='PROXY: squid.internal exists, setting containerd proxy to http://squid.internal:3128' + MOD_PARAMS[__CONTAINERD_PROXY__]=http://squid.internal:3128 +fi + +if ! has_opt --charmed-ceph-lxd && ! has_opt --microceph; then + MOD_OVERLAYS+=( "cos/charmed-ceph.yaml" ) + MOD_OVERLAYS+=( "cos/ceph-csi.yaml" ) +fi + +# Skip processing input if it includes exclusive passthrough options +! has_excl_passthrough_opt && \ +while (($# > 0)) +do + case "$1" in + --containerd-proxy) #__OPT__type: (default="" unless the hostname squid.internal resolves, then it's http://squid.internal:3128) + MOD_PARAMS[__CONTAINERD_PROXY__]=$2 + shift + ;; + --containerd-no-proxy) #__OPT__type: (default=127.0.0.1,localhost,::1,10.149.0.0/16,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16) + MOD_PARAMS[__CONTAINERD_NO_PROXY__]=$2 + shift + ;; + --microceph) + MOD_OVERLAYS+=( "cos/microceph.yaml" ) + ;; + --charmed-ceph-lxd) + MOD_OVERLAYS+=( "cos/charmed-ceph-lxd.yaml" ) + ;; + --cos-proxy) + MOD_OVERLAYS+=( "cos/cos-proxy.yaml" ) + MOD_MSGS[2_cos-proxy.0]='Once the COS deployment is complete you need to do the following:' + MOD_MSGS[2_cos-proxy.1]='juju relate grafana-dashboards:grafana-dashboard cos-proxy:downstream-grafana-dashboard' + MOD_MSGS[2_cos-proxy.2]='juju relate loki-logging:logging cos-proxy:downstream-logging' + MOD_MSGS[2_cos-proxy.3]='juju relate prometheus-scrape:metrics-endpoint cos-proxy:downstream-prometheus-scrape' + ;; + --filebeat) + MOD_OVERLAYS+=( "cos/cos-proxy-filebeat.yaml" ) + if ! has_opt --cos-proxy; then + set -- $@ --cos-proxy && cache $@ + fi + ;; + *) + echo "ERROR: invalid input '$1'" + _usage + exit 1 + ;; + esac + shift +done || true + diff --git a/cos/pipeline/03build b/cos/pipeline/03build new file mode 100644 index 00000000..62dd78f9 --- /dev/null +++ b/cos/pipeline/03build @@ -0,0 +1,5 @@ +#!/bin/bash +. $MOD_DIR/common/generate_bundle_base + +print_msgs + diff --git a/cos/resources/README_resources.md b/cos/resources/README_resources.md new file mode 100644 index 00000000..4261e46e --- /dev/null +++ b/cos/resources/README_resources.md @@ -0,0 +1,5 @@ +If you want to configure charm resources [1] in a bundle, put a directory in +this path with the name of the bundle/overlay they correspond to. Any files +within will then be copied into the generated bundles' path. + +[1] https://docs.jujucharms.com/using-resources-developer-guide diff --git a/overlays/cos/ceph-csi.yaml b/overlays/cos/ceph-csi.yaml new file mode 100644 index 00000000..1acd94aa --- /dev/null +++ b/overlays/cos/ceph-csi.yaml @@ -0,0 +1,11 @@ +applications: + ceph-csi: + charm: __CHARM_STORE____CHARM_CS_NS____CHARM_CH_PREFIX__ceph-csi + # See https://github.com/charmed-kubernetes/ceph-csi-operator/issues/22 + channel: 1.31/stable + options: + provisioner-replicas: 1 + namespace: kube-system +relations: + - [ ceph-csi:ceph-client, ceph-mon:client ] + - [ ceph-csi:kubernetes-info, microk8s ] diff --git a/overlays/cos/charmed-ceph-lxd.yaml b/overlays/cos/charmed-ceph-lxd.yaml new file mode 100644 index 00000000..10854fbc --- /dev/null +++ b/overlays/cos/charmed-ceph-lxd.yaml @@ -0,0 +1,34 @@ +machines: + '0': + constraints: __MACHINE1_CONSTRAINTS__ + series: __SERIES__ + '1': + constraints: __MACHINE2_CONSTRAINTS__ + series: __SERIES__ + '2': + constraints: __MACHINE3_CONSTRAINTS__ + series: __SERIES__ +applications: + ceph-mon: + charm: __CHARM_STORE____CHARM_CS_NS____CHARM_CH_PREFIX__ceph-mon + channel: quincy/stable + num_units: __NUM_CEPH_MON_UNITS__ + to: + - 0 + - 1 + - 2 + ceph-osd: + charm: __CHARM_STORE____CHARM_CS_NS____CHARM_CH_PREFIX__ceph-osd + channel: quincy/stable + num_units: __NUM_CEPH_OSD_UNITS__ + to: + - 0 + - 1 + - 2 + options: + osd-devices: '' # must be empty string when using juju storage + config-flags: '{"osd": {"osd memory target": 1073741824}}' # matching 2G constraint + storage: + osd-devices: cinder,10G,1 +relations: + - [ ceph-osd:mon, ceph-mon:osd ] diff --git a/overlays/cos/charmed-ceph.yaml b/overlays/cos/charmed-ceph.yaml new file mode 100644 index 00000000..10854fbc --- /dev/null +++ b/overlays/cos/charmed-ceph.yaml @@ -0,0 +1,34 @@ +machines: + '0': + constraints: __MACHINE1_CONSTRAINTS__ + series: __SERIES__ + '1': + constraints: __MACHINE2_CONSTRAINTS__ + series: __SERIES__ + '2': + constraints: __MACHINE3_CONSTRAINTS__ + series: __SERIES__ +applications: + ceph-mon: + charm: __CHARM_STORE____CHARM_CS_NS____CHARM_CH_PREFIX__ceph-mon + channel: quincy/stable + num_units: __NUM_CEPH_MON_UNITS__ + to: + - 0 + - 1 + - 2 + ceph-osd: + charm: __CHARM_STORE____CHARM_CS_NS____CHARM_CH_PREFIX__ceph-osd + channel: quincy/stable + num_units: __NUM_CEPH_OSD_UNITS__ + to: + - 0 + - 1 + - 2 + options: + osd-devices: '' # must be empty string when using juju storage + config-flags: '{"osd": {"osd memory target": 1073741824}}' # matching 2G constraint + storage: + osd-devices: cinder,10G,1 +relations: + - [ ceph-osd:mon, ceph-mon:osd ] diff --git a/overlays/cos/cos-lite-offers.yaml b/overlays/cos/cos-lite-offers.yaml new file mode 100644 index 00000000..e1ef9852 --- /dev/null +++ b/overlays/cos/cos-lite-offers.yaml @@ -0,0 +1,24 @@ +applications: + alertmanager: + offers: + alertmanager-karma-dashboard: + endpoints: + - karma-dashboard + grafana: + offers: + grafana-dashboards: + endpoints: + - grafana-dashboard + loki: + offers: + loki-logging: + endpoints: + - logging + prometheus: + offers: + prometheus-scrape: + endpoints: + - metrics-endpoint + prometheus-receive-remote-write: + endpoints: + - receive-remote-write diff --git a/overlays/cos/cos-proxy-filebeat.yaml b/overlays/cos/cos-proxy-filebeat.yaml new file mode 100644 index 00000000..48d0f7fc --- /dev/null +++ b/overlays/cos/cos-proxy-filebeat.yaml @@ -0,0 +1,10 @@ +applications: + filebeat: + charm: __CHARM_STORE____CHARM_CS_NS____CHARM_CH_PREFIX__filebeat + channel: stable + options: + logpath: /var/log/*.log /var/log/*/*.log /var/log/syslog +relations: + - [ceph-mon:juju-info, filebeat:beats-host] + - [cos-proxy:filebeat, filebeat:logstash] + - [cos-proxy:juju-info, filebeat:beats-host] diff --git a/overlays/cos/cos-proxy.yaml b/overlays/cos/cos-proxy.yaml new file mode 100644 index 00000000..ac30cb59 --- /dev/null +++ b/overlays/cos/cos-proxy.yaml @@ -0,0 +1,18 @@ +machines: + '0': + constraints: __MACHINE1_CONSTRAINTS__ + series: __SERIES__ + '1': + constraints: __MACHINE2_CONSTRAINTS__ + series: __SERIES__ + '2': + constraints: __MACHINE3_CONSTRAINTS__ + series: __SERIES__ +applications: + cos-proxy: + charm: __CHARM_STORE____CHARM_CS_NS____CHARM_CH_PREFIX__cos-proxy + channel: stable + num_units: 1 + to: + - 0 + diff --git a/overlays/cos/microceph.yaml b/overlays/cos/microceph.yaml new file mode 100644 index 00000000..1a2c6f5f --- /dev/null +++ b/overlays/cos/microceph.yaml @@ -0,0 +1,20 @@ +machines: + '0': + constraints: __MACHINE1_CONSTRAINTS__ + series: __SERIES__ + '1': + constraints: __MACHINE2_CONSTRAINTS__ + series: __SERIES__ + '2': + constraints: __MACHINE3_CONSTRAINTS__ + series: __SERIES__ +applications: + microceph: + charm: microceph + channel: quincy/stable + num_units: __NUM_CEPH_OSD_UNITS__ + to: + - 0 + - 1 + - 2 + diff --git a/overlays/unit_placement/cos-proxy.yaml.template b/overlays/unit_placement/cos-proxy.yaml.template new file mode 100644 index 00000000..56e192e1 --- /dev/null +++ b/overlays/unit_placement/cos-proxy.yaml.template @@ -0,0 +1,3 @@ + cos-proxy: + to: +__UNIT_PLACEMENT_METAL__.__UNITS__.__NUM_COS_PROXY_UNITS__ diff --git a/overlays/unit_placement/microk8s.yaml.template b/overlays/unit_placement/microk8s.yaml.template new file mode 100644 index 00000000..b688e351 --- /dev/null +++ b/overlays/unit_placement/microk8s.yaml.template @@ -0,0 +1,3 @@ + microk8s: + to: +__UNIT_PLACEMENT_METAL__.__UNITS__.__NUM_MICROK8S_UNITS__ diff --git a/tox.ini b/tox.ini index 38927b61..71bdda51 100644 --- a/tox.ini +++ b/tox.ini @@ -24,6 +24,9 @@ pyfiles = {toxinidir}/tools/juju-bundle-applications.py bashfiles = + # find -name configure| grep -v .git| sed -rn 's,^.,{toxinidir},p' + {toxinidir}/cos/configure + {toxinidir}/tools/juju-lnav # find tools -name \*.sh {toxinidir}/tools/vault-unseal-and-authorise.sh