Merge pull request #282 from dosaboy/cos

Add COS module

.module_defaults/cos

@@ -0,0 +1 @@
+../cos/module_defaults

common/generate_bundle_base

@@ -185,7 +185,7 @@ if ${MASTER_OPTS[HYPERCONVERGED_DEPLOYMENT]}; then
         num_placement_machines=\
             $((${MOD_PARAMS[__NUM_K8S_CONTROL_PLANE_UNITS__]}+
                 ${MOD_PARAMS[__NUM_K8S_WORKER_UNITS__]}))
-    elif [[ $MOD_NAME = ceph ]]; then
+    elif [[ $MOD_NAME = ceph ]] || [[ $MOD_NAME = cos ]]; then
         num_placement_machines=$((${MOD_PARAMS[__NUM_CEPH_OSD_UNITS__]}))
     else
         echo -n "ERROR: module '$MOD_NAME'" 1>&2

cos/common

@@ -0,0 +1 @@
+../common

cos/configure

@@ -0,0 +1,48 @@
+#!/bin/bash -x
+COS_MODEL=cos
+SCRIPT_DIR=$(realpath $(dirname $0))
+
+juju_run_cmd="juju run"
+if (( $(juju --version | awk -F. {'print $1'}) > 2 )); then
+    juju_run_cmd="juju exec"
+fi
+
+which kubectl || sudo snap install kubectl --classic
+mkdir -p ~/.kube
+
+if $(juju list-models| egrep -q "^${COS_MODEL}\*"); then
+    echo "WARNING: currently in '$COS_MODEL' context - switch to microk8s model to re-run microk8s config"
+else
+    mk8s_unit=$(juju status| sed -nr 's,(^microk8s/[[:digit:]]+)\*.*,\1,p')
+    $juju_run_cmd --unit $mk8s_unit microk8s.config > ~/.kube/config
+    $juju_run_cmd --unit $mk8s_unit -- 'IPADDR=$( ip r get 2.2.2.2| sed -rn "s/.+src ([0-9\.]+) .+/\1/p"); microk8s enable metallb:$IPADDR-$IPADDR'
+fi
+
+kubectl get pods -A
+
+if ! $(juju list-clouds| egrep -q "^microk8s-cos"); then
+    KUBECONFIG=~/.kube/config juju add-k8s microk8s-cos --cluster-name=microk8s-cluster --client --controller ${OS_PROJECT_NAME/_/-} --storage=ceph-xfs
+fi
+if ! $(juju list-models| egrep -q "^${COS_MODEL}"); then
+    juju add-model $COS_MODEL microk8s-cos
+    juju deploy cos-lite --overlay ${SCRIPT_DIR}/overlays/cos/cos-lite-offers.yaml --trust
+else
+    echo "INFO: model '$COS_MODEL' already exists - skipping deploy"
+    juju switch $COS_MODEL
+fi
+
+juju wait-for application grafana
+
+set +x
+echo "INFO: COS should now be reachable at the following endpoints:"
+juju run traefik/0 show-proxied-endpoints --format=yaml| yq '."traefik/0".results."proxied-endpoints"' | jq
+
+GRAFANA_PASSWORD=$(juju run grafana/leader get-admin-password --model cos 2>/dev/null| sed -rn 's/admin-password:\s+(.+)/\1/p')
+GRAFANA_USER=$(juju config grafana admin_user)
+echo "Grafana login info: ${GRAFANA_USER}/$GRAFANA_PASSWORD"
+
+echo "INFO: run the following to consume COS from your microk8s model:"
+echo "juju switch <microk8s model>"
+for offer in $(juju list-offers| tail -n+2| awk '{print $1}'); do
+    echo "juju consume ${COS_MODEL}.$offer"
+done

cos/cos.yaml.template

@@ -0,0 +1,17 @@
+series: __SERIES__
+machines:
+  '0':
+    constraints: __MACHINE1_CONSTRAINTS__
+    series: __SERIES__
+applications:
+  microk8s:
+    charm: __CHARM_STORE____CHARM_CS_NS____CHARM_CH_PREFIX__microk8s
+    num_units: 1
+    expose: true
+    options:
+      containerd_http_proxy: __CONTAINERD_PROXY__
+      containerd_https_proxy: __CONTAINERD_PROXY__
+      containerd_no_proxy: __CONTAINERD_NO_PROXY__
+    to:
+    - 0
+

cos/generate-bundle.sh

@@ -0,0 +1 @@
+common/generate-bundle.sh

cos/module_defaults

@@ -0,0 +1,18 @@
+# This file must contain defaults for all variables used in bundles/overlays.
+# They are used to render to final product in the event they are not provided
+# elsewhere. It is inserted into the global context at the start of the
+# pipeline.
+#
+# You can check that none are missing by running lint/check_var_defaults.sh
+#
+
+MOD_PARAMS[__MICROK8S_CHANNEL__]="latest/edge"
+MOD_PARAMS[__CONTAINERD_PROXY__]=''
+MOD_PARAMS[__CONTAINERD_NO_PROXY__]='127.0.0.1,localhost,::1,10.149.0.0/16,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16'
+MOD_PARAMS[__NUM_MICROK8S_UNITS__]=1
+MOD_PARAMS[__NUM_COS_PROXY_UNITS__]=1
+MOD_PARAMS[__NUM_CEPH_MON_UNITS__]=3
+MOD_PARAMS[__NUM_CEPH_OSD_UNITS__]=3
+MOD_PARAMS[__MACHINE1_CONSTRAINTS__]='mem=8G'
+MOD_PARAMS[__MACHINE2_CONSTRAINTS__]='mem=8G'
+MOD_PARAMS[__MACHINE3_CONSTRAINTS__]='mem=8G'

cos/overlays

@@ -0,0 +1 @@
+../overlays

cos/pipeline/00setup

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Globals
+export MOD_NAME=cos
+export MOD_BASE_TEMPLATE=cos.yaml.template
+export MOD_SSL_STATE_DIR=${MOD_NAME}
+[ -n "${MASTER_OPTS[BUNDLE_NAME]}" ] && \
+    MOD_SSL_STATE_DIR="${MOD_SSL_STATE_DIR}-${MASTER_OPTS[BUNDLE_NAME]}"
+
+# opts that 02configure does not recognise that get passed to the generator
+export -a MOD_PASSTHROUGH_OPTS=()
+
+# Collection of messages to display at the end
+export -A MOD_MSGS=()
+# Use order 0 to ensure this is first displayed
+MOD_MSGS[0_common.0]="run ./configure to initialise your deployment"
+
+# Array list of overlays to use with this deployment.
+export -a MOD_OVERLAYS=()
+
+export -A MOD_PARAMS=()

cos/pipeline/01import-config-defaults

@@ -0,0 +1,5 @@
+# Start with dependency defaults in case we want to override any locally
+. $MOD_DIR/../.module_defaults/ceph
+
+# Current module imports
+. $MOD_DIR/module_defaults

cos/pipeline/02configure

@@ -0,0 +1,64 @@
+#!/bin/bash
+# Global variables are first defined in 00setup and module
+# dependencies are defined in 01import-config-defaults
+#
+# All overlay/bundle variables (MOD_PARAMS) defaults must go into
+# the <module>/module_defaults file.
+
+target=$series
+[ -z "$pocket" ] || target=${target}-$pocket
+target=${target}:${MOD_PARAMS[__MICROK8S_CHANNEL__]}
+MOD_PASSTHROUGH_OPTS+=( --release-name $target )
+
+# Automatically use proxy if in prodstack only
+if $(timeout 1s getent hosts squid.internal &> /dev/null) && [ -z "${MOD_PARAMS[__CONTAINERD_PROXY__]}" ]; then
+    MOD_MSGS[1_proxy.0]='PROXY: squid.internal exists, setting containerd proxy to http://squid.internal:3128'
+    MOD_PARAMS[__CONTAINERD_PROXY__]=http://squid.internal:3128
+fi
+
+if ! has_opt --charmed-ceph-lxd && ! has_opt --microceph; then
+    MOD_OVERLAYS+=( "cos/charmed-ceph.yaml" )
+    MOD_OVERLAYS+=( "cos/ceph-csi.yaml" )
+fi
+
+# Skip processing input if it includes exclusive passthrough options
+! has_excl_passthrough_opt && \
+while (($# > 0))
+do
+    case "$1" in
+        --containerd-proxy)  #__OPT__type:<str> (default="" unless the hostname squid.internal resolves, then it's http://squid.internal:3128)
+            MOD_PARAMS[__CONTAINERD_PROXY__]=$2
+            shift
+            ;;
+        --containerd-no-proxy)  #__OPT__type:<str> (default=127.0.0.1,localhost,::1,10.149.0.0/16,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16)
+            MOD_PARAMS[__CONTAINERD_NO_PROXY__]=$2
+            shift
+            ;;
+        --microceph)
+            MOD_OVERLAYS+=( "cos/microceph.yaml" )
+            ;;
+        --charmed-ceph-lxd)
+            MOD_OVERLAYS+=( "cos/charmed-ceph-lxd.yaml" )
+            ;;
+        --cos-proxy)
+            MOD_OVERLAYS+=( "cos/cos-proxy.yaml" )
+            MOD_MSGS[2_cos-proxy.0]='Once the COS deployment is complete you need to do the following:'
+            MOD_MSGS[2_cos-proxy.1]='juju relate grafana-dashboards:grafana-dashboard cos-proxy:downstream-grafana-dashboard'
+            MOD_MSGS[2_cos-proxy.2]='juju relate loki-logging:logging cos-proxy:downstream-logging'
+            MOD_MSGS[2_cos-proxy.3]='juju relate prometheus-scrape:metrics-endpoint cos-proxy:downstream-prometheus-scrape'
+            ;;
+        --filebeat)
+            MOD_OVERLAYS+=( "cos/cos-proxy-filebeat.yaml" )
+            if ! has_opt --cos-proxy; then
+                set -- $@ --cos-proxy && cache $@
+            fi
+            ;;
+        *)
+            echo "ERROR: invalid input '$1'"
+            _usage
+            exit 1
+            ;;
+    esac
+    shift
+done || true
+

cos/pipeline/03build

@@ -0,0 +1,5 @@
+#!/bin/bash
+. $MOD_DIR/common/generate_bundle_base
+
+print_msgs
+

cos/resources/README_resources.md

@@ -0,0 +1,5 @@
+If you want to configure charm resources [1] in a bundle, put a directory in
+this path with the name of the bundle/overlay they correspond to. Any files
+within will then be copied into the generated bundles' path.
+
+[1] https://docs.jujucharms.com/using-resources-developer-guide

overlays/cos/ceph-csi.yaml

@@ -0,0 +1,11 @@
+applications:
+  ceph-csi:
+    charm: __CHARM_STORE____CHARM_CS_NS____CHARM_CH_PREFIX__ceph-csi
+    # See https://github.com/charmed-kubernetes/ceph-csi-operator/issues/22
+    channel: 1.31/stable
+    options:
+      provisioner-replicas: 1
+      namespace: kube-system
+relations:
+  - [ ceph-csi:ceph-client, ceph-mon:client ]
+  - [ ceph-csi:kubernetes-info, microk8s ]

overlays/cos/charmed-ceph-lxd.yaml

@@ -0,0 +1,34 @@
+machines:
+  '0':
+    constraints: __MACHINE1_CONSTRAINTS__
+    series: __SERIES__
+  '1':
+    constraints: __MACHINE2_CONSTRAINTS__
+    series: __SERIES__
+  '2':
+    constraints: __MACHINE3_CONSTRAINTS__
+    series: __SERIES__
+applications:
+  ceph-mon:
+    charm: __CHARM_STORE____CHARM_CS_NS____CHARM_CH_PREFIX__ceph-mon
+    channel: quincy/stable
+    num_units: __NUM_CEPH_MON_UNITS__
+    to:
+    - 0
+    - 1
+    - 2
+  ceph-osd:
+    charm: __CHARM_STORE____CHARM_CS_NS____CHARM_CH_PREFIX__ceph-osd
+    channel: quincy/stable
+    num_units: __NUM_CEPH_OSD_UNITS__
+    to:
+    - 0
+    - 1
+    - 2
+    options:
+      osd-devices: ''  # must be empty string when using juju storage
+      config-flags: '{"osd": {"osd memory target": 1073741824}}' # matching 2G constraint
+    storage:
+      osd-devices: cinder,10G,1
+relations:
+  - [ ceph-osd:mon, ceph-mon:osd ]

overlays/cos/charmed-ceph.yaml

@@ -0,0 +1,34 @@
+machines:
+  '0':
+    constraints: __MACHINE1_CONSTRAINTS__
+    series: __SERIES__
+  '1':
+    constraints: __MACHINE2_CONSTRAINTS__
+    series: __SERIES__
+  '2':
+    constraints: __MACHINE3_CONSTRAINTS__
+    series: __SERIES__
+applications:
+  ceph-mon:
+    charm: __CHARM_STORE____CHARM_CS_NS____CHARM_CH_PREFIX__ceph-mon
+    channel: quincy/stable
+    num_units: __NUM_CEPH_MON_UNITS__
+    to:
+    - 0
+    - 1
+    - 2
+  ceph-osd:
+    charm: __CHARM_STORE____CHARM_CS_NS____CHARM_CH_PREFIX__ceph-osd
+    channel: quincy/stable
+    num_units: __NUM_CEPH_OSD_UNITS__
+    to:
+    - 0
+    - 1
+    - 2
+    options:
+      osd-devices: ''  # must be empty string when using juju storage
+      config-flags: '{"osd": {"osd memory target": 1073741824}}' # matching 2G constraint
+    storage:
+      osd-devices: cinder,10G,1
+relations:
+  - [ ceph-osd:mon, ceph-mon:osd ]

overlays/cos/cos-lite-offers.yaml

@@ -0,0 +1,24 @@
+applications:
+  alertmanager:
+    offers:
+      alertmanager-karma-dashboard:
+        endpoints:
+          - karma-dashboard
+  grafana:
+    offers:
+      grafana-dashboards:
+        endpoints:
+          - grafana-dashboard
+  loki:
+    offers:
+      loki-logging:
+        endpoints:
+        - logging
+  prometheus:
+    offers:
+      prometheus-scrape:
+        endpoints:
+        - metrics-endpoint
+      prometheus-receive-remote-write:
+        endpoints:
+        - receive-remote-write

overlays/cos/cos-proxy-filebeat.yaml

@@ -0,0 +1,10 @@
+applications:
+  filebeat:
+    charm: __CHARM_STORE____CHARM_CS_NS____CHARM_CH_PREFIX__filebeat
+    channel: stable
+    options:
+      logpath: /var/log/*.log /var/log/*/*.log /var/log/syslog
+relations:
+ - [ceph-mon:juju-info, filebeat:beats-host]
+ - [cos-proxy:filebeat, filebeat:logstash]
+ - [cos-proxy:juju-info, filebeat:beats-host]

overlays/cos/cos-proxy.yaml

@@ -0,0 +1,18 @@
+machines:
+  '0':
+    constraints: __MACHINE1_CONSTRAINTS__
+    series: __SERIES__
+  '1':
+    constraints: __MACHINE2_CONSTRAINTS__
+    series: __SERIES__
+  '2':
+    constraints: __MACHINE3_CONSTRAINTS__
+    series: __SERIES__
+applications:
+  cos-proxy:
+    charm: __CHARM_STORE____CHARM_CS_NS____CHARM_CH_PREFIX__cos-proxy
+    channel: stable
+    num_units: 1
+    to:
+    - 0
+

overlays/cos/microceph.yaml

@@ -0,0 +1,20 @@
+machines:
+  '0':
+    constraints: __MACHINE1_CONSTRAINTS__
+    series: __SERIES__
+  '1':
+    constraints: __MACHINE2_CONSTRAINTS__
+    series: __SERIES__
+  '2':
+    constraints: __MACHINE3_CONSTRAINTS__
+    series: __SERIES__
+applications:
+  microceph:
+    charm: microceph
+    channel: quincy/stable
+    num_units: __NUM_CEPH_OSD_UNITS__
+    to:
+    - 0
+    - 1
+    - 2
+

overlays/unit_placement/cos-proxy.yaml.template

@@ -0,0 +1,3 @@
+  cos-proxy:
+    to:
+__UNIT_PLACEMENT_METAL__.__UNITS__.__NUM_COS_PROXY_UNITS__

overlays/unit_placement/microk8s.yaml.template

@@ -0,0 +1,3 @@
+  microk8s:
+    to:
+__UNIT_PLACEMENT_METAL__.__UNITS__.__NUM_MICROK8S_UNITS__

tox.ini

@@ -24,6 +24,9 @@ pyfiles =
     {toxinidir}/tools/juju-bundle-applications.py
 
 bashfiles =
+    # find -name configure| grep -v .git| sed -rn 's,^.,{toxinidir},p'
+    {toxinidir}/cos/configure
+
     {toxinidir}/tools/juju-lnav
     # find tools -name \*.sh
     {toxinidir}/tools/vault-unseal-and-authorise.sh