Add support for cloudtruth projects with the ability to merge data ac…

…ross projects, removed the regex dispatching from keys to configmap/secret names, moved config to a yaml file within a kubetruth configmap

Gemfile

@@ -5,6 +5,7 @@ gemspec
 
 gem "rake", "~> 12.0"
 gem "rspec", "~> 3.0"
+gem "test_construct"
 gem "vcr"
 gem "webmock"
 gem "coveralls", ">= 0.8.23"

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    kubetruth (0.2.2)
+    kubetruth (0.3.0)
       activesupport
       clamp
       gem_logger
@@ -113,6 +113,7 @@ GEM
     sync (0.5.0)
     term-ansicolor (1.7.1)
       tins (~> 1.0)
+    test_construct (2.0.2)
     thor (1.0.1)
     tins (1.26.0)
       sync
@@ -139,6 +140,7 @@ DEPENDENCIES
   rake (~> 12.0)
   rspec (~> 3.0)
   simplecov
+  test_construct
   vcr
   webmock
 

README.md

@@ -34,67 +34,109 @@ Parameterize the helm install with `--set appSettings.**` to control how kubetru
 |-----------|-------------|------|---------|:--------:|
 | appSettings.apiKey | The cloudtruth api key.  Read only access is sufficient | string | n/a | yes |
 | appSettings.environment | The cloudtruth environment to lookup parameter values for.  Use a separate helm install for each environment | string | `default` | yes |
-| appSettings.keyPrefix | Limit the parameters looked up to one of these prefixes | list(string) | n/a | no |
-| appSettings.keyPattern | The pattern to match against key names to select params and provide keywords for generating resource names via nameTemplate and keyTemplate | list(regex) | `^(?<prefix>[^\.]+)\.(?<name>[^\.]+)\.(?<key>.*)` | no |
-| appSettings.namespaceTemplate | The template for generating the namespace that resources get created in | string | n/a | no |
-| appSettings.nameTemplate | The template for generating resources (ConfigMaps and Secrets) | string | `%{name}` | no |
-| appSettings.keyTemplate | The template for generating key names within a resource | string | `%{key}` | no |
-| appSettings.skipSecrets | Do not transfer parameters that are marked as secret | flag | false | no |
-| appSettings.secretsAsConfig | Place secret parameters alongside plain parameters within a ConfigMap instead of in their own Secret resource | flag | false | no |
 | appSettings.pollingInterval | Interval to poll cloudtruth api for changes | integer | 300 | no |
 | appSettings.debug | Debug logging | flag | n/a | no |
 
-For example, for a keyspace that looks like:
+By default, Kubetruth maps the parameters from CloudTruth Projects into ConfigMaps and Secrets of the same names as the Projects. 
+
+For example, for a CloudTruth layout that looks like:
+
+`myProject`:
 ```
-service.someServiceName.oneParam=value1
-service.someServiceName.twoParam=value2
-service.otherServiceName.someParam=val1
-service.otherServiceName.mySecret=val2 (marked as a secret within CloudTruth)
+oneParam=value1
+twoParam=value2
 ```
 
-and parameterization like:
+`otherProject`:
 ```
-    --set appSettings.keyPrefix=service \
-    --set appSettings.keyPattern=^(?<prefix>[^\.]+)\.(?<name>[^\.]+)\.(?<key>.*) \
-    --set appSettings.nameTemplate=%{name} \
-    --set appSettings.keyTemplate=ACME_%{key_upcase} \
+someParam=value3
+mySecret=value4 (marked as a secret within CloudTruth)
 ```
 
-Kubetruth will generate the config maps:
+Kubetruth will generate the kubernetes resources:
 
-someServiceName:
+ConfigMap named `myProject`:
 ```yaml
-    ACME_ONEPARAM: value1
-    ACME_TWOPARAM: value2
+    oneParam: value1
+    twoParam: value2
 ```
 
-otherServiceName:
+ConfigMap named `otherProject`:
 ```yaml
-    ACME_SOMEPARAM: val1
+    someParam: value3
 ```
 
-and the Secrets:
-
-otherServiceName:
+Secret named `otherProject`:
 ```yaml
-    MYSECRET: val2
+    mySecret: val2
 ```
 
-These kubernetes resources can then be referenced in the standard ways, e.g.
+These kubernetes resources can then be referenced in the standard ways.
 
+To use them as environment variables in a pod:
 ```yaml
     envFrom:
       - configMapRef:
-          name: otherServiceName
+          name: otherProject
     envFrom:
       - secretRef:
-          name: otherServiceName
+          name: otherProject
+```
+
+To use them as files on disk in a pod:
+```yaml
+      containers:
+        - name: myProject
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/myConfig
+      volumes:
+        - name: config-volume
+          configMap:
+            name: myProject
 ```
 
 Note that config map updates don't get seen by a running pod.  You can use
 something like [Reloader](https://github.com/stakater/Reloader) to automate
 this, or read config from mounted volumes for configmaps/secrets, which do get
-updated automatically in a running pod
+updated automatically in a running pod.
+
+## Additional configuration
+
+The kubetruth ConfigMap contains a [yaml file for additional config](helm/kubetruth/templates/configmap.yaml)
+
+### Example Config
+
+To create the kubernetes Resources in namespaces named after each Project:
+```yaml
+namespace_template: %{project}
+```
+
+To include the parameters from a Project named `Base` into all other projects, without creating Resources for `Base` itself:
+```yaml
+included_projects:
+  - Base
+project_overrides:
+  - project_selector: Base
+    skip: true
+```
+
+To override the naming of kubernetes Resources on a per-Project basis:
+```yaml
+project_overrides:
+  - project_selector: funkyProject
+    configmap_name_template: notSoFunkyConfigMap
+    secret_name_template: notSoFunkySecret
+    namespace_template: notSoFunkyNsmespace
+```
+
+To limit the Projects processed to those whose names start with `service`, except for `serviceOddball`:
+```yaml
+project_selector: ^service
+project_overrides:
+  - project_selector: serviceOddball
+    skip: true
+```
 
 ## Development
 

helm/kubetruth/Chart.yaml

@@ -15,9 +15,9 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.2.2
+version: 0.3.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
-appVersion: 0.2.2
+appVersion: 0.3.0

helm/kubetruth/templates/configmap.yaml

@@ -0,0 +1,49 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "kubetruth.fullname" . }}
+  namespace:  {{ .Release.Namespace }}
+  labels:
+    {{- include "kubetruth.labels" . | nindent 4 }}
+data:
+  kubetruth.yaml: |-
+    # Configures how kubetruth fetches and maps parameters from CloudTruth to Kubernetes
+
+    #
+    # The following settings can all be overriden on a per-project basis in project_overrides
+    #
+
+    # Uses the given regexp to limit the projects acted against (client-side).  Supplies any named matches for template evaluation
+    # project_selector: ""
+
+    # Uses the given regexp to limit the keys acted against (client-side).  Supplies any named matches for template evaluation
+    # key_selector: ""
+
+    # Limits the keys fetched to contain the given substring (server-side, api search param)
+    # key_filter: ""
+
+    # The template to use in generating ConfigMap names
+    # configmap_name_template: "%{project}"
+    # The template to use in generating Secret names
+    # secret_name_template: "%{project}"
+    # The template to use in generating namespace names
+    # namespace_template: ""
+    # The template to use in generating key names
+    # key_template: "%{key}"
+
+    # Skips the generation of resources for the selected projects.  Useful for
+    # excluding projects that should only be included into others.
+    # skip: false
+
+    # Do not transfer secrets to kubernetes resources
+    # skip_secrets: false
+
+    # Include the data from other projects into the selected ones
+    # included_projects: []
+
+    # Override any of the above settings for specific projects by name pattern
+    # project_overrides:
+    #  - project_selector: oddballProject
+    #    configmap_name_template: special-name
+    #  - project_name: someOtherProject
+    #    key_template: %{key_upcase}

helm/kubetruth/templates/deployment.yaml

@@ -38,46 +38,26 @@ spec:
           envFrom:
             - secretRef:
                 name: {{ include "kubetruth.fullname" . }}
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/kubetruth
           args:
             - app
             {{- if .Values.appSettings.environment }}
             - --environment
             - {{ .Values.appSettings.environment | quote }}
             {{- end }}
-            {{- range .Values.appSettings.keyPrefix }}
-            - --key-prefix
-            - {{ . | quote }}
-            {{- end }}
-            {{- range .Values.appSettings.keyPattern }}
-            - --key-pattern
-            - {{ . | quote }}
-            {{- end }}
-            {{- if .Values.appSettings.namespaceTemplate }}
-            - --namespace-template
-            - {{ .Values.appSettings.namespaceTemplate | quote }}
-            {{- end }}
-            {{- if .Values.appSettings.nameTemplate }}
-            - --name-template
-            - {{ .Values.appSettings.nameTemplate | quote }}
-            {{- end }}
-            {{- if .Values.appSettings.keyTemplate }}
-            - --key-template
-            - {{ .Values.appSettings.keyTemplate | quote }}
-            {{- end }}
-            {{- if .Values.appSettings.skipSecrets }}
-            - --skip-secrets
-            {{- end }}
-            {{- if .Values.appSettings.secretsAsConfig }}
-            - --secrets-as-config
-            {{- end }}
             {{- if .Values.appSettings.pollingInterval }}
             - --polling-interval
             - "{{ .Values.appSettings.pollingInterval }}"
             {{- end }}
             {{- if .Values.appSettings.debug }}
             - --debug
             {{- end }}
-
+      volumes:
+        - name: config-volume
+          configMap:
+            name: {{ include "kubetruth.fullname" . }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

helm/kubetruth/values.yaml

@@ -69,12 +69,6 @@ affinity: {}
 appSettings:
   apiKey:
   environment:
-  keyPrefix: []
-  keyPattern: []
-  namespaceTemplate:
-  nameTemplate:
-  keyTemplate:
-  skipSecrets: false
-  secretsAsConfig: false
   pollingInterval:
   debug: false
+  config:

lib/kubetruth/cli.rb

@@ -26,33 +26,9 @@ class CLI < Clamp::Command
            environment_variable: 'CT_API_KEY',
            required: true
 
-    option "--namespace-template", "TMPL", "the template for generating the namespace name where configmap/secrets are created from key pattern matches.  Defaults to the namespace kubetruth is running in"
-
-    option "--name-template", "TMPL", "the template for generating the configmap/secret name from key pattern matches",
-           default: "%{name}"
-
-    option "--key-template", "TMPL", "the template for generating the configmap/secret keys from key pattern matches",
-           default: "%{key}"
-
-    option "--key-prefix", "PREFIX", "the key prefix to restrict the keys fetched from cloudtruth",
-           default: [''],
-             multivalued: true
-
-    option "--key-pattern", "REGEX", "the key pattern for mapping cloudtruth params to configmap keys.  The `name` is used for the config map naming, and the keys in that map come from the matching `key` portion.  A pattern like `^(?<key>[^\\.]+.(?<name>[^\\.]+)\\..*)`  would make the key be the entire  parameter key",
-           default: [/^(?<prefix>[^\.]+)\.(?<name>[^\.]+)\.(?<key>.*)/],
-           multivalued: true do |a|
-      Regexp.new(a)
-    rescue RegexpError => e
-      raise ArgumentError.new(e.message)
-    end
-
-    option "--skip-secrets",
-           :flag, "Do not transfer secrets to kubernetes resources",
-           default: false
-
-    option "--secrets-as-config",
-           :flag, "Secrets are placed in config maps instead of kube secrets",
-           default: false
+    option ["-f", "--config-file"],
+           'FILE', "The kubetruth.yml file",
+           default: "/etc/kubetruth/kubetruth.yaml"
 
     option "--kube-namespace",
            'NAMESPACE', "The kubernetes namespace. Defaults to runtime namespace when run in kube"
@@ -117,14 +93,12 @@ def execute
           api_url: kube_url
       }
 
-      etl = ETL.new(key_prefixes: key_prefix_list, key_patterns: key_pattern_list,
-                    namespace_template: namespace_template, name_template: name_template, key_template: key_template,
-                    ct_context: ct_context, kube_context: kube_context)
+      etl = ETL.new(config_file: config_file, ct_context: ct_context, kube_context: kube_context)
 
       while true
 
         begin
-          etl.apply(dry_run: dry_run?, skip_secrets: skip_secrets?, secrets_as_config: secrets_as_config?)
+          etl.apply(dry_run: dry_run?)
         rescue => e
           logger.log_exception(e, "Failure while applying config transforms")
         end