Add the ability to use a namespaceTemplate to determine which namespa…

…ce the kube resources will be created in
cloudtruth · Feb 11, 2021 · 23707a7 · 23707a7
1 parent db66a96
commit 23707a7
Show file tree

Hide file tree

Showing 17 changed files with 2,450 additions and 136 deletions.
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    kubetruth (0.1.0)
+    kubetruth (0.2.0)
       activesupport
       clamp
       gem_logger
@@ -12,12 +12,12 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.0.3.4)
+    activesupport (6.1.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 0.7, < 2)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
-      zeitwerk (~> 2.2, >= 2.2.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
     byebug (11.1.3)
@@ -57,7 +57,7 @@ GEM
     http-form_data (2.3.0)
     http-parser (1.2.2)
       ffi-compiler
-    i18n (1.8.5)
+    i18n (1.8.7)
       concurrent-ruby (~> 1.0)
     json (2.3.1)
     jsonpath (1.0.6)
@@ -75,7 +75,7 @@ GEM
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2020.1104)
-    minitest (5.14.2)
+    minitest (5.14.3)
     multi_json (1.15.0)
     netrc (0.11.0)
     pry (0.13.1)
@@ -114,11 +114,10 @@ GEM
     term-ansicolor (1.7.1)
       tins (~> 1.0)
     thor (1.0.1)
-    thread_safe (0.3.6)
     tins (1.26.0)
       sync
-    tzinfo (1.2.8)
-      thread_safe (~> 0.1)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)

diff --git a/README.md b/README.md
@@ -37,6 +37,7 @@ Parameterize the helm install with `--set appSettings.**` to control how kubetru
 | appSettings.environment | The cloudtruth environment to lookup parameter values for.  Use a separate helm install for each environment | string | `default` | yes |
 | appSettings.keyPrefix | Limit the parameters looked up to one of these prefixes | list(string) | n/a | no |
 | appSettings.keyPattern | The pattern to match against key names to select params and provide keywords for generating resource names via nameTemplate and keyTemplate | list(regex) | `^(?<prefix>[^\.]+)\.(?<name>[^\.]+)\.(?<key>.*)` | no |
+| appSettings.namespaceTemplate | The template for generating the namespace that resources get created in | string | n/a | no |
 | appSettings.nameTemplate | The template for generating resources (ConfigMaps and Secrets) | string | `%{name}` | no |
 | appSettings.keyTemplate | The template for generating key names within a resource | string | `%{key}` | no |
 | appSettings.skipSecrets | Do not transfer parameters that are marked as secret | flag | false | no |
@@ -100,7 +101,14 @@ updated automatically in a running pod
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `bundle exec rspec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
+To install and run via helm in a local cluster:
+``` 
+docker build -t kubetruth . && helm install \
+    --set image.repository=kubetruth --set image.pullPolicy=Never --set image.tag=latest \
+    --set appSettings.debug=true --set appSettings.apiKey=$CT_API_KEY --set appSettings.environment=development \
+    kubetruth ./helm/kubetruth/
+```
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/cloudtruth/kubetruth.
-
diff --git a/helm/kubetruth/Chart.yaml b/helm/kubetruth/Chart.yaml
@@ -15,9 +15,9 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
+version: 0.2.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
-appVersion: 0.1.0
+appVersion: 0.2.0
diff --git a/helm/kubetruth/templates/deployment.yaml b/helm/kubetruth/templates/deployment.yaml
@@ -52,6 +52,10 @@ spec:
             - --key-pattern
             - {{ toYaml . | quote }}
             {{- end }}
+            {{- if .Values.appSettings.namespaceTemplate }}
+            - --namespace-template
+            - {{ .Values.appSettings.namespaceTemplate | quote }}
+            {{- end }}
             {{- if .Values.appSettings.nameTemplate }}
             - --name-template
             - {{ .Values.appSettings.nameTemplate | quote }}

diff --git a/helm/kubetruth/values.yaml b/helm/kubetruth/values.yaml
@@ -70,6 +70,7 @@ appSettings:
   environment:
   keyPrefix: []
   keyPattern: []
+  namespaceTemplate:
   nameTemplate:
   keyTemplate:
   skipSecrets: false

diff --git a/lib/kubetruth/cli.rb b/lib/kubetruth/cli.rb
@@ -26,10 +26,12 @@ class CLI < Clamp::Command
            environment_variable: 'CT_API_KEY',
            required: true
 
-    option "--name-template", "TMPL", "the template for generating the configmap name from key pattern matches",
+    option "--namespace-template", "TMPL", "the template for generating the namespace name where configmap/secrets are created from key pattern matches.  Defaults to the namespace kubetruth is running in"
+
+    option "--name-template", "TMPL", "the template for generating the configmap/secret name from key pattern matches",
            default: "%{name}"
 
-    option "--key-template", "TMPL", "the template for generating the configmap keys from key pattern matches",
+    option "--key-template", "TMPL", "the template for generating the configmap/secret keys from key pattern matches",
            default: "%{key}"
 
     option "--key-prefix", "PREFIX", "the key prefix to restrict the keys fetched from cloudtruth",
@@ -53,13 +55,13 @@ class CLI < Clamp::Command
            default: false
 
     option "--kube-namespace",
-           'NAMESPACE', "The kubernetes namespace Defaults to runtime namespace when run  in kube"
+           'NAMESPACE', "The kubernetes namespace. Defaults to runtime namespace when run in kube"
 
     option "--kube-token",
-           'TOKEN', "The kubernetes token to use for api Defaults to mounted when run  in kube"
+           'TOKEN', "The kubernetes token to use for api. Defaults to mounted when run in kube"
 
     option "--kube-url",
-           'ENDPOINT', "The kubernetes api url Defaults to internal api endpoint when  run in kube"
+           'ENDPOINT', "The kubernetes api url. Defaults to internal api endpoint when run in kube"
 
     option "--polling-interval", "INTERVAL", "the polling interval", default: 300 do |a|
       Integer(a)
@@ -116,7 +118,7 @@ def execute
       }
 
       etl = ETL.new(key_prefixes: key_prefix_list, key_patterns: key_pattern_list,
-                    name_template: name_template, key_template: key_template,
+                    namespace_template: namespace_template, name_template: name_template, key_template: key_template,
                     ct_context: ct_context, kube_context: kube_context)
 
       while true

diff --git a/lib/kubetruth/etl.rb b/lib/kubetruth/etl.rb
@@ -8,15 +8,17 @@ class ETL
     include GemLogger::LoggerSupport
 
     def initialize(key_prefixes:, key_patterns:,
-                   name_template:, key_template:,
+                   namespace_template:, name_template:, key_template:,
                    ct_context:, kube_context:)
 
       @key_prefixes = key_prefixes
       @key_patterns = key_patterns
       @name_template = name_template
+      @namespace_template = namespace_template
       @key_template = key_template
       @ct_context = ct_context
       @kube_context = kube_context
+      @kubeapis = {}
     end
 
     def ctapi
@@ -27,8 +29,8 @@ def ctapi
       end
     end
 
-    def kubeapi
-      @kubeapi ||= KubeApi.new(**@kube_context)
+    def kubeapi(namespace)
+      @kubeapis[namespace] ||= KubeApi.new(**@kube_context.merge(namespace: namespace))
     end
 
     def apply(dry_run: false, skip_secrets: false, secrets_as_config: false)
@@ -85,7 +87,7 @@ def get_param_groups
       end
       logger.debug { "Filtered params: #{filtered_params.inspect}"}
 
-      # Group those parameters by the name selected by the key_pattern
+      # Group those parameters by the name selected by the name_pattern
       #
       param_groups = {}
       @key_patterns.each do |key_pattern|
@@ -97,11 +99,15 @@ def get_param_groups
             matches_hash = Hash[*matches_hash.collect {|k, v| [k, v, "#{k}_upcase".to_sym, v.upcase]}.flatten]
 
             logger.debug {"Pattern matches '#{param.key}' with: #{matches_hash}"}
-            name = @name_template % matches_hash
+
+            namespace = dns_friendly(@namespace_template % matches_hash) if @namespace_template
+            name = dns_friendly(@name_template % matches_hash)
             key = @key_template % matches_hash
             param.original_key, param.key = param.key, key
-            param_groups[name] ||= []
-            param_groups[name] << param
+
+            group_key = {namespace: namespace, name: name}
+            param_groups[group_key] ||= []
+            param_groups[group_key] << param
           else
             logger.debug {"Pattern does not match '#{param.key}'"}
           end
@@ -126,26 +132,32 @@ def apply_config_maps(param_groups)
       # to the config map with that name
       #
 
-      logger.debug { "Existing config maps: #{kubeapi.get_config_map_names}" }
+      param_groups.collect {|k, v| k[:namespace] }.sort.uniq.each do |ns|
+        kapi = kubeapi(ns)
+        # only create namespace when user chooses to use multiple namespaces determined from the pattern
+        kapi.ensure_namespace if @namespace_template
+        logger.debug { "Existing config maps (ns=#{ns}): #{kapi.get_config_map_names}" }
+      end
 
       param_groups.each do |k, v|
-
-        config_map_name = dns_friendly(k)
-
+        config_map_namespace = k[:namespace]
+        config_map_name = k[:name]
+        kapi = kubeapi(config_map_namespace)
         param_hash = Hash[v.collect {|param| [param.key, param.value]}]
 
         begin
-          data = kubeapi.get_config_map(config_map_name)
+          logger.debug { "Namespace '#{kapi.namespace}'" }
+          data = kapi.get_config_map(config_map_name)
           logger.debug("Config map for '#{config_map_name}': #{data.inspect}")
           if param_hash != data.transform_keys! {|k| k.to_s }
             logger.info "Updating config map '#{config_map_name}' with params: #{param_hash.inspect}"
-            kubeapi.update_config_map(config_map_name, param_hash)
+            kapi.update_config_map(config_map_name, param_hash)
           else
             logger.info "No changes needed for config map '#{config_map_name}' with params: #{param_hash.inspect}}"
           end
         rescue Kubeclient::ResourceNotFoundError
           logger.info "Creating config map '#{config_map_name}' with params: #{param_hash.inspect}}"
-          kubeapi.create_config_map(config_map_name, param_hash)
+          kapi.create_config_map(config_map_name, param_hash)
         end
       end
     end
@@ -156,26 +168,35 @@ def apply_secrets(param_groups)
       # For each set of parameters grouped by name, add those parameters
       # to the secret with that name
       #
-      logger.debug { "Existing secrets: #{kubeapi.get_secret_names}" }
+
+      param_groups.collect {|k, v| k[:namespace] }.uniq.each do |ns|
+        kapi = kubeapi(ns)
+        # only create namespace when user chooses to use multiple namespaces determined from the pattern
+        kapi.ensure_namespace if @namespace_template
+        logger.debug { "Existing secrets (ns=#{kapi.namespace}): #{kapi.get_secret_names}" }
+      end
 
       param_groups.each do |k, v|
 
-        secret_name = dns_friendly(k)
+        secret_namespace = k[:namespace]
+        secret_name = k[:name]
+        kapi = kubeapi(secret_namespace)
 
         param_hash = Hash[v.collect {|param| [param.key, param.value]}]
 
         begin
-          data = kubeapi.get_secret(secret_name)
+          logger.debug { "Namespace '#{kapi.namespace}'" }
+          data = kapi.get_secret(secret_name)
           logger.debug("Secret for '#{secret_name}': #{data}")
           if param_hash != data.transform_keys! {|k| k.to_s }
             logger.info "Updating secret '#{secret_name}' with params: #{param_hash.keys.inspect}"
-            kubeapi.update_secret(secret_name, param_hash)
+            kapi.update_secret(secret_name, param_hash)
           else
             logger.info "No changes needed for secret '#{secret_name}' with params: #{param_hash.keys.inspect}}"
           end
         rescue Kubeclient::ResourceNotFoundError
           logger.info "Creating secret '#{secret_name}' with params: #{param_hash.keys.inspect}}"
-          kubeapi.create_secret(secret_name, param_hash)
+          kapi.create_secret(secret_name, param_hash)
         end
       end
     end

diff --git a/lib/kubetruth/kubeapi.rb b/lib/kubetruth/kubeapi.rb
@@ -40,6 +40,17 @@ def client
       )
     end
 
+    def ensure_namespace
+      begin
+        client.get_namespace(namespace)
+      rescue Kubeclient::ResourceNotFoundError
+        ns = Kubeclient::Resource.new
+        ns.metadata = {}
+        ns.metadata.name = namespace
+        client.create_namespace(ns)
+      end
+    end
+
     def get_config_map_names
       client.get_config_maps(namespace: namespace).collect(&:metadata).collect(&:name)
     end

diff --git a/lib/kubetruth/version.rb b/lib/kubetruth/version.rb
@@ -1,3 +1,3 @@
 module Kubetruth
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end