Add cloudtruth_metadata attribute to kubernetes resources (#5)

* Add cloudtruth_metadata attribute to kuberenetes resources.  Used to show which project a param is getting its value from due to project inclusion/overrides

* add project_heirarchy metadata, use yaml for easier reading when displayed in tools

* ignore files for local dev

* prevent a project from including itself

* describe metadata output

.gitignore

@@ -2,6 +2,7 @@
 /.yardoc
 /_yardoc/
 /coverage/
+/local/
 /doc/
 /pkg/
 /spec/reports/

README.md

@@ -40,7 +40,8 @@ Parameterize the helm install with `--set appSettings.**` to control how kubetru
 | appSettings.apiKey | The cloudtruth api key.  Read only access is sufficient | string | n/a | yes |
 | appSettings.environment | The cloudtruth environment to lookup parameter values for.  Use a separate helm install for each environment | string | `default` | yes |
 | appSettings.pollingInterval | Interval to poll cloudtruth api for changes | integer | 300 | no |
-| appSettings.debug | Debug logging | flag | n/a | no |
+| appSettings.noMetadata | Do not write cloudtruth metadata (e.g. param value origins) to kubernetes resources | flag | false | no |
+| appSettings.debug | Debug logging | flag | false | no |
 | projectMappings.root.project_selector | A regexp to limit the projects acted against (client-side).  Supplies any named matches for template evaluation | string | "" | no |
 | projectMappings.root.key_selector | A regexp to limit the keys acted against (client-side).  Supplies any named matches for template evaluation | string | "" | no |
 | projectMappings.root.key_filter | Limits the keys fetched to contain the given substring (server-side, api search param) | string | "" | no |
@@ -148,6 +149,17 @@ ones:
  * dns_safe - ensures the string is safe for use as a kubernetes resource name (i.e. Namespace/ConfigMap/Secret names)
  * env_safe - ensures the string is safe for setting as a shell environment variable
 
+By default, kubetruth will add the `cloudtruth_metadata` key to each ConfigMap
+and Secret under management.  This can be disabled with the `noMetadata` helm
+setting at install time.  The data contained by this key helps to illustrate how
+project inclusion affects the project the resources were written for.   It
+currently shows the project heirarchy and the project each parameter originates
+from, for example an entry like `timeout: myService (commonService -> common)`
+indicates that the timeout parameter is getting its value from the `myService`
+project, and if you removed it from there, it would then get it from the
+`commonService` project, and if you removed that, it would then get it from the
+`common` project.
+
 ### Example Config
 
 The `projectmapping` resource has a shortname of `pm` for convenience when using kubectl.

helm/kubetruth/templates/deployment.yaml

@@ -48,6 +48,9 @@ spec:
             - --polling-interval
             - "{{ .Values.appSettings.pollingInterval }}"
             {{- end }}
+            {{- if .Values.appSettings.noMetadata }}
+            - --no-metadata
+            {{- end }}
             {{- if .Values.appSettings.debug }}
             - --debug
             {{- end }}

helm/kubetruth/values.yaml

@@ -69,9 +69,9 @@ affinity: {}
 appSettings:
   apiKey:
   environment:
+  noMetadata: false
   pollingInterval:
   debug: false
-  config:
 
 # Create instances of the ProjectMapping CRD.  A single mapping with scope=root
 # is required (named root below).  You can also add multiple override mappings

lib/kubetruth/cli.rb

@@ -42,16 +42,20 @@ class CLI < Clamp::Command
       Integer(a)
     end
 
+    option "--[no-]metadata",
+           :flag, "Saves additional cloudtruth metadata in the kubernetes resources, e.g. the project origin for param values after inclusions/overrides are applied",
+           default: true
+
     option ["-n", "--dry-run"],
-           :flag, "perform a dry run",
+           :flag, "Perform a dry run",
            default: false
 
     option ["-q", "--quiet"],
-           :flag, "suppress output",
+           :flag, "Suppress output",
            default: false
 
     option ["-d", "--debug"],
-           :flag, "debug output",
+           :flag, "Debug output",
            default: false
 
     option ["-c", "--[no-]color"],
@@ -92,7 +96,7 @@ def execute
           api_url: kube_url
       }
 
-      etl = ETL.new(ct_context: ct_context, kube_context: kube_context, dry_run: dry_run?)
+      etl = ETL.new(ct_context: ct_context, kube_context: kube_context, dry_run: dry_run?, metadata: metadata?)
 
       etl.with_polling(polling_interval) do
         etl.apply

lib/kubetruth/etl.rb

@@ -9,10 +9,11 @@ module Kubetruth
   class ETL
     include GemLogger::LoggerSupport
 
-    def initialize(ct_context:, kube_context:, dry_run: false)
+    def initialize(ct_context:, kube_context:, dry_run: false, metadata: true)
       @ct_context = ct_context
       @kube_context = kube_context
       @dry_run = dry_run
+      @metadata = metadata
       @kubeapis = {}
     end
 
@@ -131,17 +132,34 @@ def apply
           next
         end
 
+        param_origins = {}
+
         # TODO: make project inclusion recursive?
         included_params = []
         project_spec.included_projects.each do |included_project|
+          if included_project == project
+            logger.warn("Skipping project's import of itself, included_projects for '#{project}' are: #{project_spec.included_projects.inspect}")
+            next
+          end
           included_data = project_data[included_project]
           if included_data.nil?
             logger.warn "Skipping the included project not selected by root selector: #{included_project}"
             next
           end
+
+          included_data[:params].each do |p|
+            param_origins[p.key] ||= []
+            param_origins[p.key] << included_project
+          end
+
           included_params.concat(included_data[:params])
         end
 
+        data[:params].each do |p|
+          param_origins[p.key] ||= []
+          param_origins[p.key] << project
+        end
+
         # constructing the hash will cause any overrides to happen in the right
         # order (includer wins over last included over first included)
         params = included_params + data[:params]
@@ -150,6 +168,26 @@ def apply
         config_param_hash = params_to_hash(config_params)
         secret_param_hash = params_to_hash(secret_params)
 
+        if @metadata
+          metadata = {}
+          metadata["project_heirarchy"] = (project_spec.included_projects + [project]).reverse.join(" -> ")
+
+          param_origins.merge!(param_origins) do |_, v|
+            origin = "#{v.pop}"
+            if v.length > 0
+              origin << " (#{v.reverse.join(" -> ")})"
+            end
+            origin
+          end
+
+          param_origins_parts = param_origins.group_by {|k, v| config_param_hash.has_key?(k) }
+          config_origins = Hash[param_origins_parts[true] || []]
+          secret_origins = Hash[param_origins_parts[false] || []]
+
+          config_param_hash[:cloudtruth_metadata] = metadata.merge({ "parameter_origins" => config_origins }).to_yaml
+          secret_param_hash[:cloudtruth_metadata] = metadata.merge({ "parameter_origins" => secret_origins }).to_yaml
+        end
+
         apply_config_map(namespace: data[:namespace], name: data[:configmap_name], param_hash: config_param_hash)
 
         if ! project_spec.skip_secrets

spec/kubetruth/cli_spec.rb

@@ -95,6 +95,7 @@ def all_usage(clazz, path=[])
             --kube-url ku
             --dry-run
             --polling-interval 27
+            --no-metadata
         ]
         etl = double(ETL)
         expect(ETL).to receive(:new).with(ct_context: {
@@ -107,7 +108,8 @@ def all_usage(clazz, path=[])
                                               token: "kt",
                                               api_url: "ku"
                                           },
-                                          dry_run: true).and_return(etl)
+                                          dry_run: true,
+                                          metadata: false).and_return(etl)
         expect(etl).to receive(:with_polling).with(27)
         cli.run(args)
       end

spec/kubetruth/etl_spec.rb

@@ -407,8 +407,8 @@ class ForceExit < Exception; end
         ]
         expect(etl.ctapi).to receive(:project_names).and_return(["default"])
         expect(etl).to receive(:get_params).and_return(params)
-        expect(etl).to receive(:apply_config_map).with(namespace: '', name: "default", param_hash: etl.params_to_hash([params[0]]))
-        expect(etl).to receive(:apply_secret).with(namespace: '', name: "default", param_hash: etl.params_to_hash([params[1]]))
+        expect(etl).to receive(:apply_config_map).with(namespace: '', name: "default", param_hash: hash_including(etl.params_to_hash([params[0]])))
+        expect(etl).to receive(:apply_secret).with(namespace: '', name: "default", param_hash: hash_including(etl.params_to_hash([params[1]])))
         etl.apply()
       end
 
@@ -420,7 +420,7 @@ class ForceExit < Exception; end
         etl.load_config.root_spec.skip_secrets = true
         expect(etl.ctapi).to receive(:project_names).and_return(["default"])
         expect(etl).to receive(:get_params).and_return(params)
-        expect(etl).to receive(:apply_config_map).with(namespace: '', name: "default", param_hash: etl.params_to_hash([params[0]]))
+        expect(etl).to receive(:apply_config_map).with(namespace: '', name: "default", param_hash: hash_including(etl.params_to_hash([params[0]])))
         expect(etl).to_not receive(:apply_secret)
         etl.apply()
       end
@@ -486,7 +486,7 @@ class ForceExit < Exception; end
         expect(etl).to receive(:apply_config_map).
           with(namespace: 'bar-foo',
                name: "foo.bar",
-               param_hash: etl.params_to_hash([Parameter.new(key: "foo:bar:foo.bar:param1", original_key: "param1", value: "value1", secret: false),]))
+               param_hash: hash_including(etl.params_to_hash([Parameter.new(key: "foo:bar:foo.bar:param1", original_key: "param1", value: "value1", secret: false),])))
         expect(etl).to receive(:apply_secret)
         etl.apply
       end
@@ -502,23 +502,119 @@ class ForceExit < Exception; end
         ]
 
         expect(etl).to receive(:load_config).and_return(Kubetruth::Config.new([
-          {
-            scope: "root",
-            included_projects: ["base"]
-          },
-          {scope: "override", project_selector: "^base$", skip: true}
-        ]))
+                                                                                {
+                                                                                  scope: "root",
+                                                                                  included_projects: ["base"]
+                                                                                },
+                                                                                {scope: "override", project_selector: "^base$", skip: true}
+                                                                              ]))
 
         expect(etl.ctapi).to receive(:project_names).and_return(["base", "foo"])
         expect(etl).to receive(:get_params).with("base", any_args).and_return(base_params)
         expect(etl).to receive(:get_params).with("foo", any_args).and_return(foo_params)
-        expect(etl).to receive(:apply_config_map).with(namespace: '', name: "foo", param_hash: {
-          "param0" => "value0",
-          "param1" => "value1",
-          "param2" => "value2"
-        })
+        expect(etl).to receive(:apply_config_map).with(namespace: '', name: "foo", param_hash: hash_including({
+                                                                                                                "param0" => "value0",
+                                                                                                                "param1" => "value1",
+                                                                                                                "param2" => "value2"
+                                                                                                              }))
+        allow(etl).to receive(:apply_secret)
+        etl.apply()
+      end
+
+      it "skips project include of self" do
+        base_params = [
+          Parameter.new(key: "param0", value: "value0", secret: false),
+        ]
+
+        expect(etl).to receive(:load_config).and_return(Kubetruth::Config.new([
+                                                                                {
+                                                                                  scope: "root",
+                                                                                  included_projects: ["base"]
+                                                                                }
+                                                                              ]))
+
+        expect(etl.ctapi).to receive(:project_names).and_return(["base"])
+        expect(etl).to receive(:get_params).with("base", any_args).and_return(base_params)
+        expect(etl).to receive(:apply_config_map)
         allow(etl).to receive(:apply_secret)
         etl.apply()
+        expect(Logging.contents).to include("Skipping project's import of itself")
+      end
+
+      it "indicates param's project origin in metadata" do
+        base_params = [
+          Parameter.new(key: "param0", value: "value0", secret: false),
+          Parameter.new(key: "param2", value: "basevalue2", secret: false),
+          Parameter.new(key: "param3", value: "basevalue3", secret: false),
+          Parameter.new(key: "sparam0", value: "svalue0", secret: true),
+          Parameter.new(key: "sparam2", value: "sbasevalue2", secret: true),
+          Parameter.new(key: "sparam3", value: "sbasevalue3", secret: true)
+        ]
+        bar_params = [
+          Parameter.new(key: "param3", value: "barvalue3", secret: false),
+          Parameter.new(key: "sparam3", value: "sbarvalue3", secret: true),
+        ]
+        foo_params = [
+          Parameter.new(key: "param1", value: "value1", secret: false),
+          Parameter.new(key: "param2", value: "value2", secret: false),
+          Parameter.new(key: "param3", value: "value3", secret: false),
+          Parameter.new(key: "sparam1", value: "svalue1", secret: true),
+          Parameter.new(key: "sparam2", value: "svalue2", secret: true),
+          Parameter.new(key: "sparam3", value: "svalue3", secret: true)
+        ]
+
+        expect(etl).to receive(:load_config).and_return(Kubetruth::Config.new([
+                                                                                {
+                                                                                  scope: "root",
+                                                                                  included_projects: ["base", "bar"]
+                                                                                },
+                                                                                {scope: "override", project_selector: "^ba.*$", skip: true}
+                                                                              ]))
+
+        expect(etl.ctapi).to receive(:project_names).and_return(["base", "bar", "foo"])
+        expect(etl).to receive(:get_params).with("base", any_args).and_return(base_params)
+        expect(etl).to receive(:get_params).with("bar", any_args).and_return(bar_params)
+        expect(etl).to receive(:get_params).with("foo", any_args).and_return(foo_params)
+        expect(etl).to receive(:apply_config_map) do |*args, **kwargs|
+          expect(YAML.load(kwargs[:param_hash][:cloudtruth_metadata])).to eq({
+                                                                               "project_heirarchy" => "foo -> bar -> base",
+                                                                                "parameter_origins" => {
+                                                                                  "param0" => "base",
+                                                                                  "param1" => "foo",
+                                                                                  "param2" => "foo (base)",
+                                                                                  "param3" => "foo (bar -> base)"
+                                                                                }
+                                                                              })
+        end
+        expect(etl).to receive(:apply_secret) do |*args, **kwargs|
+          expect(YAML.load(kwargs[:param_hash][:cloudtruth_metadata])).to eq({
+                                                                                "project_heirarchy" => "foo -> bar -> base",
+                                                                                "parameter_origins" => {
+                                                                                  "sparam0" => "base",
+                                                                                  "sparam1" => "foo",
+                                                                                  "sparam2" => "foo (base)",
+                                                                                  "sparam3" => "foo (bar -> base)"
+                                                                                }
+                                                                              })
+        end
+        etl.apply()
+      end
+
+      it "can turn off metadata" do
+        etl = described_class.new(init_args.merge(metadata: false))
+        params = [
+          Parameter.new(key: "param1", value: "value1", secret: false),
+          Parameter.new(key: "param2", value: "value2", secret: true)
+        ]
+        expect(etl.ctapi).to receive(:project_names).and_return(["default"])
+        expect(etl).to receive(:get_params).and_return(params)
+        expect(etl).to receive(:apply_config_map) do |*args, **kwargs|
+          expect(kwargs[:param_hash]).to_not include(:cloudtruth_metadata)
+        end
+        expect(etl).to receive(:apply_secret) do |*args, **kwargs|
+          expect(kwargs[:param_hash]).to_not include(:cloudtruth_metadata)
+        end
+        etl.apply()
       end
 
     end