diff --git a/.gitignore b/.gitignore index 43152df..89af2f2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /.yardoc /_yardoc/ /coverage/ +/local/ /doc/ /pkg/ /spec/reports/ diff --git a/README.md b/README.md index 022616b..24eb146 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,8 @@ Parameterize the helm install with `--set appSettings.**` to control how kubetru | appSettings.apiKey | The cloudtruth api key. Read only access is sufficient | string | n/a | yes | | appSettings.environment | The cloudtruth environment to lookup parameter values for. Use a separate helm install for each environment | string | `default` | yes | | appSettings.pollingInterval | Interval to poll cloudtruth api for changes | integer | 300 | no | -| appSettings.debug | Debug logging | flag | n/a | no | +| appSettings.noMetadata | Do not write cloudtruth metadata (e.g. param value origins) to kubernetes resources | flag | false | no | +| appSettings.debug | Debug logging | flag | false | no | | projectMappings.root.project_selector | A regexp to limit the projects acted against (client-side). Supplies any named matches for template evaluation | string | "" | no | | projectMappings.root.key_selector | A regexp to limit the keys acted against (client-side). Supplies any named matches for template evaluation | string | "" | no | | projectMappings.root.key_filter | Limits the keys fetched to contain the given substring (server-side, api search param) | string | "" | no | @@ -148,6 +149,17 @@ ones: * dns_safe - ensures the string is safe for use as a kubernetes resource name (i.e. Namespace/ConfigMap/Secret names) * env_safe - ensures the string is safe for setting as a shell environment variable +By default, kubetruth will add the `cloudtruth_metadata` key to each ConfigMap +and Secret under management. This can be disabled with the `noMetadata` helm +setting at install time. The data contained by this key helps to illustrate how +project inclusion affects the project the resources were written for. It +currently shows the project heirarchy and the project each parameter originates +from, for example an entry like `timeout: myService (commonService -> common)` +indicates that the timeout parameter is getting its value from the `myService` +project, and if you removed it from there, it would then get it from the +`commonService` project, and if you removed that, it would then get it from the +`common` project. + ### Example Config The `projectmapping` resource has a shortname of `pm` for convenience when using kubectl. diff --git a/helm/kubetruth/templates/deployment.yaml b/helm/kubetruth/templates/deployment.yaml index cb2c40e..cef9ce4 100644 --- a/helm/kubetruth/templates/deployment.yaml +++ b/helm/kubetruth/templates/deployment.yaml @@ -48,6 +48,9 @@ spec: - --polling-interval - "{{ .Values.appSettings.pollingInterval }}" {{- end }} + {{- if .Values.appSettings.noMetadata }} + - --no-metadata + {{- end }} {{- if .Values.appSettings.debug }} - --debug {{- end }} diff --git a/helm/kubetruth/values.yaml b/helm/kubetruth/values.yaml index faf80b7..02c5244 100644 --- a/helm/kubetruth/values.yaml +++ b/helm/kubetruth/values.yaml @@ -69,9 +69,9 @@ affinity: {} appSettings: apiKey: environment: + noMetadata: false pollingInterval: debug: false - config: # Create instances of the ProjectMapping CRD. A single mapping with scope=root # is required (named root below). You can also add multiple override mappings diff --git a/lib/kubetruth/cli.rb b/lib/kubetruth/cli.rb index 3f17c6d..1419e8a 100755 --- a/lib/kubetruth/cli.rb +++ b/lib/kubetruth/cli.rb @@ -42,16 +42,20 @@ class CLI < Clamp::Command Integer(a) end + option "--[no-]metadata", + :flag, "Saves additional cloudtruth metadata in the kubernetes resources, e.g. the project origin for param values after inclusions/overrides are applied", + default: true + option ["-n", "--dry-run"], - :flag, "perform a dry run", + :flag, "Perform a dry run", default: false option ["-q", "--quiet"], - :flag, "suppress output", + :flag, "Suppress output", default: false option ["-d", "--debug"], - :flag, "debug output", + :flag, "Debug output", default: false option ["-c", "--[no-]color"], @@ -92,7 +96,7 @@ def execute api_url: kube_url } - etl = ETL.new(ct_context: ct_context, kube_context: kube_context, dry_run: dry_run?) + etl = ETL.new(ct_context: ct_context, kube_context: kube_context, dry_run: dry_run?, metadata: metadata?) etl.with_polling(polling_interval) do etl.apply diff --git a/lib/kubetruth/etl.rb b/lib/kubetruth/etl.rb index afe922c..8efb446 100644 --- a/lib/kubetruth/etl.rb +++ b/lib/kubetruth/etl.rb @@ -9,10 +9,11 @@ module Kubetruth class ETL include GemLogger::LoggerSupport - def initialize(ct_context:, kube_context:, dry_run: false) + def initialize(ct_context:, kube_context:, dry_run: false, metadata: true) @ct_context = ct_context @kube_context = kube_context @dry_run = dry_run + @metadata = metadata @kubeapis = {} end @@ -131,17 +132,34 @@ def apply next end + param_origins = {} + # TODO: make project inclusion recursive? included_params = [] project_spec.included_projects.each do |included_project| + if included_project == project + logger.warn("Skipping project's import of itself, included_projects for '#{project}' are: #{project_spec.included_projects.inspect}") + next + end included_data = project_data[included_project] if included_data.nil? logger.warn "Skipping the included project not selected by root selector: #{included_project}" next end + + included_data[:params].each do |p| + param_origins[p.key] ||= [] + param_origins[p.key] << included_project + end + included_params.concat(included_data[:params]) end + data[:params].each do |p| + param_origins[p.key] ||= [] + param_origins[p.key] << project + end + # constructing the hash will cause any overrides to happen in the right # order (includer wins over last included over first included) params = included_params + data[:params] @@ -150,6 +168,26 @@ def apply config_param_hash = params_to_hash(config_params) secret_param_hash = params_to_hash(secret_params) + if @metadata + metadata = {} + metadata["project_heirarchy"] = (project_spec.included_projects + [project]).reverse.join(" -> ") + + param_origins.merge!(param_origins) do |_, v| + origin = "#{v.pop}" + if v.length > 0 + origin << " (#{v.reverse.join(" -> ")})" + end + origin + end + + param_origins_parts = param_origins.group_by {|k, v| config_param_hash.has_key?(k) } + config_origins = Hash[param_origins_parts[true] || []] + secret_origins = Hash[param_origins_parts[false] || []] + + config_param_hash[:cloudtruth_metadata] = metadata.merge({ "parameter_origins" => config_origins }).to_yaml + secret_param_hash[:cloudtruth_metadata] = metadata.merge({ "parameter_origins" => secret_origins }).to_yaml + end + apply_config_map(namespace: data[:namespace], name: data[:configmap_name], param_hash: config_param_hash) if ! project_spec.skip_secrets diff --git a/spec/kubetruth/cli_spec.rb b/spec/kubetruth/cli_spec.rb index 14b0560..f3ccefd 100644 --- a/spec/kubetruth/cli_spec.rb +++ b/spec/kubetruth/cli_spec.rb @@ -95,6 +95,7 @@ def all_usage(clazz, path=[]) --kube-url ku --dry-run --polling-interval 27 + --no-metadata ] etl = double(ETL) expect(ETL).to receive(:new).with(ct_context: { @@ -107,7 +108,8 @@ def all_usage(clazz, path=[]) token: "kt", api_url: "ku" }, - dry_run: true).and_return(etl) + dry_run: true, + metadata: false).and_return(etl) expect(etl).to receive(:with_polling).with(27) cli.run(args) end diff --git a/spec/kubetruth/etl_spec.rb b/spec/kubetruth/etl_spec.rb index f9bab29..7879641 100644 --- a/spec/kubetruth/etl_spec.rb +++ b/spec/kubetruth/etl_spec.rb @@ -407,8 +407,8 @@ class ForceExit < Exception; end ] expect(etl.ctapi).to receive(:project_names).and_return(["default"]) expect(etl).to receive(:get_params).and_return(params) - expect(etl).to receive(:apply_config_map).with(namespace: '', name: "default", param_hash: etl.params_to_hash([params[0]])) - expect(etl).to receive(:apply_secret).with(namespace: '', name: "default", param_hash: etl.params_to_hash([params[1]])) + expect(etl).to receive(:apply_config_map).with(namespace: '', name: "default", param_hash: hash_including(etl.params_to_hash([params[0]]))) + expect(etl).to receive(:apply_secret).with(namespace: '', name: "default", param_hash: hash_including(etl.params_to_hash([params[1]]))) etl.apply() end @@ -420,7 +420,7 @@ class ForceExit < Exception; end etl.load_config.root_spec.skip_secrets = true expect(etl.ctapi).to receive(:project_names).and_return(["default"]) expect(etl).to receive(:get_params).and_return(params) - expect(etl).to receive(:apply_config_map).with(namespace: '', name: "default", param_hash: etl.params_to_hash([params[0]])) + expect(etl).to receive(:apply_config_map).with(namespace: '', name: "default", param_hash: hash_including(etl.params_to_hash([params[0]]))) expect(etl).to_not receive(:apply_secret) etl.apply() end @@ -486,7 +486,7 @@ class ForceExit < Exception; end expect(etl).to receive(:apply_config_map). with(namespace: 'bar-foo', name: "foo.bar", - param_hash: etl.params_to_hash([Parameter.new(key: "foo:bar:foo.bar:param1", original_key: "param1", value: "value1", secret: false),])) + param_hash: hash_including(etl.params_to_hash([Parameter.new(key: "foo:bar:foo.bar:param1", original_key: "param1", value: "value1", secret: false),]))) expect(etl).to receive(:apply_secret) etl.apply end @@ -502,23 +502,119 @@ class ForceExit < Exception; end ] expect(etl).to receive(:load_config).and_return(Kubetruth::Config.new([ - { - scope: "root", - included_projects: ["base"] - }, - {scope: "override", project_selector: "^base$", skip: true} - ])) + { + scope: "root", + included_projects: ["base"] + }, + {scope: "override", project_selector: "^base$", skip: true} + ])) expect(etl.ctapi).to receive(:project_names).and_return(["base", "foo"]) expect(etl).to receive(:get_params).with("base", any_args).and_return(base_params) expect(etl).to receive(:get_params).with("foo", any_args).and_return(foo_params) - expect(etl).to receive(:apply_config_map).with(namespace: '', name: "foo", param_hash: { - "param0" => "value0", - "param1" => "value1", - "param2" => "value2" - }) + expect(etl).to receive(:apply_config_map).with(namespace: '', name: "foo", param_hash: hash_including({ + "param0" => "value0", + "param1" => "value1", + "param2" => "value2" + })) + allow(etl).to receive(:apply_secret) + etl.apply() + end + + it "skips project include of self" do + base_params = [ + Parameter.new(key: "param0", value: "value0", secret: false), + ] + + expect(etl).to receive(:load_config).and_return(Kubetruth::Config.new([ + { + scope: "root", + included_projects: ["base"] + } + ])) + + expect(etl.ctapi).to receive(:project_names).and_return(["base"]) + expect(etl).to receive(:get_params).with("base", any_args).and_return(base_params) + expect(etl).to receive(:apply_config_map) allow(etl).to receive(:apply_secret) etl.apply() + expect(Logging.contents).to include("Skipping project's import of itself") + end + + it "indicates param's project origin in metadata" do + base_params = [ + Parameter.new(key: "param0", value: "value0", secret: false), + Parameter.new(key: "param2", value: "basevalue2", secret: false), + Parameter.new(key: "param3", value: "basevalue3", secret: false), + Parameter.new(key: "sparam0", value: "svalue0", secret: true), + Parameter.new(key: "sparam2", value: "sbasevalue2", secret: true), + Parameter.new(key: "sparam3", value: "sbasevalue3", secret: true) + ] + bar_params = [ + Parameter.new(key: "param3", value: "barvalue3", secret: false), + Parameter.new(key: "sparam3", value: "sbarvalue3", secret: true), + ] + foo_params = [ + Parameter.new(key: "param1", value: "value1", secret: false), + Parameter.new(key: "param2", value: "value2", secret: false), + Parameter.new(key: "param3", value: "value3", secret: false), + Parameter.new(key: "sparam1", value: "svalue1", secret: true), + Parameter.new(key: "sparam2", value: "svalue2", secret: true), + Parameter.new(key: "sparam3", value: "svalue3", secret: true) + ] + + expect(etl).to receive(:load_config).and_return(Kubetruth::Config.new([ + { + scope: "root", + included_projects: ["base", "bar"] + }, + {scope: "override", project_selector: "^ba.*$", skip: true} + ])) + + expect(etl.ctapi).to receive(:project_names).and_return(["base", "bar", "foo"]) + expect(etl).to receive(:get_params).with("base", any_args).and_return(base_params) + expect(etl).to receive(:get_params).with("bar", any_args).and_return(bar_params) + expect(etl).to receive(:get_params).with("foo", any_args).and_return(foo_params) + expect(etl).to receive(:apply_config_map) do |*args, **kwargs| + expect(YAML.load(kwargs[:param_hash][:cloudtruth_metadata])).to eq({ + "project_heirarchy" => "foo -> bar -> base", + "parameter_origins" => { + "param0" => "base", + "param1" => "foo", + "param2" => "foo (base)", + "param3" => "foo (bar -> base)" + } + }) + end + expect(etl).to receive(:apply_secret) do |*args, **kwargs| + expect(YAML.load(kwargs[:param_hash][:cloudtruth_metadata])).to eq({ + "project_heirarchy" => "foo -> bar -> base", + "parameter_origins" => { + "sparam0" => "base", + "sparam1" => "foo", + "sparam2" => "foo (base)", + "sparam3" => "foo (bar -> base)" + } + }) + end + etl.apply() + end + + it "can turn off metadata" do + etl = described_class.new(init_args.merge(metadata: false)) + params = [ + Parameter.new(key: "param1", value: "value1", secret: false), + Parameter.new(key: "param2", value: "value2", secret: true) + ] + expect(etl.ctapi).to receive(:project_names).and_return(["default"]) + expect(etl).to receive(:get_params).and_return(params) + expect(etl).to receive(:apply_config_map) do |*args, **kwargs| + expect(kwargs[:param_hash]).to_not include(:cloudtruth_metadata) + end + expect(etl).to receive(:apply_secret) do |*args, **kwargs| + expect(kwargs[:param_hash]).to_not include(:cloudtruth_metadata) + end + etl.apply() end end