enable setting of log level from project mapping crds

README.md

@@ -48,6 +48,7 @@ Parameterize the helm install with `--set *` or `--values yourConfig.yaml` to co
 | projectMappings.root.project_selector | A regexp to limit the projects acted against (client-side).  Supplies any named matches for template evaluation | string | "" | no |
 | projectMappings.root.key_selector | A regexp to limit the keys acted against (client-side).  Supplies any named matches for template evaluation | string | "" | no |
 | projectMappings.root.skip | Skips the generation of resources for the selected projects | flag | false | no |
+| projectMappings.root.log_level | Sets the kubetruth logging level while handling the selected projects | enum(debug, info, warn, error, fatal) | as set by cli | no |
 | projectMappings.root.included_projects | Include the parameters from other projects into the selected ones.  This can be recursive in a depth first fashion, so if A imports B and B imports C, then A will get B's and C's parameters.  For key conflicts, if A includes B and B includes C, then the precendence is A overrides B overrides C.  If A includes \[B, C], then the precendence is A overrides C overrides B. | list | [] | no |
 | projectMappings.root.context | Additional variables made available to the resource templates.  Can also be templates | map | [default](helm/kubetruth/values.yaml#L93-L129) | no |
 | projectMappings.root.resource_templates | The templates to use in generating kubernetes resources (ConfigMap/Secrets/other) | map | [default](helm/kubetruth/values.yaml#L93-L129) | no |

helm/helmv2/templates/projectmapping.yaml

@@ -30,6 +30,10 @@ spec:
         skip:
           type: boolean
           description: Skips the generation of resources for the selected projects.  Useful for excluding projects that should only be included into others.
+        log_level:
+          type: string
+          description: The level of logging to use
+          enum: ["debug", "info", "warn", "error", "fatal"]
         included_projects:
           type: array
           items:

helm/kubetruth/crds/projectmapping.yaml

@@ -32,6 +32,10 @@ spec:
                 skip:
                   type: boolean
                   description: Skips the generation of resources for the selected projects
+                log_level:
+                  type: string
+                  description: The level of logging to use
+                  enum: ["debug", "info", "warn", "error", "fatal"]
                 included_projects:
                   type: array
                   items:

lib/kubetruth/config.rb

@@ -14,6 +14,7 @@ class DuplicateSelection < Kubetruth::Error; end
       :key_selector,
       :environment,
       :skip,
+      :log_level,
       :included_projects,
       :context,
       :resource_templates,

lib/kubetruth/etl.rb

@@ -115,6 +115,16 @@ def load_config
       configs
     end
 
+    def with_log_level(level)
+      original_root_log_level = Kubetruth::Logging.root_log_level
+      begin
+        Kubetruth::Logging.root_log_level = level if level
+        yield
+      ensure
+        Kubetruth::Logging.root_log_level = original_root_log_level
+      end
+    end
+
     def apply
       async(annotation: "ETL Event Loop") do
         logger.warn("Performing dry-run") if @dry_run
@@ -123,89 +133,93 @@ def apply
         CtApi.reset
 
         load_config do |namespace, config|
-          project_collection = ProjectCollection.new
-
-          # Load all projects that are used
-          all_specs = [config.root_spec] + config.override_specs
-          project_selectors = all_specs.collect(&:project_selector)
-          included_projects = all_specs.collect(&:included_projects).flatten.uniq
-
-          project_collection.names.each do |project_name|
-            active = included_projects.any? {|p| p == project_name }
-            active ||= project_selectors.any? {|s| s =~ project_name }
-            if active
-              project_spec = config.spec_for_project(project_name)
-              project_collection.create_project(name: project_name, spec: project_spec)
+          with_log_level(config.root_spec.log_level) do
+            project_collection = ProjectCollection.new
+
+            # Load all projects that are used
+            all_specs = [config.root_spec] + config.override_specs
+            project_selectors = all_specs.collect(&:project_selector)
+            included_projects = all_specs.collect(&:included_projects).flatten.uniq
+
+            project_collection.names.each do |project_name|
+              active = included_projects.any? {|p| p == project_name }
+              active ||= project_selectors.any? {|s| s =~ project_name }
+              if active
+                project_spec = config.spec_for_project(project_name)
+                project_collection.create_project(name: project_name, spec: project_spec)
+              end
             end
-          end
 
-          project_collection.projects.values.each do |project|
+            project_collection.projects.values.each do |project|
+              with_log_level(project.spec.log_level) do
+                logger.info "Processing project '#{project.name}'"
 
-            match = project.name.match(project.spec.project_selector)
-            if match.nil?
-              logger.info "Skipping project '#{project.name}' as it does not match any selectors"
-              next
-            end
+                match = project.name.match(project.spec.project_selector)
+                if match.nil?
+                  logger.info "Skipping project '#{project.name}' as it does not match any selectors"
+                  next
+                end
 
-            if project.spec.skip
-              logger.info "Skipping project '#{project.name}'"
-              next
-            end
+                if project.spec.skip
+                  logger.info "Skipping project '#{project.name}'"
+                  next
+                end
 
-            async(annotation: "Project: #{project.name}") do
+                async(annotation: "Project: #{project.name}") do
 
-              # constructing the hash will cause any overrides to happen in the right
-              # order (includer wins over last included over first included)
-              params = project.all_parameters
-              parts = params.group_by(&:secret)
-              config_params, secret_params = (parts[false] || []), (parts[true] || [])
-              config_param_hash = params_to_hash(config_params)
-              secret_param_hash = params_to_hash(secret_params)
+                  # constructing the hash will cause any overrides to happen in the right
+                  # order (includer wins over last included over first included)
+                  params = project.all_parameters
+                  parts = params.group_by(&:secret)
+                  config_params, secret_params = (parts[false] || []), (parts[true] || [])
+                  config_param_hash = params_to_hash(config_params)
+                  secret_param_hash = params_to_hash(secret_params)
 
-              parameter_origins = project.parameter_origins
-              param_origins_parts = parameter_origins.group_by {|k, v| config_param_hash.has_key?(k) }
-              config_origins = Hash[param_origins_parts[true] || []]
-              secret_origins = Hash[param_origins_parts[false] || []]
+                  parameter_origins = project.parameter_origins
+                  param_origins_parts = parameter_origins.group_by {|k, v| config_param_hash.has_key?(k) }
+                  config_origins = Hash[param_origins_parts[true] || []]
+                  secret_origins = Hash[param_origins_parts[false] || []]
 
-              config_param_hash = config_param_hash.reject do |k, v|
-                logger.debug { "Excluding parameter with nil value: #{k}" } if v.nil?
-                v.nil?
-              end
-              secret_param_hash = secret_param_hash.reject do |k, v|
-                logger.debug { "Excluding secret parameter with nil value: #{k}" } if v.nil?
-                v.nil?
-              end
+                  config_param_hash = config_param_hash.reject do |k, v|
+                    logger.debug { "Excluding parameter with nil value: #{k}" } if v.nil?
+                    v.nil?
+                  end
+                  secret_param_hash = secret_param_hash.reject do |k, v|
+                    logger.debug { "Excluding secret parameter with nil value: #{k}" } if v.nil?
+                    v.nil?
+                  end
+
+                  project.spec.resource_templates.each_with_index do |pair, i|
+                    template_name, template = *pair
+                    logger.debug { "Processing template '#{template_name}' (#{i+1}/#{project.spec.resource_templates.size})" }
+                    resource_yml = template.render(
+                      template: template_name,
+                      kubetruth_namespace: kubeapi.namespace,
+                      mapping_namespace: namespace,
+                      project: project.name,
+                      project_heirarchy: project.heirarchy,
+                      debug: logger.debug?,
+                      parameters: config_param_hash,
+                      parameter_origins: config_origins,
+                      secrets: secret_param_hash,
+                      secret_origins: secret_origins,
+                      templates: Template::TemplatesDrop.new(project: project.name, environment: project.spec.environment),
+                      context: project.spec.context
+                    )
+
+                    template_id = "mapping: #{project.spec.name}, mapping_namespace: #{namespace}, project: #{project.name}, template: #{template_name}"
+                    parsed_ymls = YAML.safe_load_stream(resource_yml, template_id)
+                    logger.debug {"Skipping empty template"} if parsed_ymls.empty?
+                    parsed_ymls.each do |parsed_yml|
+                      async(annotation: "Apply Template: #{template_id}") do
+                        kube_apply(parsed_yml)
+                      end
+                    end
 
-              project.spec.resource_templates.each_with_index do |pair, i|
-                template_name, template = *pair
-                logger.debug { "Processing template '#{template_name}' (#{i+1}/#{project.spec.resource_templates.size})" }
-                resource_yml = template.render(
-                  template: template_name,
-                  kubetruth_namespace: kubeapi.namespace,
-                  mapping_namespace: namespace,
-                  project: project.name,
-                  project_heirarchy: project.heirarchy,
-                  debug: logger.debug?,
-                  parameters: config_param_hash,
-                  parameter_origins: config_origins,
-                  secrets: secret_param_hash,
-                  secret_origins: secret_origins,
-                  templates: Template::TemplatesDrop.new(project: project.name, environment: project.spec.environment),
-                  context: project.spec.context
-                )
-
-                template_id = "mapping: #{project.spec.name}, mapping_namespace: #{namespace}, project: #{project.name}, template: #{template_name}"
-                parsed_ymls = YAML.safe_load_stream(resource_yml, template_id)
-                logger.debug {"Skipping empty template"} if parsed_ymls.empty?
-                parsed_ymls.each do |parsed_yml|
-                  async(annotation: "Apply Template: #{template_id}") do
-                    kube_apply(parsed_yml)
                   end
                 end
-
               end
             end
-
           end
         end
       end.wait

lib/kubetruth/logging.rb

@@ -67,6 +67,14 @@ def self.clear
       sio&.clear
     end
 
+    def self.root_log_level=(level)
+      ::Logging.logger.root.level = level
+    end
+
+    def self.root_log_level
+      ::Logging.levelify(::Logging::LNAMES[::Logging.logger.root.level])
+    end
+
     def self.setup_logging(level: :info, color: true)
       init_logger
 

spec/kubetruth/etl_spec.rb

@@ -207,6 +207,26 @@ class ForceExit < Exception; end
 
     end
 
+    describe "#with_log_level" do
+
+      it "does not set nil log level" do
+        expect(Kubetruth::Logging.root_log_level).to eq("debug")
+        etl.with_log_level(nil) do
+          expect(Kubetruth::Logging.root_log_level).to eq("debug")
+        end
+        expect(Kubetruth::Logging.root_log_level).to eq("debug")
+      end
+
+      it "temporarily sets log level" do
+        expect(Kubetruth::Logging.root_log_level).to eq("debug")
+        etl.with_log_level("error") do
+          expect(Kubetruth::Logging.root_log_level).to eq("error")
+        end
+        expect(Kubetruth::Logging.root_log_level).to eq("debug")
+      end
+
+    end
+
     describe "#kube_apply" do
 
       it "calls kube to create new resource" do
@@ -510,7 +530,7 @@ class ForceExit < Exception; end
                                                             ])
           project
         end
-        expect(collection).to receive(:names).and_return(["proj1"])
+        allow(collection).to receive(:names).and_return(["proj1"])
         allow(etl).to receive(:kube_apply)
       end
 
@@ -546,6 +566,38 @@ class ForceExit < Exception; end
         etl.apply()
       end
 
+      it "sets log level when supplied by root pm in config" do
+        Kubetruth::Logging.root_log_level = "error" # undo debug logging set by test harness
+        config.root_spec.log_level = "error"
+
+        allow(etl).to receive(:load_config).and_yield(@ns, config)
+        allow(config.root_spec.resource_templates["configmap"]).to receive(:render).and_return("")
+
+        etl.apply()
+        expect(Logging.contents).to_not match(/DEBUG/)
+        Logging.clear
+
+        config.root_spec.log_level = "debug"
+        etl.apply()
+        expect(Logging.contents).to match(/DEBUG/)
+      end
+
+      it "sets log level when supplied by override pm in config" do
+        Kubetruth::Logging.root_log_level = "error" # undo debug logging set by test harness
+
+        override_crd = {scope: "override", project_selector: "proj1", log_level: "debug"}
+        config = Kubetruth::Config.new([root_spec_crd, override_crd])
+
+        allow(etl).to receive(:load_config).and_yield(@ns, config)
+        allow(config.root_spec.resource_templates["configmap"]).to receive(:render).and_return("")
+
+        etl.apply()
+        expect(Logging.contents).to_not match(/DEBUG.*Config ProjectSpec for root mapping/)
+        expect(Logging.contents).to match(/INFO.*ETL Processing project 'proj1'/)
+        expect(Logging.contents).to match(/DEBUG.*Template Evaluating template/)
+      end
+
+
     end
 
     describe "verify async behavior" do

spec/kubetruth/logging_spec.rb

@@ -39,5 +39,24 @@ module Kubetruth
 
     end
 
+    describe "#root_log_level" do
+
+      it "gets and sets root log level" do
+        expect(described_class.root_log_level).to eq("debug")
+        described_class.root_log_level = "error"
+        expect(described_class.root_log_level).to eq("error")
+      end
+
+      it "logs at set log level" do
+        described_class.root_log_level = "info"
+        logger.info("infolog")
+        expect(Logging.contents).to include("infolog")
+        logger.debug("debuglog")
+        expect(Logging.contents).to_not include("debuglog")
+      end
+
+    end
+
   end
+
 end