From ddd233ff4162d5e304b8aead0de659863e6cfaa2 Mon Sep 17 00:00:00 2001 From: Matt Conway Date: Wed, 20 Dec 2023 09:09:11 -0500 Subject: [PATCH] allow project specific PMs to override environment and tag --- .app.yml | 2 +- helm/kubetruth/Chart.yaml | 4 ++-- helm/kubetruth/values.yaml | 10 ---------- lib/kubetruth/ctapi.rb | 14 +++++++++++++- lib/kubetruth/etl.rb | 8 ++++++-- lib/kubetruth/project_collection.rb | 20 +++++++++++++------- spec/kubetruth/ctapi_spec.rb | 15 +++++++++++++++ spec/kubetruth/etl_spec.rb | 16 ++++++++++++++++ spec/kubetruth/project_collection_spec.rb | 4 ++-- 9 files changed, 68 insertions(+), 25 deletions(-) diff --git a/.app.yml b/.app.yml index 8f7694c..c01a01d 100644 --- a/.app.yml +++ b/.app.yml @@ -1,4 +1,4 @@ org: cloudtruth name: kubetruth -version: 1.2.5 +version: 1.2.6 diff --git a/helm/kubetruth/Chart.yaml b/helm/kubetruth/Chart.yaml index dac282a..3d87742 100644 --- a/helm/kubetruth/Chart.yaml +++ b/helm/kubetruth/Chart.yaml @@ -15,13 +15,13 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.2.5 +version: 1.2.6 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. -appVersion: 1.2.5 +appVersion: 1.2.6 diff --git a/helm/kubetruth/values.yaml b/helm/kubetruth/values.yaml index 4607617..246ce6a 100644 --- a/helm/kubetruth/values.yaml +++ b/helm/kubetruth/values.yaml @@ -124,11 +124,6 @@ projectMappings: namespace: "{{ context.resource_namespace }}" labels: version: "{{ parameters | sort | to_json | sha256 | slice: 0, 7 }}" - annotations: - kubetruth/project_heirarchy: | - {{ project_heirarchy | to_yaml | indent: 6 | lstrip }} - kubetruth/parameter_origins: | - {{ parameter_origins | to_yaml | indent: 6 | lstrip }} data: {%- for parameter in parameters %} {{ parameter[0] | key_safe | stringify }}: {{ parameter[1] | stringify }} @@ -144,11 +139,6 @@ projectMappings: namespace: "{{ context.resource_namespace }}" labels: version: "{{ secrets | sort | to_json | sha256 | slice: 0, 7 }}" - annotations: - kubetruth/project_heirarchy: | - {{ project_heirarchy | to_yaml | indent: 6 | lstrip }} - kubetruth/parameter_origins: | - {{ secret_origins | to_yaml | indent: 6 | lstrip }} data: {%- for secret in secrets %} {{ secret[0] | key_safe | stringify }}: {{ secret[1] | encode64 | stringify }} diff --git a/lib/kubetruth/ctapi.rb b/lib/kubetruth/ctapi.rb index 82d92d7..ec62c56 100644 --- a/lib/kubetruth/ctapi.rb +++ b/lib/kubetruth/ctapi.rb @@ -8,6 +8,7 @@ class CtApi include GemLogger::LoggerSupport @@config = nil + @@ctapis = {} def self.configure(api_key:, api_url:) if api_key.nil? || api_url.nil? @@ -29,7 +30,7 @@ def self.configure(api_key:, api_url:) @@config = config end - attr_reader :client, :apis + attr_reader :client, :apis, :environment, :tag class ApiConfiguration < CloudtruthClient::Configuration @@ -50,6 +51,17 @@ def auth_settings end + # Factory methods to allow caching of CtApi instances through a single + # polling cycle to mitigate costs of fetching all projects/environments for + # ID lookup + def self.create(environment: "default", tag: nil) + @@ctapis[[environment, tag]] ||= CtApi.new(environment: environment, tag: tag) + end + + def self.reset + @@ctapis = {} + end + def initialize(environment: "default", tag: nil) @environments_mutex = Mutex.new @projects_mutex = Mutex.new diff --git a/lib/kubetruth/etl.rb b/lib/kubetruth/etl.rb index de712d2..d7f472f 100644 --- a/lib/kubetruth/etl.rb +++ b/lib/kubetruth/etl.rb @@ -218,6 +218,9 @@ def params end def apply + # clear the ctapi cache before we start + Kubetruth::CtApi.reset + async(annotation: "ETL Event Loop") do # Only do the concurrency limit across ctapi calls for project listing @@ -233,7 +236,7 @@ def apply load_config do |namespace, config| with_log_level(config.root_spec.log_level) do - project_collection = ProjectCollection.new(config.root_spec) + project_collection = ProjectCollection.new(config) # Load all projects that are used all_specs = [config.root_spec] + config.override_specs @@ -291,7 +294,8 @@ def apply secrets: proc { param_data.params[:secrets] }, secret_origins: proc { param_data.params[:secret_origins] }, templates: Template::TemplatesDrop.new(project: project.name, ctapi: project.ctapi), - context: project.spec.context + context: project.spec.context, + environment: project.spec.environment ) template_id = "mapping: #{project.spec.name}, mapping_namespace: #{namespace}, project: #{project.name}, template: #{template_name}" diff --git a/lib/kubetruth/project_collection.rb b/lib/kubetruth/project_collection.rb index 7650bdf..e2bb7bd 100644 --- a/lib/kubetruth/project_collection.rb +++ b/lib/kubetruth/project_collection.rb @@ -7,20 +7,26 @@ class ProjectCollection attr_accessor :projects - def initialize(project_spec) + def initialize(config) @projects = {} - @project_spec = project_spec - end - - def ctapi - @ctapi ||= Kubetruth::CtApi.new(environment: @project_spec.environment, tag: @project_spec.tag) + @config = config end def names - ctapi.project_names + # NOTE: listing projects is done using env/tag from root spec, and not the + # env/tag from project specific overrides. This could cause an issue if + # the tag on the root spec differs from the tag on the project spec in + # such a way that the listing of projects gets (doesn't) one that is not + # (is) visible to the project tag, but makes for a better UX by allowing + # an override spec to override env/tag in root spec + # the ctapi create factory method caches based on env/tag + Kubetruth::CtApi.create(environment: @config.root_spec.environment, tag: @config.root_spec.tag).project_names end def create_project(*args, **kwargs) + # the ctapi create factory method caches based on env/tag + spec = kwargs[:spec] + ctapi = Kubetruth::CtApi.create(environment: spec.environment, tag: spec.tag) project = Project.new(*args, **kwargs.merge(collection: self, ctapi: ctapi)) projects[project.name] = project project diff --git a/spec/kubetruth/ctapi_spec.rb b/spec/kubetruth/ctapi_spec.rb index 6336b70..206f962 100644 --- a/spec/kubetruth/ctapi_spec.rb +++ b/spec/kubetruth/ctapi_spec.rb @@ -50,6 +50,21 @@ def create_project_fixture end + describe "#create/reset" do + + it "caches instance" do + described_class.configure(api_key: api_key, api_url: api_url) + expect(described_class.create).to eq(described_class.create) + expect(described_class.create).to_not eq(described_class.create(environment: "foo")) + expect(described_class.create).to_not eq(described_class.create(tag: "foo")) + + orig = described_class.create + described_class.reset + expect(described_class.create).to_not eq(orig) + end + + end + describe "#cookies" do it "uses session cookie" do diff --git a/spec/kubetruth/etl_spec.rb b/spec/kubetruth/etl_spec.rb index c557935..36110d1 100644 --- a/spec/kubetruth/etl_spec.rb +++ b/spec/kubetruth/etl_spec.rb @@ -763,6 +763,22 @@ class ForceExit < Exception; end expect(Logging.contents).to match(/DEBUG.*Template Evaluating template/) end + it "overrides environment from override pm in config" do + # ensure just a single template to make render check on any_instance easy and valid + root_spec_crd[:resource_templates].delete_if {|k, v| k.to_s != "configmap"} + override_crd = {scope: "override", project_selector: "proj1", environment: "production", skip: false} + allow(etl.kubeapi).to receive(:get_project_mappings).and_return({"primary-ns" => {"root" => root_spec_crd.merge(skip: true)}, "secondary-ns" => {"proj1" => override_crd}}) + + # Once for root spec to get project listing + expect(Kubetruth::CtApi).to receive(:new).with(environment: "default", tag: nil).and_return(@ctapi).once + # Once for rendering for a project + expect(Kubetruth::CtApi).to receive(:new).with(environment: "production", tag: nil).and_return(@ctapi).once + # render the template using the right environment + expect_any_instance_of(Kubetruth::Template).to receive(:render).with(hash_including(environment: "production")).once + + etl.apply() + end + end diff --git a/spec/kubetruth/project_collection_spec.rb b/spec/kubetruth/project_collection_spec.rb index 283cb7c..a7aabf1 100644 --- a/spec/kubetruth/project_collection_spec.rb +++ b/spec/kubetruth/project_collection_spec.rb @@ -5,11 +5,11 @@ module Kubetruth describe ProjectCollection do - let(:collection) { described_class.new(Kubetruth::Config::ProjectSpec.new) } + let(:collection) { described_class.new(Kubetruth::Config.new({})) } before(:each) do @ctapi = double(Kubetruth::CtApi) - allow(Kubetruth::CtApi).to receive(:new).and_return(@ctapi) + allow(Kubetruth::CtApi).to receive(:create).and_return(@ctapi) end describe "#names" do