From 90c7e983420c89e3df370510cc661a8e973ff35a Mon Sep 17 00:00:00 2001 From: Julien Duchesne Date: Tue, 22 Oct 2024 10:25:39 -0400 Subject: [PATCH] Do not run `helm` or `kustomize` when listing envs Helm charts do not generate Tanka environments, by not running `helm` or `kustomize` we can avoid some runtime --- cmd/tk/env.go | 11 +++++++++-- go.work.sum | 7 +++++++ pkg/jsonnet/eval.go | 2 +- pkg/jsonnet/implementations/goimpl/impl.go | 7 ++++--- pkg/jsonnet/implementations/goimpl/vm.go | 4 ++-- pkg/jsonnet/imports.go | 2 +- pkg/jsonnet/imports_test.go | 2 +- pkg/jsonnet/lint.go | 2 +- pkg/jsonnet/native/funcs.go | 23 ++++++++++++++++++---- pkg/jsonnet/native/funcs_test.go | 2 +- pkg/process/data_test.go | 2 +- pkg/tanka/find.go | 13 +++++++++--- pkg/tanka/load.go | 4 +++- pkg/tanka/tanka.go | 2 ++ 14 files changed, 62 insertions(+), 21 deletions(-) diff --git a/cmd/tk/env.go b/cmd/tk/env.go index 7659cb541..063c63d57 100644 --- a/cmd/tk/env.go +++ b/cmd/tk/env.go @@ -236,6 +236,8 @@ func envListCmd() *cli.Command { Args: args, } + dummyHelm := cmd.Flags().Bool("dummy-helm", false, "do not run helm within the environments. This speeds up listing if your helm charts do not generate any Tanka environments.") + dummyKustomize := cmd.Flags().Bool("dummy-kustomize", false, "do not run kustomize within the environments. This speeds up listing if kustomize does not generate any Tanka environments.") useJSON := cmd.Flags().Bool("json", false, "json output") getLabelSelector := labelSelectorFlag(cmd.Flags()) @@ -254,8 +256,13 @@ func envListCmd() *cli.Command { return nil } } - - envs, err := tanka.FindEnvs(path, tanka.FindOpts{Selector: getLabelSelector(), JsonnetOpts: getJsonnetOpts()}) + findOpts := tanka.FindOpts{ + Selector: getLabelSelector(), + JsonnetOpts: getJsonnetOpts(), + DummyHelm: *dummyHelm, + DummyKustomize: *dummyKustomize, + } + envs, err := tanka.FindEnvs(path, findOpts) if err != nil { return err } diff --git a/go.work.sum b/go.work.sum index e2bc36cd6..e93b7d38f 100644 --- a/go.work.sum +++ b/go.work.sum @@ -73,6 +73,7 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -103,14 +104,18 @@ golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA= golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= @@ -122,6 +127,7 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240208230135-b75ee8823808/go.mod h1:KG1lNk5ZFNssSZLrpVb4sMXKMpGwGXOxSG3rnu2gZQQ= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0= @@ -137,6 +143,7 @@ golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58 golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= +golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE= diff --git a/pkg/jsonnet/eval.go b/pkg/jsonnet/eval.go index 16901982f..722e0e3a8 100644 --- a/pkg/jsonnet/eval.go +++ b/pkg/jsonnet/eval.go @@ -115,7 +115,7 @@ func evaluateSnippet(jsonnetImpl types.JsonnetImplementation, evalFunc evalFunc, opts.ImportPaths = jpath evaluator := jsonnetImpl.MakeEvaluator(opts.ImportPaths, opts.ExtCode, opts.TLACode, opts.MaxStack) // We're using the go implementation to deal with imports because we're not evaluating, we're reading the AST - importVM := goimpl.MakeRawVM(opts.ImportPaths, opts.ExtCode, opts.TLACode, opts.MaxStack) + importVM := goimpl.MakeRawVM(opts.ImportPaths, opts.ExtCode, opts.TLACode, opts.MaxStack, false, false) var hash string if cache != nil { diff --git a/pkg/jsonnet/implementations/goimpl/impl.go b/pkg/jsonnet/implementations/goimpl/impl.go index 53e1a5869..5348169de 100644 --- a/pkg/jsonnet/implementations/goimpl/impl.go +++ b/pkg/jsonnet/implementations/goimpl/impl.go @@ -20,13 +20,14 @@ func (vm *JsonnetGoVM) EvaluateFile(filename string) (string, error) { } type JsonnetGoImplementation struct { - Path string + Path string + DummyHelm bool + DummyKustomize bool } func (i *JsonnetGoImplementation) MakeEvaluator(importPaths []string, extCode map[string]string, tlaCode map[string]string, maxStack int) types.JsonnetEvaluator { return &JsonnetGoVM{ - vm: MakeRawVM(importPaths, extCode, tlaCode, maxStack), - + vm: MakeRawVM(importPaths, extCode, tlaCode, maxStack, i.DummyHelm, i.DummyKustomize), path: i.Path, } } diff --git a/pkg/jsonnet/implementations/goimpl/vm.go b/pkg/jsonnet/implementations/goimpl/vm.go index cafb79d7b..758667bac 100644 --- a/pkg/jsonnet/implementations/goimpl/vm.go +++ b/pkg/jsonnet/implementations/goimpl/vm.go @@ -10,7 +10,7 @@ import ( // - extCode and tlaCode applied // - native functions registered // This is exposed because Go is used for advanced use cases, like finding transitive imports or linting. -func MakeRawVM(importPaths []string, extCode map[string]string, tlaCode map[string]string, maxStack int) *jsonnet.VM { +func MakeRawVM(importPaths []string, extCode map[string]string, tlaCode map[string]string, maxStack int, dummyHelm bool, dummyKustomize bool) *jsonnet.VM { vm := jsonnet.MakeVM() vm.Importer(newExtendedImporter(importPaths)) @@ -21,7 +21,7 @@ func MakeRawVM(importPaths []string, extCode map[string]string, tlaCode map[stri vm.TLACode(k, v) } - for _, nf := range native.Funcs() { + for _, nf := range native.Funcs(dummyHelm, dummyKustomize) { vm.NativeFunction(nf) } diff --git a/pkg/jsonnet/imports.go b/pkg/jsonnet/imports.go index 59c27171d..fd9925509 100644 --- a/pkg/jsonnet/imports.go +++ b/pkg/jsonnet/imports.go @@ -48,7 +48,7 @@ func TransitiveImports(dir string) ([]string, error) { return nil, errors.Wrap(err, "resolving JPATH") } - vm := goimpl.MakeRawVM(jpath, nil, nil, 0) + vm := goimpl.MakeRawVM(jpath, nil, nil, 0, false, false) node, err := jsonnet.SnippetToAST(filepath.Base(entrypoint), string(sonnet)) if err != nil { return nil, errors.Wrap(err, "creating Jsonnet AST") diff --git a/pkg/jsonnet/imports_test.go b/pkg/jsonnet/imports_test.go index 24e0ccd26..00c8a76b5 100644 --- a/pkg/jsonnet/imports_test.go +++ b/pkg/jsonnet/imports_test.go @@ -54,7 +54,7 @@ func BenchmarkGetSnippetHash(b *testing.B) { // Create a VM. It's important to reuse the same VM // While there is a caching mechanism that normally shouldn't be shared in a benchmark iteration, // it's useful to evaluate its impact here, because the caching will also improve the evaluation performance afterwards. - vm := goimpl.MakeRawVM([]string{tempDir}, nil, nil, 0) + vm := goimpl.MakeRawVM([]string{tempDir}, nil, nil, 0, false, false) content, err := os.ReadFile(filepath.Join(tempDir, "main.jsonnet")) require.NoError(b, err) diff --git a/pkg/jsonnet/lint.go b/pkg/jsonnet/lint.go index 98989ad90..1fd659960 100644 --- a/pkg/jsonnet/lint.go +++ b/pkg/jsonnet/lint.go @@ -112,7 +112,7 @@ func lintWithRecover(file string) (buf bytes.Buffer, success bool) { fmt.Fprintf(&buf, "got an error getting jpath for %s: %v\n\n", file, err) return } - vm := goimpl.MakeRawVM(jpaths, nil, nil, 0) + vm := goimpl.MakeRawVM(jpaths, nil, nil, 0, false, false) failed := linter.LintSnippet(vm, &buf, []linter.Snippet{{FileName: file, Code: string(content)}}) return buf, !failed diff --git a/pkg/jsonnet/native/funcs.go b/pkg/jsonnet/native/funcs.go index d42fc33d3..385e51c2f 100644 --- a/pkg/jsonnet/native/funcs.go +++ b/pkg/jsonnet/native/funcs.go @@ -19,8 +19,8 @@ import ( // Funcs returns a slice of native Go functions that shall be available // from Jsonnet using `std.nativeFunc` -func Funcs() []*jsonnet.NativeFunction { - return []*jsonnet.NativeFunction{ +func Funcs(dummyHelm, dummyKustomize bool) []*jsonnet.NativeFunction { + funcs := []*jsonnet.NativeFunction{ // Parse serialized data into dicts parseJSON(), parseYAML(), @@ -36,10 +36,25 @@ func Funcs() []*jsonnet.NativeFunction { // Hash functions hashSha256(), + } + + helmTemplate := helm.NativeFunc(helm.ExecHelm{}) + if dummyHelm { + helmTemplate.Func = func([]interface{}) (interface{}, error) { + return map[string]interface{}{}, nil + } + } + funcs = append(funcs, helmTemplate) - helm.NativeFunc(helm.ExecHelm{}), - kustomize.NativeFunc(kustomize.ExecKustomize{}), + kustomizeExec := kustomize.NativeFunc(kustomize.ExecKustomize{}) + if dummyKustomize { + kustomizeExec.Func = func([]interface{}) (interface{}, error) { + return map[string]interface{}{}, nil + } } + funcs = append(funcs, kustomizeExec) + + return funcs } // parseJSON wraps `json.Unmarshal` to convert a json string into a dict diff --git a/pkg/jsonnet/native/funcs_test.go b/pkg/jsonnet/native/funcs_test.go index afa13a8f3..b0385953a 100644 --- a/pkg/jsonnet/native/funcs_test.go +++ b/pkg/jsonnet/native/funcs_test.go @@ -10,7 +10,7 @@ import ( // callNative calls a native function used by jsonnet VM. func callNative(name string, data []interface{}) (res interface{}, err error, callerr error) { - for _, fun := range Funcs() { + for _, fun := range Funcs(false, false) { if fun.Name == name { // Call the function ret, err := fun.Func(data) diff --git a/pkg/process/data_test.go b/pkg/process/data_test.go index 89e8e7024..9a3f7ebf9 100644 --- a/pkg/process/data_test.go +++ b/pkg/process/data_test.go @@ -18,7 +18,7 @@ type testData struct { func loadFixture(name string) testData { filename := filepath.Join("./testdata", name) - vm := goimpl.MakeRawVM([]string{"./testdata"}, nil, nil, 0) + vm := goimpl.MakeRawVM([]string{"./testdata"}, nil, nil, 0, false, false) data, err := vm.EvaluateFile(filename) if err != nil { diff --git a/pkg/tanka/find.go b/pkg/tanka/find.go index 1f7bbf591..693e13676 100644 --- a/pkg/tanka/find.go +++ b/pkg/tanka/find.go @@ -17,8 +17,10 @@ import ( // FindOpts are optional arguments for FindEnvs type FindOpts struct { JsonnetOpts - Selector labels.Selector - Parallelism int + DummyHelm bool + DummyKustomize bool + Selector labels.Selector + Parallelism int } // FindEnvs returns metadata of all environments recursively found in 'path'. @@ -130,10 +132,15 @@ func findEnvsFromJsonnetFiles(jsonnetFiles []string, opts FindOpts) ([]*v1alpha1 // We need to create a copy of the opts for each goroutine because // the content may be modified down the line: jsonnetOpts := opts.JsonnetOpts.Clone() + listOpts := Opts{ + JsonnetOpts: jsonnetOpts, + DummyHelm: opts.DummyHelm, + DummyKustomize: opts.DummyKustomize, + } for jsonnetFile := range jsonnetFilesChan { // try if this has envs - list, err := List(jsonnetFile, Opts{JsonnetOpts: jsonnetOpts}) + list, err := List(jsonnetFile, listOpts) if err != nil && // expected when looking for environments !errors.As(err, &jpath.ErrorNoBase{}) && diff --git a/pkg/tanka/load.go b/pkg/tanka/load.go index 0721c6f7b..7c4888aaf 100644 --- a/pkg/tanka/load.go +++ b/pkg/tanka/load.go @@ -125,7 +125,9 @@ func getJsonnetImplementation(path string, opts Opts) (types.JsonnetImplementati switch opts.JsonnetImplementation { case "go", "": return &goimpl.JsonnetGoImplementation{ - Path: path, + Path: path, + DummyHelm: opts.DummyHelm, + DummyKustomize: opts.DummyKustomize, }, nil default: return nil, fmt.Errorf("unknown jsonnet implementation: %s", opts.JsonnetImplementation) diff --git a/pkg/tanka/tanka.go b/pkg/tanka/tanka.go index eee730ce7..843df71f0 100644 --- a/pkg/tanka/tanka.go +++ b/pkg/tanka/tanka.go @@ -19,6 +19,8 @@ type JsonnetOpts = jsonnet.Opts type Opts struct { JsonnetOpts JsonnetImplementation string + DummyHelm bool + DummyKustomize bool // Filters are used to optionally select a subset of the resources Filters process.Matchers