diff --git a/Makefile b/Makefile index 07b7b1a..6c0065b 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,6 @@ build-local: test CGO_ENABLED=0 go build -ldflags "-X main.version=$(VERSION)" -o $(PROJECT_NAME) clean: - rm -r buildenv + rm -rf buildenv rm -f *.tar.gz rm -rf pkg diff --git a/README.md b/README.md index ce81145..2ad0c44 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ buildenv ======== -A tool for generating environment exports from a YAML file. Variables can be set in plain test, or by specifying vault key-value (version 2) paths and keys (`kv_secrets`) or the older generic / kv paths (`secrets`) where the key name "value" is assumed. +A tool for generating environment exports from a YAML file. Variables can be set in plain test, or by specifying vault key-value (version 2) paths and keys (`kv_secrets`) or the older generic / kv paths (`secrets`) where the key name "value" is assumed. Buildenv will autodetect between version 2 and version 1 `kv_secret` paths _unless it can't read the mount details_. For that case, `kv_secrets` will assume version 2, and `kv1_secrets` will use version 1. Usage ----- @@ -29,6 +29,11 @@ kv_secrets: vars: KV_GENERIC: "value" +kv1_secrets: +- path: "old/test" + vars: + KV1SPECIFIC: "value" + environments: stage: vars: @@ -49,7 +54,7 @@ environments: Output would look like this: -``` +```bash % buildenv -c -e stage -d ndc_one # Global Variables export GLOBAL="global" @@ -57,6 +62,7 @@ export KV2_ONE="1" # Path: secret/test, Key: one export KV2_TWO="2" # Path: secret/test, Key: two export KV1="old" # Path: old/test, Key: value export KV_GENERIC="generic" # Path: gen/test, Key: value +export KV1SPECIFIC="old" # Path: old/test, Key: value export GENERIC_SECRET="generic" # Path: gen/test, Key: value export KV_SECRET="old" # Path: old/test, Key: value export KV2_SECRET="default" # Path: secret/oldstyle, Key: value diff --git a/reader/reader.go b/reader/reader.go index 277816d..bf6aac1 100644 --- a/reader/reader.go +++ b/reader/reader.go @@ -11,8 +11,9 @@ import ( ) type Reader struct { - client *vault.Client - mounts Mounts + client *vault.Client + canDetectMounts bool + mounts Mounts } type EnvVars map[string]string @@ -65,21 +66,21 @@ func (s KVSecretBlock) GetOutput(ctx context.Context, r *Reader) (OutputList, er } } - // The first thing we need to do is get the mount point for the KV engine + // Get the Mount Point for the Secret mountPoint, secretPath := r.MountAndPath(s.Path) if mountPoint == "" { return nil, fmt.Errorf("no mount point found for path %s", s.Path) } - // V2 KV Secrets - if r.mounts[mountPoint].Type == "kv" && r.mounts[mountPoint].Version == "2" { + // Assume v2 if we can detect mounts and it's a KV engine, or if it's explicitly v2 + if !r.canDetectMounts || (r.mounts[mountPoint].Type == "kv" && r.mounts[mountPoint].Version == "2") { // Get Secret resp, err := r.client.Secrets.KvV2Read(ctx, secretPath, vault.WithMountPath(mountPoint)) if err != nil { if vault.IsErrorStatus(err, http.StatusNotFound) { - return nil, fmt.Errorf("secret does not exist: '%s'", s.Path) + return nil, fmt.Errorf("kv2 secret does not exist: '%s'", s.Path) } - return nil, fmt.Errorf("error reading path '%s': %w", s.Path, err) + return nil, fmt.Errorf("error reading kv2 path '%s': %w", s.Path, err) } // For testing purposes, we want to order this envVars := []string{} @@ -103,7 +104,7 @@ func (s KVSecretBlock) GetOutput(ctx context.Context, r *Reader) (OutputList, er // Treat it as a KVv1 secret resp, err := r.client.Secrets.KvV1Read(ctx, secretPath, vault.WithMountPath(mountPoint)) if err != nil { - return nil, fmt.Errorf("error reading path %s: %w", s.Path, err) + return nil, fmt.Errorf("error reading kv1 path %s: %w", s.Path, err) } for varName, varKey := range s.Vars { if _, hasValue := resp.Data[varKey]; !hasValue { @@ -133,23 +134,83 @@ func (s KVSecrets) GetOutput(ctx context.Context, r *Reader) (OutputList, error) return output, nil } +// KV1Secrets is a list of Key-Value Version 1 Secrets +type KV1Secrets []KV1SecretBlock + +type KV1SecretBlock struct { + Path string + Vars KVSecret +} + +func (s KV1SecretBlock) GetOutput(ctx context.Context, r *Reader) (OutputList, error) { + output := OutputList{} + + // Initialize the Vault Client if Necessary + if r.client == nil { + err := r.InitVault() + if err != nil { + return nil, err + } + } + + // The first thing we need to do is get the mount point for the KV engine + mountPoint, secretPath := r.MountAndPath(s.Path) + if mountPoint == "" { + return nil, fmt.Errorf("no mount point found for path %s", s.Path) + } + + // Treat it as a KVv1 secret + resp, err := r.client.Secrets.KvV1Read(ctx, secretPath, vault.WithMountPath(mountPoint)) + if err != nil { + return nil, fmt.Errorf("error reading kv1 path %s: %w", s.Path, err) + } + for varName, varKey := range s.Vars { + if _, hasValue := resp.Data[varKey]; !hasValue { + return nil, fmt.Errorf("key %s not found in path %s", varKey, s.Path) + } + val := fmt.Sprintf("%s", resp.Data[varKey]) + output = append(output, Output{ + Key: varName, + Value: val, + Comment: fmt.Sprintf("Path: %s, Key: %s", s.Path, varKey), + }) + } + + return output, nil +} + +func (s KV1Secrets) GetOutput(ctx context.Context, r *Reader) (OutputList, error) { + output := OutputList{} + for _, block := range s { + blockOutput, err := block.GetOutput(ctx, r) + if err != nil { + return nil, err + } + output = append(output, blockOutput...) + } + return output, nil +} + type DC struct { - Vars EnvVars `yaml:"vars,omitempty"` - Secrets Secrets `yaml:"secrets,omitempty"` - KVSecrets KVSecrets `yaml:"kv_secrets,omitempty"` + Vars EnvVars `yaml:"vars,omitempty"` + Secrets Secrets `yaml:"secrets,omitempty"` + KVSecrets KVSecrets `yaml:"kv_secrets,omitempty"` + KV1Secrets KVSecrets `yaml:"kv1_secrets,omitempty"` } type Environment struct { - Vars EnvVars `yaml:"vars,omitempty"` - Secrets Secrets `yaml:"secrets,omitempty"` - KVSecrets KVSecrets `yaml:"kv_secrets,omitempty"` - Dcs map[string]DC `yaml:"dcs,omitempty"` + Vars EnvVars `yaml:"vars,omitempty"` + Secrets Secrets `yaml:"secrets,omitempty"` + KVSecrets KVSecrets `yaml:"kv_secrets,omitempty"` + KV1Secrets KVSecrets `yaml:"kv1_secrets,omitempty"` + Dcs map[string]DC `yaml:"dcs,omitempty"` } type Variables struct { Vars EnvVars `yaml:"vars,omitempty"` Secrets Secrets `yaml:"secrets,omitempty"` KVSecrets KVSecrets `yaml:"kv_secrets,omitempty"` + KV1Secrets KVSecrets `yaml:"kv1_secrets,omitempty"` Environments map[string]Environment `yaml:"environments,omitempty"` } @@ -192,29 +253,30 @@ func (r *Reader) InitVault() error { return err } r.client = vaultClient + r.canDetectMounts = false // Get mount info resp, err := vaultClient.System.MountsListSecretsEngines(context.Background()) - if err != nil { - return fmt.Errorf("failure reading secret mounts: %w", err) - } - - mounts := Mounts{} - for mount, details := range resp.Data { - detailMap := details.(map[string]interface{}) - thisMount := MountInfo{ - Type: detailMap["type"].(string), - } - if options, hasOptions := detailMap["options"]; hasOptions && options != nil { - optionMap := options.(map[string]interface{}) - if version, hasVersion := optionMap["version"]; hasVersion { - thisMount.Version = version.(string) + if err == nil { + r.canDetectMounts = true + mounts := Mounts{} + for mount, details := range resp.Data { + detailMap := details.(map[string]interface{}) + thisMount := MountInfo{ + Type: detailMap["type"].(string), } + if options, hasOptions := detailMap["options"]; hasOptions && options != nil { + optionMap := options.(map[string]interface{}) + if version, hasVersion := optionMap["version"]; hasVersion { + thisMount.Version = version.(string) + } + } + mounts[mount] = thisMount } - mounts[mount] = thisMount + + r.mounts = mounts } - r.mounts = mounts return nil } @@ -223,10 +285,16 @@ func NewReader() (*Reader, error) { } func (r *Reader) MountAndPath(path string) (string, string) { - for mount := range r.mounts { - if strings.HasPrefix(path, mount) { - return mount, strings.TrimPrefix(path, mount) + if r.canDetectMounts { + for mount := range r.mounts { + if strings.HasPrefix(path, mount) { + return mount, strings.TrimPrefix(path, mount) + } } + } else { + // Take the first part of the path + parts := strings.SplitN(path, "/", 2) + return parts[0], parts[1] } return "", "" } @@ -246,6 +314,11 @@ func (r *Reader) Read(ctx context.Context, input *Variables, env string, dc stri return nil, fmt.Errorf("kv secret error: %w", err) } output = append(output, kvOut...) + kv1Out, err := input.KV1Secrets.GetOutput(ctx, r) + if err != nil { + return nil, fmt.Errorf("kv1 secret error: %w", err) + } + output = append(output, kv1Out...) secretOut, err := input.Secrets.GetOutput(ctx, r) if err != nil { return nil, fmt.Errorf("secret error: %w", err) @@ -258,11 +331,19 @@ func (r *Reader) Read(ctx context.Context, input *Variables, env string, dc stri Comment: fmt.Sprintf("Environment: %s", env), }) output = append(output, input.Environments[env].Vars.GetOutput()...) + // KV (autodetect or v2) kvOut, err := input.Environments[env].KVSecrets.GetOutput(ctx, r) if err != nil { return nil, fmt.Errorf("kv secret error: %w", err) } output = append(output, kvOut...) + // KV1 + kv1Out, err := input.Environments[env].KV1Secrets.GetOutput(ctx, r) + if err != nil { + return nil, fmt.Errorf("kv1 secret error: %w", err) + } + output = append(output, kv1Out...) + // Secrets secretOut, err := input.Environments[env].Secrets.GetOutput(ctx, r) if err != nil { return nil, fmt.Errorf("secret error: %w", err) @@ -276,11 +357,19 @@ func (r *Reader) Read(ctx context.Context, input *Variables, env string, dc stri Comment: fmt.Sprintf("Datacenter: %s", dc), }) output = append(output, input.Environments[env].Dcs[dc].Vars.GetOutput()...) + // KV (autodetect or v2) kvOut, err := input.Environments[env].Dcs[dc].KVSecrets.GetOutput(ctx, r) if err != nil { return nil, fmt.Errorf("kv secret error: %w", err) } output = append(output, kvOut...) + // KV1 + kv1Out, err := input.Environments[env].Dcs[dc].KV1Secrets.GetOutput(ctx, r) + if err != nil { + return nil, fmt.Errorf("kv1 secret error: %w", err) + } + output = append(output, kv1Out...) + // Secrets secretOut, err := input.Environments[env].Dcs[dc].Secrets.GetOutput(ctx, r) if err != nil { return nil, fmt.Errorf("secret error: %w", err) diff --git a/reader/reader_test.go b/reader/reader_test.go index 78121d9..c7424a1 100644 --- a/reader/reader_test.go +++ b/reader/reader_test.go @@ -135,6 +135,142 @@ func TestReader_Read(t *testing.T) { } } +func TestKVSecretBlock_GetOutputNoDetect(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + log.Printf("%+v", r) + var resp []byte + var status = http.StatusOK + + // KV Data + switch r.URL.Path { + case "/v1/kv2/data/test": + resp = []byte(`{"request_id":"bf3b02c0-096e-84d3-dad7-196aa9f112ed","lease_id":"","renewable":false,"lease_duration":0,"data":{"data":{"one":"1","two":"2","three":"3"},"metadata":{"created_time":"2023-12-20T15:32:32.814115685Z","custom_metadata":null,"deletion_time":"","destroyed":false,"version":1}},"wrap_info":null,"warnings":null,"auth":null}`) + case "/v1/kv/test": + resp = []byte(`{"request_id":"63c8c31b-f03f-81ac-cfaa-324239789c3f","lease_id":"","renewable":false,"lease_duration":2764800,"data":{"value":"old"},"wrap_info":null,"warnings":null,"auth":null}`) + default: + status = http.StatusNotFound + resp = []byte(`{"errors":[]}`) + } + + w.WriteHeader(status) + w.Write(resp) + })) + defer server.Close() + + client, _ := vault.New(vault.WithAddress(server.URL)) + reader := &Reader{ + client: client, + canDetectMounts: false, + } + + type fields struct { + Path string + Vars KVSecret + } + type args struct { + r *Reader + } + tests := []struct { + name string + fields fields + args args + want OutputList + wantErr bool + }{ + { + name: "No KV Path", + args: args{ + r: reader, + }, + fields: fields{ + Path: "kv2/path", + Vars: KVSecret{ + "NOT": "here", + }, + }, + wantErr: true, + want: nil, + }, + { + name: "No KV2 Key", + args: args{ + r: reader, + }, + fields: fields{ + Path: "kv2/test", + Vars: KVSecret{ + "THREE": "nope", + }, + }, + wantErr: true, + want: nil, + }, + { + name: "With no autodection, KV Read Fails", + args: args{ + r: reader, + }, + fields: fields{ + Path: "kv/test", + Vars: KVSecret{ + "VALUE": "value", + }, + }, + wantErr: true, + }, + { + name: "Test KV2 Read", + args: args{ + r: reader, + }, + fields: fields{ + Path: "kv2/test", + Vars: KVSecret{ + "ONE": "one", + "TWO": "two", + "THREE": "three", + }, + }, + want: OutputList{ + { + Key: "ONE", + Value: "1", + Comment: "Path: kv2/test, Key: one", + }, + { + Key: "THREE", + Value: "3", + Comment: "Path: kv2/test, Key: three", + }, + { + Key: "TWO", + Value: "2", + Comment: "Path: kv2/test, Key: two", + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + s := KVSecretBlock{ + Path: tt.fields.Path, + Vars: tt.fields.Vars, + } + got, err := s.GetOutput(ctx, tt.args.r) + if (err != nil) != tt.wantErr { + t.Errorf("KVSecretBlock.GetOutput() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("KVSecretBlock.GetOutput() = %v, want %v", got, tt.want) + } + }) + } + +} + func TestKVSecretBlock_GetOutput(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -160,7 +296,8 @@ func TestKVSecretBlock_GetOutput(t *testing.T) { client, _ := vault.New(vault.WithAddress(server.URL)) reader := &Reader{ - client: client, + client: client, + canDetectMounts: true, mounts: Mounts{ "kv2/": { Type: "kv", @@ -301,3 +438,123 @@ func TestKVSecretBlock_GetOutput(t *testing.T) { }) } } + +func TestKV1SecretBlock_GetOutput(t *testing.T) { + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + log.Printf("%+v", r) + var resp []byte + var status = http.StatusOK + + // KV Data + switch r.URL.Path { + case "/v1/kv/test": + resp = []byte(`{"request_id":"63c8c31b-f03f-81ac-cfaa-324239789c3f","lease_id":"","renewable":false,"lease_duration":2764800,"data":{"value":"old"},"wrap_info":null,"warnings":null,"auth":null}`) + default: + status = http.StatusNotFound + resp = []byte(`{"errors":[]}`) + } + + w.WriteHeader(status) + w.Write(resp) + })) + defer server.Close() + + client, _ := vault.New(vault.WithAddress(server.URL)) + reader := &Reader{ + client: client, + canDetectMounts: true, + mounts: Mounts{ + "kv2/": { + Type: "kv", + Version: "2", + }, + "kv/": { + Type: "kv", + }, + "generic/": { + Type: "generic", + }, + }, + } + + type fields struct { + Path string + Vars KVSecret + } + type args struct { + r *Reader + } + tests := []struct { + name string + fields fields + args args + want OutputList + wantErr bool + }{ + { + name: "No Mount", + args: args{ + r: reader, + }, + fields: fields{ + Path: "secret/test", + Vars: KVSecret{ + "should": "fail", + }, + }, + want: nil, + wantErr: true, + }, + { + name: "No KV Path", + args: args{ + r: reader, + }, + fields: fields{ + Path: "kv/path", + Vars: KVSecret{ + "NOT": "here", + }, + }, + wantErr: true, + want: nil, + }, + { + name: "Test KV Read", + args: args{ + r: reader, + }, + fields: fields{ + Path: "kv/test", + Vars: KVSecret{ + "VALUE": "value", + }, + }, + want: OutputList{ + { + Key: "VALUE", + Value: "old", + Comment: "Path: kv/test, Key: value", + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + s := KV1SecretBlock{ + Path: tt.fields.Path, + Vars: tt.fields.Vars, + } + got, err := s.GetOutput(ctx, tt.args.r) + if (err != nil) != tt.wantErr { + t.Errorf("KVSecretBlock.GetOutput() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("KVSecretBlock.GetOutput() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/variables.yml b/variables.yml index 90090dc..df983f1 100644 --- a/variables.yml +++ b/variables.yml @@ -19,6 +19,11 @@ kv_secrets: vars: KV_GENERIC: "value" +kv1_secrets: + - path: "old/test" + vars: + KV1SPECIFIC: "value" + environments: stage: vars: