diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f5214fd..d20396a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,7 @@ name: Test on: push: - branches: [main] + branches: ["*"] pull_request: branches: [main] @@ -30,7 +30,7 @@ jobs: run: | make coverage env: - FEATKEY : /tmp + FEATKEY_DIR : /tmp ASC_DOCKER_USER : ${{secrets.DOCKER_USER}} ASC_DOCKER_PASS : ${{secrets.DOCKER_PASS}} - name: Upload coverage to Codecov diff --git a/.gitmodules b/.gitmodules index fe2863f..783df96 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "schemas"] path = schema/schemas - url = https://github.com/aerospike/schemas + url = https://github.com/aerospike/schemas \ No newline at end of file diff --git a/README.md b/README.md index 0944c46..6329d45 100644 --- a/README.md +++ b/README.md @@ -70,21 +70,21 @@ make unit ### Integration Tests Integration tests require that docker is installed and running. -A path to an Aerospike feature key file should be defined at the `FEATKEY` environment variable. +A path to an Aerospike feature key file should be defined at the `FEATKEY_DIR` environment variable. For more information about the feature key file see the [feature-key docs](https://docs.aerospike.com/server/operations/configure/feature-key). ```shell -FEATKEY=/path/to/aerospike/features.conf make integration +FEATKEY_DIR=/path/to/aerospike/features/dir make integration ``` ### All Tests ```shell -FEATKEY=/path/to/aerospike/features.conf make test +FEATKEY_DIR=/path/to/aerospike/features/dir make test ``` ### Test Coverage ```shell -FEATKEY=/path/to/aerospike/features.conf make view-coverage +FEATKEY_DIR=/path/to/aerospike/features/dir make view-coverage ``` diff --git a/asconf/asconf.go b/asconf/asconf.go deleted file mode 100644 index c88a552..0000000 --- a/asconf/asconf.go +++ /dev/null @@ -1,256 +0,0 @@ -package asconf - -import ( - "bytes" - "errors" - "fmt" - "sort" - "strings" - - "github.com/aerospike/aerospike-management-lib/asconfig" - "github.com/go-logr/logr" - "github.com/sirupsen/logrus" - "gopkg.in/yaml.v3" -) - -type Format string - -const ( - Invalid Format = "" - YAML Format = "yaml" - AsConfig Format = "asconfig" -) - -var ( - ErrInvalidFormat = fmt.Errorf("invalid config format") - ErrConfigValidation = fmt.Errorf("error while validating config") -) - -// TODO maybe use mockery here -type confMarshalValidator interface { - IsValid(log logr.Logger, version string) (bool, []*asconfig.ValidationErr, error) - ToMap() *asconfig.Conf - ToConfFile() asconfig.DotConf - GetFlatMap() *asconfig.Conf -} - -type asconf struct { - cfg confMarshalValidator - logger *logrus.Logger - managementLibLogger logr.Logger - srcFmt Format - // TODO decouple output format from asconf, probably pass it as an arg to marshal text - outFmt Format - src []byte - aerospikeVersion string -} - -func NewAsconf(source []byte, srcFmt, outFmt Format, aerospikeVersion string, logger *logrus.Logger, managementLibLogger logr.Logger) (*asconf, error) { - - ac := &asconf{ - logger: logger, - managementLibLogger: managementLibLogger, - srcFmt: srcFmt, - outFmt: outFmt, - src: source, - aerospikeVersion: aerospikeVersion, - } - - // sets ac.cfg - err := ac.load() - - return ac, err -} - -type ValidationErr struct { - asconfig.ValidationErr -} - -type VErrSlice []ValidationErr - -func (a VErrSlice) Len() int { return len(a) } -func (a VErrSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a VErrSlice) Less(i, j int) bool { return strings.Compare(a[i].Error(), a[j].Error()) == -1 } - -func (o ValidationErr) Error() string { - verrTemplate := "description: %s, error-type: %s" - return fmt.Sprintf(verrTemplate, o.Description, o.ErrType) -} - -type ValidationErrors struct { - Errors VErrSlice -} - -func (o ValidationErrors) Error() string { - errorsByContext := map[string]VErrSlice{} - - sort.Sort(o.Errors) - - for _, err := range o.Errors { - errorsByContext[err.Context] = append(errorsByContext[err.Context], err) - } - - contexts := []string{} - for ctx := range errorsByContext { - contexts = append(contexts, ctx) - } - - sort.Strings(contexts) - - errString := "" - - for _, ctx := range contexts { - errString += fmt.Sprintf("context: %s\n", ctx) - - errList := errorsByContext[ctx] - for _, err := range errList { - - // filter "Must validate one and only one schema " errors - // I have never seen a useful one and they seem to always be - // accompanied by another more useful error that will be displayed - if err.ErrType == "number_one_of" { - continue - } - - errString += fmt.Sprintf("\t- %s\n", err.Error()) - } - } - - return errString -} - -// Validate validates the parsed configuration in ac against -// the Aerospike schema matching ac.aerospikeVersion. -// ValidationErrors is not nil if any errors during validation occur. -// ValidationErrors Error() method outputs a human readable string of validation error details. -// error is not nil if validation, or any other type of error occurs. -func (ac *asconf) Validate() (*ValidationErrors, error) { - - valid, tempVerrs, err := ac.cfg.IsValid(ac.managementLibLogger, ac.aerospikeVersion) - - verrs := ValidationErrors{} - for _, v := range tempVerrs { - verr := ValidationErr{ - ValidationErr: *v, - } - verrs.Errors = append(verrs.Errors, verr) - } - - if !valid || err != nil || len(verrs.Errors) > 0 { - return &verrs, errors.Join(ErrConfigValidation, err) - } - - return nil, nil -} - -func (ac *asconf) MarshalText() (text []byte, err error) { - - switch ac.outFmt { - case AsConfig: - text = []byte(ac.cfg.ToConfFile()) - case YAML: - m := ac.cfg.ToMap() - text, err = yaml.Marshal(m) - default: - err = fmt.Errorf("%w %s", ErrInvalidFormat, ac.outFmt) - } - - return -} - -func (ac *asconf) GetIntermediateConfig() map[string]any { - return *ac.cfg.GetFlatMap() -} - -func (ac *asconf) load() (err error) { - - switch ac.srcFmt { - case YAML: - err = ac.loadYAML() - case AsConfig: - err = ac.loadAsConf() - default: - return fmt.Errorf("%w %s", ErrInvalidFormat, ac.srcFmt) - } - - if err != nil { - return err - } - - // recreate the management lib config - // with a sorted config map so that output - // is always in the same order - cmap := *ac.cfg.ToMap() - - mutateMap(cmap, []mapping{ - sortLists, - }) - - ac.cfg, err = asconfig.NewMapAsConfig( - ac.managementLibLogger, - ac.aerospikeVersion, - cmap, - ) - - return -} - -func (ac *asconf) loadYAML() error { - - var data map[string]any - - err := yaml.Unmarshal(ac.src, &data) - if err != nil { - return err - } - - c, err := asconfig.NewMapAsConfig( - ac.managementLibLogger, - ac.aerospikeVersion, - data, - ) - - if err != nil { - return fmt.Errorf("failed to initialize asconfig from yaml: %w", err) - } - - ac.cfg = c - - return nil -} - -func (ac *asconf) loadAsConf() error { - - reader := bytes.NewReader(ac.src) - - c, err := asconfig.FromConfFile(ac.managementLibLogger, ac.aerospikeVersion, reader) - if err != nil { - return fmt.Errorf("failed to parse asconfig file: %w", err) - } - - // the aerospike management lib parses asconfig files into - // a format that its validation rejects - // this is because the schema files are meant to - // validate the aerospike kubernetes operator's asconfig yaml format - // so we modify the map here to match that format - cmap := *c.ToMap() - - mutateMap(cmap, []mapping{ - typedContextsToObject, - toPlural, - }) - - c, err = asconfig.NewMapAsConfig( - ac.managementLibLogger, - ac.aerospikeVersion, - cmap, - ) - - if err != nil { - return err - } - - ac.cfg = c - - return nil -} diff --git a/asconf/asconf_test.go b/asconf/asconf_test.go deleted file mode 100644 index f5551df..0000000 --- a/asconf/asconf_test.go +++ /dev/null @@ -1,279 +0,0 @@ -//go:build unit -// +build unit - -package asconf - -import ( - "fmt" - "reflect" - "testing" - - "github.com/aerospike/aerospike-management-lib/asconfig" - "github.com/go-logr/logr" - "github.com/sirupsen/logrus" -) - -type mockCFG struct { - valid bool - err error - validationErrors []*asconfig.ValidationErr - confMap *asconfig.Conf - confText string - flatConf *asconfig.Conf -} - -func (o *mockCFG) IsValid(log logr.Logger, version string) (bool, []*asconfig.ValidationErr, error) { - return o.valid, o.validationErrors, o.err -} - -func (o *mockCFG) ToMap() *asconfig.Conf { - return o.confMap -} - -func (o *mockCFG) ToConfFile() asconfig.DotConf { - return o.confText -} - -func (o *mockCFG) GetFlatMap() *asconfig.Conf { - return o.flatConf -} - -var _testYaml = ` -namespaces: - - index-type: - mounts: - - /test/dev/xvdf-index - mounts-size-limit: 4294967296 - type: flash - memory-size: 3000000000 - name: test - replication-factor: 2 - storage-engine: - devices: - - /test/dev/xvdf - type: device -` - -// { -// name: "yaml-to-conf", -// fields: fields{ -// cfg: &mockCFG{ -// valid: true, -// err: nil, -// validationErrors: []*asconfig.ValidationErr{}, -// confMap: &asconfig.Conf{ -// "namespaces": []string{"ns1", "ns1"}, -// }, -// confText: "namespace ns1 {}\n namespace ns2 {}", -// flatConf: &asconfig.Conf{ -// "namespaces.ns1": "device", -// "namespaces.ns2": "memory", -// }, -// }, -// logger: logrus.New(), -// managementLibLogger: logrusr.New(logrus.New()), -// srcFmt: YAML, -// outFmt: AsConfig, -// src: []byte(_testYaml), -// aerospikeVersion: "6.2.0.2", -// }, -// wantErr: false, -// }, - -func Test_asconf_Validate(t *testing.T) { - type fields struct { - cfg confMarshalValidator - logger *logrus.Logger - managementLibLogger logr.Logger - srcFmt Format - outFmt Format - src []byte - aerospikeVersion string - } - tests := []struct { - name string - fields fields - wantErr bool - }{ - { - name: "pos1", - fields: fields{ - cfg: &mockCFG{ - valid: true, - err: nil, - validationErrors: []*asconfig.ValidationErr{}, - }, - logger: logrus.New(), - }, - wantErr: false, - }, - { - name: "neg1", - fields: fields{ - cfg: &mockCFG{ - valid: false, - err: nil, - validationErrors: []*asconfig.ValidationErr{}, - }, - logger: logrus.New(), - }, - wantErr: true, - }, - { - name: "neg2", - fields: fields{ - cfg: &mockCFG{ - valid: true, - err: fmt.Errorf("test_err"), - validationErrors: []*asconfig.ValidationErr{}, - }, - logger: logrus.New(), - }, - wantErr: true, - }, - { - name: "neg3", - fields: fields{ - cfg: &mockCFG{ - valid: true, - err: nil, - validationErrors: []*asconfig.ValidationErr{{}}, - }, - logger: logrus.New(), - }, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ac := &asconf{ - cfg: tt.fields.cfg, - logger: tt.fields.logger, - managementLibLogger: tt.fields.managementLibLogger, - srcFmt: tt.fields.srcFmt, - outFmt: tt.fields.outFmt, - src: tt.fields.src, - aerospikeVersion: tt.fields.aerospikeVersion, - } - if _, err := ac.Validate(); (err != nil) != tt.wantErr { - t.Errorf("asconf.Validate() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func Test_asconf_MarshalText(t *testing.T) { - type fields struct { - cfg confMarshalValidator - logger *logrus.Logger - managementLibLogger logr.Logger - srcFmt Format - outFmt Format - src []byte - aerospikeVersion string - } - tests := []struct { - name string - fields fields - wantText []byte - wantErr bool - }{ - { - name: "valid asconfig format", - fields: fields{ - cfg: &mockCFG{ - confText: "namespace ns1 {}\n namespace ns2 {}", - }, - outFmt: AsConfig, - }, - wantErr: false, - wantText: []byte("namespace ns1 {}\n namespace ns2 {}"), - }, - { - name: "valid yaml format", - fields: fields{ - cfg: &mockCFG{ - confMap: &asconfig.Conf{ - "namespaces": "ns1", - }, - confText: "", - }, - outFmt: YAML, - }, - wantErr: false, - wantText: []byte("namespaces: ns1\n"), - }, - { - name: "invalid format", - fields: fields{ - outFmt: Invalid, - }, - wantErr: true, - wantText: nil, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ac := &asconf{ - cfg: tt.fields.cfg, - logger: tt.fields.logger, - managementLibLogger: tt.fields.managementLibLogger, - srcFmt: tt.fields.srcFmt, - outFmt: tt.fields.outFmt, - src: tt.fields.src, - aerospikeVersion: tt.fields.aerospikeVersion, - } - gotText, err := ac.MarshalText() - if (err != nil) != tt.wantErr { - t.Errorf("asconf.MarshalText() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(gotText, tt.wantText) { - t.Errorf("asconf.MarshalText() = %v, want %v", gotText, tt.wantText) - } - }) - } -} - -func Test_asconf_GetIntermediateConfig(t *testing.T) { - type fields struct { - cfg confMarshalValidator - logger *logrus.Logger - managementLibLogger logr.Logger - srcFmt Format - outFmt Format - src []byte - aerospikeVersion string - } - tests := []struct { - name string - fields fields - want map[string]any - }{ - { - name: "return flat conf", - fields: fields{ - cfg: &mockCFG{ - flatConf: &configMap{"ns": 1}, - }, - }, - want: map[string]any{"ns": 1}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ac := &asconf{ - cfg: tt.fields.cfg, - logger: tt.fields.logger, - managementLibLogger: tt.fields.managementLibLogger, - srcFmt: tt.fields.srcFmt, - outFmt: tt.fields.outFmt, - src: tt.fields.src, - aerospikeVersion: tt.fields.aerospikeVersion, - } - if got := ac.GetIntermediateConfig(); !reflect.DeepEqual(got, tt.want) { - t.Errorf("asconf.GetIntermediateConfig() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/cmd/convert.go b/cmd/convert.go index 2c4b375..353dbd2 100644 --- a/cmd/convert.go +++ b/cmd/convert.go @@ -8,8 +8,8 @@ import ( "strings" "github.com/aerospike/aerospike-management-lib/asconfig" - "github.com/aerospike/asconfig/asconf" - "github.com/aerospike/asconfig/asconf/metadata" + "github.com/aerospike/asconfig/conf" + "github.com/aerospike/asconfig/conf/metadata" "github.com/spf13/cobra" ) @@ -24,8 +24,8 @@ var ( errMissingAerospikeVersion = fmt.Errorf("missing required flag '--aerospike-version'") errInvalidAerospikeVersion = fmt.Errorf("aerospike version must be in the form ..") errUnsupportedAerospikeVersion = fmt.Errorf("aerospike version unsupported") - errInvalidOutput = fmt.Errorf("invalid output flag") errInvalidFormat = fmt.Errorf("invalid format flag") + errMissingFormat = fmt.Errorf("missing format flag") ) func init() { @@ -71,12 +71,12 @@ func newConvertCmd() *cobra.Command { srcPath = args[0] } - version, err := cmd.Flags().GetString("aerospike-version") + asVersion, err := cmd.Flags().GetString("aerospike-version") if err != nil { return err } - logger.Debugf("Processing flag aerospike-version value=%s", version) + logger.Debugf("Processing flag aerospike-version value=%s", asVersion) force, err := cmd.Flags().GetBool("force") if err != nil { @@ -92,56 +92,55 @@ func newConvertCmd() *cobra.Command { logger.Debugf("Processing flag format value=%v", srcFormat) - var outFmt asconf.Format + var outFmt conf.Format switch srcFormat { - case asconf.AsConfig: - outFmt = asconf.YAML - case asconf.YAML: - outFmt = asconf.AsConfig + case conf.AsConfig: + outFmt = conf.YAML + case conf.YAML: + outFmt = conf.AsConfig default: return fmt.Errorf("%w: %s", errInvalidFormat, srcFormat) } // if the version option is empty, // try populating from the metadata - if version == "" { - version, err = getMetaDataItem(cfgData, metaKeyAerospikeVersion) + if asVersion == "" { + asVersion, err = getMetaDataItem(cfgData, metaKeyAerospikeVersion) if err != nil && !force { return errors.Join(errMissingAerospikeVersion, err) } } - conf, err := asconf.NewAsconf( - cfgData, - srcFormat, - outFmt, - version, - logger, - managementLibLogger, - ) + // load + asconfig, err := conf.NewASConfigFromBytes(mgmtLibLogger, cfgData, srcFormat) if err != nil { return err } + // validate if !force { - verrs, err := conf.Validate() + verrs, err := conf.NewConfigValidator(asconfig, mgmtLibLogger, asVersion).Validate() if err != nil || verrs != nil { return errors.Join(err, verrs) } } - out, err := conf.MarshalText() + // convert + out, err := conf.NewConfigMarshaller(asconfig, outFmt).MarshalText() if err != nil { return err } // prepend metadata to the config output - mtext, err := genMetaDataText(metaDataArgs{ - src: cfgData, - aerospikeVersion: version, - asconfigVersion: VERSION, - }) + mtext, err := genMetaDataText( + cfgData, + nil, + map[string]string{ + metaKeyAerospikeVersion: asVersion, + metaKeyAsconfigVersion: VERSION, + }, + ) if err != nil { return err } @@ -162,9 +161,9 @@ func newConvertCmd() *cobra.Command { outFileName = strings.TrimSuffix(outFileName, filepath.Ext(outFileName)) outputPath = filepath.Join(outputPath, outFileName) - if outFmt == asconf.YAML { + if outFmt == conf.YAML { outputPath += ".yaml" - } else if outFmt == asconf.AsConfig { + } else if outFmt == conf.AsConfig { outputPath += ".conf" } else { return fmt.Errorf("output format unrecognized %w", errInvalidFormat) @@ -259,16 +258,27 @@ func newConvertCmd() *cobra.Command { } } + formatString, err := cmd.Flags().GetString("format") + if err != nil { + return errors.Join(errMissingFormat, err) + } + + _, err = ParseFmtString(formatString) + if err != nil && formatString != "" { + return errors.Join(errInvalidFormat, err) + } + return nil }, } // flags and configuration settings // aerospike-version is marked required in this cmd's PreRun if the --force flag is not provided - commonFlags := getCommonFlags() - res.Flags().AddFlagSet(commonFlags) + asCommonFlags := getCommonFlags() + res.Flags().AddFlagSet(asCommonFlags) res.Flags().BoolP("force", "f", false, "Override checks for supported server version and config validation") res.Flags().StringP("output", "o", os.Stdout.Name(), "File path to write output to") + res.Flags().StringP("format", "F", "conf", "The format of the source file(s). Valid options are: yaml, yml, and conf.") res.Version = VERSION diff --git a/cmd/convert_test.go b/cmd/convert_test.go index 94e7eaf..7765c86 100644 --- a/cmd/convert_test.go +++ b/cmd/convert_test.go @@ -1,11 +1,12 @@ //go:build unit -// +build unit package cmd import ( "errors" "testing" + + "github.com/aerospike/asconfig/conf" ) type preTestConvert struct { @@ -46,6 +47,20 @@ var preTestsConvert = []preTestConvert{ arguments: []string{"./convert_test.go"}, expectedErrors: []error{nil}, }, + { + flags: []string{"--format", "bad_fmt"}, + arguments: []string{"./convert_test.go"}, + expectedErrors: []error{ + conf.ErrInvalidFormat, + }, + }, + { + flags: []string{"-F", "bad_fmt"}, + arguments: []string{"./convert_test.go"}, + expectedErrors: []error{ + conf.ErrInvalidFormat, + }, + }, } func TestPreRunConvert(t *testing.T) { diff --git a/cmd/diff.go b/cmd/diff.go index 7536aea..e52eb80 100644 --- a/cmd/diff.go +++ b/cmd/diff.go @@ -8,7 +8,7 @@ import ( "sort" "strings" - "github.com/aerospike/asconfig/asconf" + "github.com/aerospike/asconfig/conf" "github.com/spf13/cobra" ) @@ -19,7 +19,7 @@ const ( ) var ( - errDiffConfigsDiffer = errors.Join(fmt.Errorf("configuration files are not equal"), SilentError) + errDiffConfigsDiffer = errors.Join(fmt.Errorf("configuration files are not equal"), ErrSilent) errDiffTooFewArgs = fmt.Errorf("diff requires atleast %d file paths as arguments", diffArgMin) errDiffTooManyArgs = fmt.Errorf("diff requires no more than %d file paths as arguments", diffArgMax) ) @@ -85,38 +85,25 @@ func newDiffCmd() *cobra.Command { // not performing any validation so server version is "" (not needed) // won't be marshaling these configs to text so use Invalid output format - // TODO decouple output format from asconf, probably pass it as an arg to marshal text - conf1, err := asconf.NewAsconf( - f1, - fmt1, - asconf.Invalid, - "", - logger, - managementLibLogger, - ) + // TODO decouple output format from asconf, probably pass it as an + // arg to marshal text + conf1, err := conf.NewASConfigFromBytes(mgmtLibLogger, f1, fmt1) if err != nil { return err } - conf2, err := asconf.NewAsconf( - f2, - fmt2, - asconf.Invalid, - "", - logger, - managementLibLogger, - ) + conf2, err := conf.NewASConfigFromBytes(mgmtLibLogger, f2, fmt2) if err != nil { return err } // get flattened config maps - map1 := conf1.GetIntermediateConfig() - map2 := conf2.GetIntermediateConfig() + map1 := conf1.GetFlatMap() + map2 := conf2.GetFlatMap() diffs := diffFlatMaps( - map1, - map2, + *map1, + *map2, ) if len(diffs) > 0 { @@ -129,6 +116,8 @@ func newDiffCmd() *cobra.Command { }, } + res.Flags().StringP("format", "F", "conf", "The format of the source file(s). Valid options are: yaml, yml, and conf.") + return res } @@ -157,7 +146,7 @@ func diffFlatMaps(m1 map[string]any, m2 map[string]any) []string { // "index" is a metadata key added by // the management lib to these flat maps // ignore it - if strings.HasSuffix(k, ".index") { + if strings.HasSuffix(k, ".") { continue } diff --git a/cmd/diff_test.go b/cmd/diff_test.go index 596c516..355ca5b 100644 --- a/cmd/diff_test.go +++ b/cmd/diff_test.go @@ -1,5 +1,4 @@ //go:build unit -// +build unit package cmd @@ -59,6 +58,16 @@ var testDiffArgs = []runTestDiff{ arguments: []string{"../testdata/expected/all_flash_cluster_cr.conf", "../testdata/expected/all_flash_cluster_cr.conf"}, expectError: false, }, + { + flags: []string{"--format", "bad_fmt"}, + arguments: []string{}, + expectError: true, + }, + { + flags: []string{"-F", "bad_fmt"}, + arguments: []string{}, + expectError: true, + }, } func TestRunEDiff(t *testing.T) { diff --git a/cmd/generate.go b/cmd/generate.go new file mode 100644 index 0000000..7d03d80 --- /dev/null +++ b/cmd/generate.go @@ -0,0 +1,153 @@ +package cmd + +import ( + "errors" + "fmt" + "os" + + "github.com/aerospike/aerospike-management-lib/asconfig" + "github.com/aerospike/aerospike-management-lib/info" + "github.com/aerospike/asconfig/conf" + "github.com/aerospike/tools-common-go/client" + "github.com/aerospike/tools-common-go/flags" + "github.com/spf13/cobra" +) + +var generateArgMax = 1 + +func init() { + rootCmd.AddCommand(generateCmd) +} + +var generateCmd = newGenerateCmd() + +func newGenerateCmd() *cobra.Command { + asCommonFlags := flags.NewDefaultAerospikeFlags() + disclaimer := []byte(`# +# This configuration file is generated by asconfig, this feature is currently in beta. +# We appreciate your feedback on any issues encountered. These can be reported +# to our support team or via GitHub. Please ensure to verify the configuration +# file before use. Current limitations include the inability to generate the +# following contexts and parameters: logging.syslog, mod-lua, service.user, +# service.group. Please note that this configuration file may not be compatible +# with all versions of Aerospike or the Community Edition.`) + res := &cobra.Command{ + Use: "generate [flags]", + Short: "BETA: Generate a configuration file from a running Aerospike node.", + Long: `BETA: Generate a configuration file from a running Aerospike node. This can be useful if you have changed the configuration of a node dynamically (e.g. xdr) and would like to persist the changes.`, + RunE: func(cmd *cobra.Command, args []string) error { + logger.Debug("Running generate command") + + outputPath, err := cmd.Flags().GetString("output") + if err != nil { + return err + } + + outFormat, err := getConfFileFormat(outputPath, cmd) + if err != nil { + return err + } + + logger.Debugf("Generating config from Aerospike node") + + asCommonConfig := client.NewDefaultAerospikeHostConfig() + + flags.SetAerospikeConf(asCommonConfig, asCommonFlags) + + asPolicy, err := asCommonConfig.NewClientPolicy() + if err != nil { + return errors.Join(fmt.Errorf("unable to create client policy"), err) + } + + logger.Infof("Retrieving Aerospike configuration from node %s", &asCommonFlags.Seeds) + + asHosts := asCommonConfig.NewHosts() + asinfo := info.NewAsInfo(mgmtLibLogger, asHosts[0], asPolicy) + + generatedConf, err := asconfig.GenerateConf(mgmtLibLogger, asinfo, true) + if err != nil { + return errors.Join(fmt.Errorf("unable to generate config file"), err) + } + + asconfig, err := asconfig.NewMapAsConfig(mgmtLibLogger, generatedConf.Version, generatedConf.Conf) + if err != nil { + return errors.Join(fmt.Errorf("unable to parse the generated conf file"), err) + } + + marshaller := conf.NewConfigMarshaller(asconfig, outFormat) + + fdata, err := marshaller.MarshalText() + if err != nil { + return errors.Join(fmt.Errorf("unable to marshal the generated conf file"), err) + } + + mdata := map[string]string{ + metaKeyAerospikeVersion: generatedConf.Version, + metaKeyAsconfigVersion: VERSION, + } + // prepend metadata to the config output + mtext, err := genMetaDataText(fdata, disclaimer, mdata) + if err != nil { + return err + } + + fdata = append(mtext, fdata...) + + var outFile *os.File + if outputPath == os.Stdout.Name() { + outFile = os.Stdout + } else { + outFile, err = os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return err + } + + defer outFile.Close() + } + + logger.Debugf("Writing converted data to: %s", outputPath) + _, err = outFile.Write(fdata) + + logger.Warning( + "Community Edition is not supported. Generated static configuration does not save logging.syslog, mod-lua, service.user and service.group", + ) + logger.Warning( + "This feature is currently in beta. Use at your own risk and please report any issue to support.", + ) + + return err + }, + PreRunE: func(cmd *cobra.Command, args []string) error { + if len(args) > generateArgMax { + return errTooManyArguments + } + + // validate flags + _, err := cmd.Flags().GetString("output") + if err != nil { + return err + } + + formatString, err := cmd.Flags().GetString("format") + if err != nil { + return errors.Join(errMissingFormat, err) + } + + _, err = ParseFmtString(formatString) + if err != nil && formatString != "" { + return errors.Join(errInvalidFormat, err) + } + + return nil + }, + } + + res.Version = VERSION + res.Flags().AddFlagSet( + flags.SetAerospikeFlags(asCommonFlags, flags.DefaultWrapHelpString), + ) + res.Flags().StringP("output", "o", os.Stdout.Name(), flags.DefaultWrapHelpString("File path to write output to")) + res.Flags().StringP("format", "F", "conf", flags.DefaultWrapHelpString("The format of the destination file(s). Valid options are: yaml, yml, and conf.")) + + return res +} diff --git a/cmd/generate_test.go b/cmd/generate_test.go new file mode 100644 index 0000000..3531466 --- /dev/null +++ b/cmd/generate_test.go @@ -0,0 +1,43 @@ +//go:build unit + +package cmd + +import ( + "errors" + "testing" +) + +func TestRunEGenerate(t *testing.T) { + type runTestGen struct { + flags []string + arguments []string + expectError error + } + + var testGenArgs = []runTestGen{ + { + flags: []string{}, + arguments: []string{"too", "many", "args"}, + expectError: errTooManyArguments, + }, + { + flags: []string{"--format", "bad_fmt"}, + arguments: []string{}, + expectError: errInvalidFormat, + }, + { + flags: []string{"-F", "bad_fmt"}, + arguments: []string{}, + expectError: errInvalidFormat, + }, + } + cmd := newGenerateCmd() + + for i, test := range testGenArgs { + cmd.ParseFlags(test.flags) + err := cmd.PreRunE(cmd, test.arguments) + if !errors.Is(err, test.expectError) { + t.Fatalf("case: %d, expectError: %v does not match err: %v", i, test.expectError, err) + } + } +} diff --git a/cmd/root.go b/cmd/root.go index 777022d..64d5d01 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -10,8 +10,6 @@ import ( "github.com/aerospike/asconfig/schema" - "github.com/aerospike/asconfig/asconf" - "github.com/aerospike/aerospike-management-lib/asconfig" "github.com/bombsimon/logrusr/v4" "github.com/go-logr/logr" @@ -47,22 +45,11 @@ func newRootCmd() *cobra.Command { lvlCode, err := logrus.ParseLevel(lvl) if err != nil { - logger.Errorf("Invalid log-level %s", lvl) - multiErr = fmt.Errorf("%w, %w %w", multiErr, errInvalidLogLevel, err) + multiErr = errors.Join(multiErr, errInvalidLogLevel, err) } logger.SetLevel(lvlCode) - formatString, err := cmd.Flags().GetString("format") - if err != nil { - multiErr = fmt.Errorf("%w, %w", multiErr, err) - } - - _, err = asconf.ParseFmtString(formatString) - if err != nil && formatString != "" { - multiErr = fmt.Errorf("%w, %w", multiErr, err) - } - return multiErr }, } @@ -73,7 +60,7 @@ func newRootCmd() *cobra.Command { logLevelUsage := fmt.Sprintf("Set the logging detail level. Valid levels are: %v", log.GetLogLevels()) res.PersistentFlags().StringP("log-level", "l", "info", logLevelUsage) res.PersistentFlags().BoolP("version", "V", false, "Version for asconfig.") - res.PersistentFlags().StringP("format", "F", "conf", "The format of the source file(s). Valid options are: yaml, yml, and conf.") + res.PersistentFlags().BoolP("help", "u", false, "Display help information") res.SilenceErrors = true res.SilenceUsage = true @@ -81,7 +68,7 @@ func newRootCmd() *cobra.Command { res.SetFlagErrorFunc(func(cmd *cobra.Command, err error) error { logger.Error(err) cmd.Println(cmd.UsageString()) - return errors.Join(err, SilentError) + return errors.Join(err, ErrSilent) }) return res @@ -92,7 +79,7 @@ func newRootCmd() *cobra.Command { func Execute() { err := rootCmd.Execute() if err != nil { - if !errors.Is(err, SilentError) { + if !errors.Is(err, ErrSilent) { // handle wrapped errors errs := strings.Split(err.Error(), "\n") @@ -105,7 +92,7 @@ func Execute() { } var logger *logrus.Logger -var managementLibLogger logr.Logger +var mgmtLibLogger logr.Logger func init() { logger = logrus.New() @@ -120,6 +107,6 @@ func init() { panic(err) } - managementLibLogger = logrusr.New(logger) - asconfig.InitFromMap(managementLibLogger, schemaMap) + mgmtLibLogger = logrusr.New(logger) + asconfig.InitFromMap(mgmtLibLogger, schemaMap) } diff --git a/cmd/root_test.go b/cmd/root_test.go index 1257b0c..2e2cfd8 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -1,11 +1,7 @@ -//go:build unit -// +build unit - package cmd import ( "errors" - "github.com/aerospike/asconfig/asconf" "testing" ) @@ -33,20 +29,6 @@ var preTestsRoot = []preTestRoot{ errInvalidLogLevel, }, }, - { - flags: []string{"--format", "bad_fmt"}, - arguments: []string{}, - expectedErrors: []error{ - asconf.ErrInvalidFormat, - }, - }, - { - flags: []string{"-F", "bad_fmt"}, - arguments: []string{}, - expectedErrors: []error{ - asconf.ErrInvalidFormat, - }, - }, } func TestPersistentPreRunRoot(t *testing.T) { @@ -57,7 +39,7 @@ func TestPersistentPreRunRoot(t *testing.T) { err := cmd.PersistentPreRunE(cmd, test.arguments) for _, expectedErr := range test.expectedErrors { if !errors.Is(err, expectedErr) { - t.Errorf("actual err: %v\n is not expected err: %v", err, expectedErr) + t.Errorf("%v\n actual err: %v\n is not expected err: %v", test.flags, err, expectedErr) } } } diff --git a/cmd/utils.go b/cmd/utils.go index 22286e7..fb0b2e8 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -6,8 +6,8 @@ import ( "path/filepath" "strings" - "github.com/aerospike/asconfig/asconf" - "github.com/aerospike/asconfig/asconf/metadata" + "github.com/aerospike/asconfig/conf" + "github.com/aerospike/asconfig/conf/metadata" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -25,31 +25,28 @@ type metaDataArgs struct { asconfigVersion string } -func genMetaDataText(args metaDataArgs) ([]byte, error) { +func genMetaDataText(src []byte, msg []byte, mdata map[string]string) ([]byte, error) { metaHeader := "# *** Aerospike Metadata Generated by Asconfig ***" - src := args.src - aev := args.aerospikeVersion - asv := args.asconfigVersion - - mdata := map[string]string{} err := metadata.Unmarshal(src, mdata) if err != nil { return nil, err } - mdata[metaKeyAerospikeVersion] = aev - mdata[metaKeyAsconfigVersion] = asv - mtext, err := metadata.Marshal(mdata) if err != nil { return nil, err } metaFooter := "# *** End Aerospike Metadata ***" + strMsg := string(msg) - mtext = []byte(fmt.Sprintf("%s\n%s%s\n", metaHeader, mtext, metaFooter)) + if len(msg) > 0 { + strMsg = strMsg + "\n#\n" + } + + mtext = []byte(fmt.Sprintf("%s\n%s%s%s\n\n", metaHeader, strMsg, mtext, metaFooter)) return mtext, nil } @@ -92,27 +89,44 @@ func getCommonFlags() *pflag.FlagSet { // this function implements the defaults scheme for file formats in asconfig // if the --format flag is defined use that, else if the path has an extension // use that, else use the default value from --format -func getConfFileFormat(path string, cmd *cobra.Command) (asconf.Format, error) { +func getConfFileFormat(path string, cmd *cobra.Command) (conf.Format, error) { ext := filepath.Ext(path) ext = strings.TrimPrefix(ext, ".") fmtStr, err := cmd.Flags().GetString("format") if err != nil { - return asconf.Invalid, err + return conf.Invalid, err } + logger.Debugf("Processing flag format value=%v", fmtStr) + // if the user did not supply format, and // the input file has an extension, overwrite it with ext if !cmd.Flags().Changed("format") && ext != "" { fmtStr = ext } - fmt, err := asconf.ParseFmtString(fmtStr) + fmt, err := ParseFmtString(fmtStr) if err != nil { - return asconf.Invalid, err + return conf.Invalid, err } return fmt, nil } -var SilentError = errors.New("SILENT") +var ErrSilent = errors.New("SILENT") + +func ParseFmtString(in string) (f conf.Format, err error) { + + switch strings.ToLower(in) { + case "yaml", "yml": + f = conf.YAML + case "asconfig", "conf", "asconf": + f = conf.AsConfig + default: + f = conf.Invalid + err = fmt.Errorf("%w: %s", conf.ErrInvalidFormat, in) + } + + return +} diff --git a/cmd/utils_test.go b/cmd/utils_test.go index 571672c..8a29ce5 100644 --- a/cmd/utils_test.go +++ b/cmd/utils_test.go @@ -1,13 +1,13 @@ //go:build unit -// +build unit package cmd import ( - "github.com/aerospike/asconfig/asconf" "reflect" "testing" + "github.com/aerospike/asconfig/conf" + "github.com/spf13/cobra" ) @@ -28,7 +28,7 @@ func Test_getConfFileFormat(t *testing.T) { tests := []struct { name string args args - want asconf.Format + want conf.Format wantErr bool }{ { @@ -37,7 +37,7 @@ func Test_getConfFileFormat(t *testing.T) { path: "conf.yaml", cmd: &mockCmdNoFmt, }, - want: asconf.YAML, + want: conf.YAML, wantErr: false, }, { @@ -46,7 +46,7 @@ func Test_getConfFileFormat(t *testing.T) { path: "conf.conf", cmd: &mockCmdNoFmt, }, - want: asconf.AsConfig, + want: conf.AsConfig, wantErr: false, }, { @@ -55,7 +55,7 @@ func Test_getConfFileFormat(t *testing.T) { path: "conf.yaml", cmd: &mockCmd, }, - want: asconf.AsConfig, + want: conf.AsConfig, wantErr: false, }, { @@ -64,7 +64,7 @@ func Test_getConfFileFormat(t *testing.T) { path: "../testdata/sources/all_flash_cluster_cr.bad", cmd: &mockCmdNoFmt, }, - want: asconf.Invalid, + want: conf.Invalid, wantErr: true, }, } diff --git a/cmd/validate.go b/cmd/validate.go index c284f9b..4324639 100644 --- a/cmd/validate.go +++ b/cmd/validate.go @@ -5,7 +5,7 @@ import ( "fmt" "os" - "github.com/aerospike/asconfig/asconf" + "github.com/aerospike/asconfig/conf" "github.com/spf13/cobra" ) @@ -84,27 +84,18 @@ func newValidateCmd() *cobra.Command { logger.Debugf("Processing flag aerospike-version value=%s", version) - conf, err := asconf.NewAsconf( - fdata, - srcFormat, - // we aren't converting to anything so set - // output format to Invalid as a place holder - asconf.Invalid, - version, - logger, - managementLibLogger, - ) + asconfig, err := conf.NewASConfigFromBytes(mgmtLibLogger, fdata, srcFormat) if err != nil { return err } - verrs, err := conf.Validate() + verrs, err := conf.NewConfigValidator(asconfig, mgmtLibLogger, version).Validate() if verrs != nil { // force validation errors to be written to stdout // so they can more easily be grepd etc. cmd.Print(verrs.Error()) - return errors.Join(asconf.ErrConfigValidation, SilentError) + return errors.Join(conf.ErrConfigValidation, ErrSilent) } if err != nil { return err @@ -119,6 +110,7 @@ func newValidateCmd() *cobra.Command { // is in the input config file's metadata commonFlags := getCommonFlags() res.Flags().AddFlagSet(commonFlags) + res.Flags().StringP("format", "F", "conf", "The format of the source file(s). Valid options are: yaml, yml, and conf.") res.Version = VERSION diff --git a/cmd/validate_test.go b/cmd/validate_test.go index 6964af6..020610a 100644 --- a/cmd/validate_test.go +++ b/cmd/validate_test.go @@ -1,5 +1,4 @@ //go:build unit -// +build unit package cmd @@ -50,6 +49,16 @@ var testValidateArgs = []runTestValidate{ arguments: []string{"../testdata/cases/server70/server70.conf"}, expectError: false, }, + { + flags: []string{"--format", "bad_fmt"}, + arguments: []string{}, + expectError: true, + }, + { + flags: []string{"-F", "bad_fmt"}, + arguments: []string{}, + expectError: true, + }, } func TestRunEValidate(t *testing.T) { diff --git a/conf/config_handler.go b/conf/config_handler.go new file mode 100644 index 0000000..6f55f7b --- /dev/null +++ b/conf/config_handler.go @@ -0,0 +1,13 @@ +package conf + +import ( + "github.com/aerospike/aerospike-management-lib/asconfig" + "github.com/go-logr/logr" +) + +type ConfHandler interface { + IsValid(log logr.Logger, version string) (bool, []*asconfig.ValidationErr, error) + ToMap() *asconfig.Conf + ToConfFile() asconfig.DotConf + GetFlatMap() *asconfig.Conf +} diff --git a/conf/config_handler_mock.go b/conf/config_handler_mock.go new file mode 100644 index 0000000..4a0f5ed --- /dev/null +++ b/conf/config_handler_mock.go @@ -0,0 +1,35 @@ +//go:build unit + +package conf + +import ( + "github.com/aerospike/aerospike-management-lib/asconfig" + "github.com/go-logr/logr" +) + +// TODO: Use a gomock instead. It is more robust in its assertions. + +type mockCFG struct { + valid bool + err error + validationErrors []*asconfig.ValidationErr + confMap *asconfig.Conf + confText string + flatConf *asconfig.Conf +} + +func (o *mockCFG) IsValid(log logr.Logger, version string) (bool, []*asconfig.ValidationErr, error) { + return o.valid, o.validationErrors, o.err +} + +func (o *mockCFG) ToMap() *asconfig.Conf { + return o.confMap +} + +func (o *mockCFG) ToConfFile() asconfig.DotConf { + return o.confText +} + +func (o *mockCFG) GetFlatMap() *asconfig.Conf { + return o.flatConf +} diff --git a/conf/config_marshaller.go b/conf/config_marshaller.go new file mode 100644 index 0000000..5cbcb82 --- /dev/null +++ b/conf/config_marshaller.go @@ -0,0 +1,34 @@ +package conf + +import ( + "fmt" + + "gopkg.in/yaml.v3" +) + +type ConfigMarshaller struct { + Format Format + ConfHandler +} + +func NewConfigMarshaller(conf ConfHandler, format Format) ConfigMarshaller { + return ConfigMarshaller{ + Format: format, + ConfHandler: conf, + } +} + +func (cm ConfigMarshaller) MarshalText() (text []byte, err error) { + + switch cm.Format { + case AsConfig: + text = []byte(cm.ToConfFile()) + case YAML: + m := cm.ToMap() + text, err = yaml.Marshal(m) + default: + err = fmt.Errorf("%w %s", ErrInvalidFormat, cm.Format) + } + + return +} diff --git a/conf/config_marshaller_test.go b/conf/config_marshaller_test.go new file mode 100644 index 0000000..8f4d0eb --- /dev/null +++ b/conf/config_marshaller_test.go @@ -0,0 +1,71 @@ +//go:build unit + +package conf + +import ( + "reflect" + "testing" + + "github.com/aerospike/aerospike-management-lib/asconfig" +) + +func Test_asconf_MarshalText(t *testing.T) { + type fields struct { + cfg ConfHandler + outFmt Format + } + tests := []struct { + name string + fields fields + wantText []byte + wantErr bool + }{ + { + name: "valid asconfig format", + fields: fields{ + cfg: &mockCFG{ + confText: "namespace ns1 {}\n namespace ns2 {}", + }, + outFmt: AsConfig, + }, + wantErr: false, + wantText: []byte("namespace ns1 {}\n namespace ns2 {}"), + }, + { + name: "valid yaml format", + fields: fields{ + cfg: &mockCFG{ + confMap: &asconfig.Conf{ + "namespaces": "ns1", + }, + confText: "", + }, + outFmt: YAML, + }, + wantErr: false, + wantText: []byte("namespaces: ns1\n"), + }, + { + name: "invalid format", + fields: fields{ + outFmt: Invalid, + }, + wantErr: true, + wantText: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ac := NewConfigMarshaller(tt.fields.cfg, tt.fields.outFmt) + gotText, err := ac.MarshalText() + + if (err != nil) != tt.wantErr { + t.Errorf("asconf.MarshalText() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotText, tt.wantText) { + t.Errorf("asconf.MarshalText() = %v, want %v", gotText, tt.wantText) + } + }) + } +} diff --git a/conf/config_validator.go b/conf/config_validator.go new file mode 100644 index 0000000..934e101 --- /dev/null +++ b/conf/config_validator.go @@ -0,0 +1,109 @@ +package conf + +import ( + "errors" + "fmt" + "sort" + "strings" + + "github.com/aerospike/aerospike-management-lib/asconfig" + "github.com/go-logr/logr" +) + +var ( + ErrConfigValidation = fmt.Errorf("error while validating config") +) + +type ConfigValidator struct { + ConfHandler + mgmtLogger logr.Logger + version string +} + +func NewConfigValidator(confHandler ConfHandler, mgmtLogger logr.Logger, version string) *ConfigValidator { + return &ConfigValidator{ + ConfHandler: confHandler, + mgmtLogger: mgmtLogger, + version: version, + } +} + +// Validate validates the parsed configuration against the schema for the given versions. +// ValidationErrors is not nil if any errors during validation occur. +func (cv *ConfigValidator) Validate() (*ValidationErrors, error) { + + valid, tempVerrs, err := cv.IsValid(cv.mgmtLogger, cv.version) + + verrs := ValidationErrors{} + for _, v := range tempVerrs { + verr := ValidationErr{ + ValidationErr: *v, + } + verrs.Errors = append(verrs.Errors, verr) + } + + if !valid || err != nil || len(verrs.Errors) > 0 { + return &verrs, errors.Join(ErrConfigValidation, err) + } + + return nil, nil +} + +type ValidationErr struct { + asconfig.ValidationErr +} + +type VErrSlice []ValidationErr + +func (a VErrSlice) Len() int { return len(a) } +func (a VErrSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a VErrSlice) Less(i, j int) bool { return strings.Compare(a[i].Error(), a[j].Error()) == -1 } + +// Outputs a human readable string of validation error details. +// error is not nil if validation, or any other type of error occurs. +func (o ValidationErr) Error() string { + verrTemplate := "description: %s, error-type: %s" + return fmt.Sprintf(verrTemplate, o.Description, o.ErrType) +} + +type ValidationErrors struct { + Errors VErrSlice +} + +func (o ValidationErrors) Error() string { + errorsByContext := map[string]VErrSlice{} + + sort.Sort(o.Errors) + + for _, err := range o.Errors { + errorsByContext[err.Context] = append(errorsByContext[err.Context], err) + } + + contexts := []string{} + for ctx := range errorsByContext { + contexts = append(contexts, ctx) + } + + sort.Strings(contexts) + + errString := "" + + for _, ctx := range contexts { + errString += fmt.Sprintf("context: %s\n", ctx) + + errList := errorsByContext[ctx] + for _, err := range errList { + + // filter "Must validate one and only one schema " errors + // I have never seen a useful one and they seem to always be + // accompanied by another more useful error that will be displayed + if err.ErrType == "number_one_of" { + continue + } + + errString += fmt.Sprintf("\t- %s\n", err.Error()) + } + } + + return errString +} diff --git a/conf/config_validator_test.go b/conf/config_validator_test.go new file mode 100644 index 0000000..dc802a7 --- /dev/null +++ b/conf/config_validator_test.go @@ -0,0 +1,83 @@ +//go:build unit + +package conf + +import ( + "fmt" + "testing" + + "github.com/aerospike/aerospike-management-lib/asconfig" + "github.com/go-logr/logr" + "github.com/sirupsen/logrus" +) + +func Test_asconf_Validate(t *testing.T) { + type fields struct { + cfg ConfHandler + logger *logrus.Logger + managementLibLogger logr.Logger + aerospikeVersion string + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + { + name: "pos1", + fields: fields{ + cfg: &mockCFG{ + valid: true, + err: nil, + validationErrors: []*asconfig.ValidationErr{}, + }, + logger: logrus.New(), + }, + wantErr: false, + }, + { + name: "neg1", + fields: fields{ + cfg: &mockCFG{ + valid: false, + err: nil, + validationErrors: []*asconfig.ValidationErr{}, + }, + logger: logrus.New(), + }, + wantErr: true, + }, + { + name: "neg2", + fields: fields{ + cfg: &mockCFG{ + valid: true, + err: fmt.Errorf("test_err"), + validationErrors: []*asconfig.ValidationErr{}, + }, + logger: logrus.New(), + }, + wantErr: true, + }, + { + name: "neg3", + fields: fields{ + cfg: &mockCFG{ + valid: true, + err: nil, + validationErrors: []*asconfig.ValidationErr{{}}, + }, + logger: logrus.New(), + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ac := NewConfigValidator(tt.fields.cfg, tt.fields.managementLibLogger, tt.fields.aerospikeVersion) + if _, err := ac.Validate(); (err != nil) != tt.wantErr { + t.Errorf("asconf.Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/conf/constants.go b/conf/constants.go new file mode 100644 index 0000000..9849cca --- /dev/null +++ b/conf/constants.go @@ -0,0 +1,13 @@ +package conf + +import "fmt" + +type Format string + +const ( + Invalid Format = "" + YAML Format = "yaml" + AsConfig Format = "asconfig" +) + +var ErrInvalidFormat = fmt.Errorf("invalid config format") diff --git a/asconf/utils.go b/conf/loader.go similarity index 67% rename from asconf/utils.go rename to conf/loader.go index 80ae068..d6e97ea 100644 --- a/asconf/utils.go +++ b/conf/loader.go @@ -1,49 +1,108 @@ -package asconf +package conf import ( + "bytes" "fmt" "sort" "strings" lib "github.com/aerospike/aerospike-management-lib" + "github.com/aerospike/aerospike-management-lib/asconfig" + "github.com/go-logr/logr" + "gopkg.in/yaml.v3" ) -// copied from the management lib asconfig package -var singularToPlural = map[string]string{ - "access-address": "access-addresses", - "address": "addresses", - "alternate-access-address": "alternate-access-addresses", - "datacenter": "datacenters", - "dc": "dcs", - "dc-int-ext-ipmap": "dc-int-ext-ipmap", - "dc-node-address-port": "dc-node-address-ports", - "device": "devices", - "file": "files", - "feature-key-file": "feature-key-files", - "mount": "mounts", - "http-url": "http-urls", - "ignore-bin": "ignore-bins", - "ignore-set": "ignore-sets", - "logging": "logging", - "mesh-seed-address-port": "mesh-seed-address-ports", - "multicast-group": "multicast-groups", - "namespace": "namespaces", - "node-address-port": "node-address-ports", - "report-data-op": "report-data-op", - "report-data-op-user": "report-data-op-user", - "report-data-op-role": "report-data-op-role", - "role-query-pattern": "role-query-patterns", - "set": "sets", - "ship-bin": "ship-bins", - "ship-set": "ship-sets", - "tls": "tls", - "tls-access-address": "tls-access-addresses", - "tls-address": "tls-addresses", - "tls-alternate-access-address": "tls-alternate-access-addresses", - "tls-mesh-seed-address-port": "tls-mesh-seed-address-ports", - "tls-node": "tls-nodes", - "xdr-remote-datacenter": "xdr-remote-datacenters", - "tls-authenticate-client": "tls-authenticate-client", +func NewASConfigFromBytes(log logr.Logger, src []byte, srcFmt Format) (*asconfig.AsConfig, error) { + var err error + var cfg *asconfig.AsConfig + + switch srcFmt { + case YAML: + cfg, err = loadYAML(log, src) + case AsConfig: + cfg, err = loadAsConf(log, src) + default: + return nil, fmt.Errorf("%w %s", ErrInvalidFormat, srcFmt) + } + + if err != nil { + return nil, err + } + + // recreate the management lib config + // with a sorted config map so that output + // is always in the same order + cmap := cfg.ToMap() + + mutateMap(*cmap, []mapping{ + sortLists, + }) + + cfg, err = asconfig.NewMapAsConfig( + log, + "", // TODO: Remove when management lib merges PR #41 + *cmap, + ) + + return cfg, err +} + +func loadYAML(log logr.Logger, src []byte) (*asconfig.AsConfig, error) { + + var data map[string]any + + err := yaml.Unmarshal(src, &data) + if err != nil { + return nil, err + } + + c, err := asconfig.NewMapAsConfig( + log, + "", // TODO: Remove when management lib merges PR #41 + data, + ) + + if err != nil { + return nil, fmt.Errorf("failed to initialize asconfig from yaml: %w", err) + } + + return c, nil +} + +func loadAsConf(log logr.Logger, src []byte) (*asconfig.AsConfig, error) { + + reader := bytes.NewReader(src) + + // TODO: Why doesn't the management lib do the map mutation? FromConfFile + // implies it does. + c, err := asconfig.FromConfFile(log, "", reader) // TODO: Remove "" when management lib merges PR #41 + if err != nil { + return nil, fmt.Errorf("failed to parse asconfig file: %w", err) + } + + // the aerospike management lib parses asconfig files into + // a format that its validation rejects + // this is because the schema files are meant to + // validate the aerospike kubernetes operator's asconfig yaml format + // so we modify the map here to match that format + cmap := *c.ToMap() + + mutateMap(cmap, []mapping{ + typedContextsToObject, + toPlural, + }) + + c, err = asconfig.NewMapAsConfig( + log, + "", // TODO: Remove when management lib merges PR #41 + cmap, + ) + + if err != nil { + return nil, err + } + + return c, nil } type configMap = lib.Stats @@ -73,85 +132,6 @@ func mutateMap(in configMap, funcs []mapping) { } } -/* -sortLists sorts slices of config sections by the "name" or "type" -key that the management lib adds to config list items -Ex config: -namespace ns2 {} -namespace ns1 {} --> -namespace ns1 {} -namespace ns2 {} - -Ex matching configMap - - configMap{ - "namespace": []configMap{ - configMap{ - "name": "ns2", - }, - configMap{ - "name": "ns1", - }, - } - } - --> - - configMap{ - "namespace": []configMap{ - configMap{ - "name": "ns1", - }, - configMap{ - "name": "ns2", - }, - } - } -*/ -func sortLists(k string, v any, m configMap) { - if v, ok := v.([]configMap); ok { - sort.Slice(v, func(i int, j int) bool { - iv, iok := v[i]["name"] - jv, jok := v[j]["name"] - - // sections may also use the "type" field to identify themselves - if !iok { - iv, iok = v[i]["type"] - } - - if !jok { - jv, jok = v[j]["type"] - } - - // if i or both don't have id fields, consider them i >= j - if !iok { - return false - } - - // if only j has an id field consider i < j - if !jok { - return true - } - - iname := iv.(string) - jname := jv.(string) - - gt := strings.Compare(iname, jname) - - switch gt { - case 1: - return true - case -1, 0: - return false - default: - panic("unexpected gt value") - } - }) - m[k] = v - } -} - /* typedContextsToObject converts config entries that the management lib parses as literal strings into the objects that the yaml schemas expect. @@ -186,6 +166,18 @@ func typedContextsToObject(k string, v any, m configMap) { } } +// isTypedContext returns true for asconfig contexts +// that can map to strings instead of contexts +func isTypedContext(in string) bool { + + switch in { + case "storage-engine", "index-type", "sindex-type": + return true + default: + return false + } +} + /* toPlural converts the keys that the management lib asconf parser parses as singular, to the plural keys that the yaml schemas expect @@ -209,7 +201,7 @@ func toPlural(k string, v any, m configMap) { // convert asconfig fields/contexts that need to be plural // in order to create valid asconfig yaml. - if plural, ok := singularToPlural[k]; ok { + if plural := asconfig.PluralOf(k); plural != k { // if the config item can be plural or singular and is not a slice // then the item should not be converted to the plural form. // If the management lib ever parses list entries as anything other @@ -247,29 +239,81 @@ func isListOrString(name string) bool { } } -// isTypedContext returns true for asconfig contexts -// that can map to strings instead of contexts -func isTypedContext(in string) bool { +/* +sortLists sorts slices of config sections by the "name" or "type" +key that the management lib adds to config list items +Ex config: +namespace ns2 {} +namespace ns1 {} +-> +namespace ns1 {} +namespace ns2 {} - switch in { - case "storage-engine", "index-type", "sindex-type": - return true - default: - return false +Ex matching configMap + + configMap{ + "namespace": []configMap{ + configMap{ + "name": "ns2", + }, + configMap{ + "name": "ns1", + }, + } } -} -func ParseFmtString(in string) (f Format, err error) { +-> - switch strings.ToLower(in) { - case "yaml", "yml": - f = YAML - case "asconfig", "conf", "asconf": - f = AsConfig - default: - f = Invalid - err = fmt.Errorf("%w: %s", ErrInvalidFormat, in) + configMap{ + "namespace": []configMap{ + configMap{ + "name": "ns1", + }, + configMap{ + "name": "ns2", + }, + } } +*/ +func sortLists(k string, v any, m configMap) { + if v, ok := v.([]configMap); ok { + sort.Slice(v, func(i int, j int) bool { + iv, iok := v[i]["name"] + jv, jok := v[j]["name"] + + // sections may also use the "type" field to identify themselves + if !iok { + iv, iok = v[i]["type"] + } - return + if !jok { + jv, jok = v[j]["type"] + } + + // if i or both don't have id fields, consider them i >= j + if !iok { + return false + } + + // if only j has an id field consider i < j + if !jok { + return true + } + + iname := iv.(string) + jname := jv.(string) + + gt := strings.Compare(iname, jname) + + switch gt { + case 1: + return true + case -1, 0: + return false + default: + panic("unexpected gt value") + } + }) + m[k] = v + } } diff --git a/asconf/metadata/metadata.go b/conf/metadata/metadata.go similarity index 100% rename from asconf/metadata/metadata.go rename to conf/metadata/metadata.go diff --git a/asconf/metadata/metadata_test.go b/conf/metadata/metadata_test.go similarity index 97% rename from asconf/metadata/metadata_test.go rename to conf/metadata/metadata_test.go index 8ceb001..9df2aab 100644 --- a/asconf/metadata/metadata_test.go +++ b/conf/metadata/metadata_test.go @@ -1,13 +1,10 @@ -//go:build unit -// +build unit - package metadata_test import ( "reflect" "testing" - "github.com/aerospike/asconfig/asconf/metadata" + "github.com/aerospike/asconfig/conf/metadata" ) var testBasic = ` diff --git a/constants.go b/constants.go new file mode 100644 index 0000000..490c4f5 --- /dev/null +++ b/constants.go @@ -0,0 +1,9 @@ +package main + +type Format string + +const ( + Invalid Format = "" + YAML Format = "yaml" + AsConfig Format = "asconfig" +) diff --git a/go.mod b/go.mod index 8f16b0c..433706a 100644 --- a/go.mod +++ b/go.mod @@ -3,37 +3,45 @@ module github.com/aerospike/asconfig go 1.20 require ( - github.com/aerospike/aerospike-management-lib v0.0.0-20231106202816-b2438dbb7e03 - github.com/bombsimon/logrusr/v4 v4.0.0 - github.com/docker/docker v24.0.6+incompatible - github.com/go-logr/logr v1.2.4 + github.com/aerospike/aerospike-client-go/v6 v6.14.1 + github.com/aerospike/aerospike-management-lib v1.1.0 + github.com/aerospike/tools-common-go v0.0.0-20240119223231-5879272be88e + github.com/bombsimon/logrusr/v4 v4.1.0 + github.com/docker/docker v24.0.7+incompatible + github.com/go-logr/logr v1.4.1 github.com/opencontainers/image-spec v1.0.2 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/cobra v1.7.0 + github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 gopkg.in/yaml.v3 v3.0.1 ) require ( + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/docker/distribution v2.8.2+incompatible // indirect - github.com/docker/go-connections v0.4.0 // indirect + github.com/deckarep/golang-set/v2 v2.6.0 // indirect + github.com/distribution/reference v0.5.0 // indirect + github.com/docker/distribution v2.8.3+incompatible // indirect + github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/moby/term v0.5.0 // indirect - github.com/morikuni/aec v1.0.0 // indirect + github.com/kr/pretty v0.3.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/qdm12/reprint v0.0.0-20200326205758-722754a53494 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect - golang.org/x/mod v0.9.0 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.7.0 // indirect - gotest.tools/v3 v3.5.1 // indirect - k8s.io/apimachinery v0.27.2 // indirect + github.com/yuin/gopher-lua v1.1.1 // indirect + golang.org/x/mod v0.14.0 // indirect + golang.org/x/net v0.20.0 // indirect + golang.org/x/sync v0.6.0 // indirect + golang.org/x/sys v0.16.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/tools v0.16.1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240108191215-35c7eff3a6b1 // indirect + google.golang.org/grpc v1.60.1 // indirect + google.golang.org/protobuf v1.32.0 // indirect ) diff --git a/go.sum b/go.sum index 46549fa..ba89267 100644 --- a/go.sum +++ b/go.sum @@ -1,35 +1,58 @@ github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/aerospike/aerospike-management-lib v0.0.0-20231106202816-b2438dbb7e03 h1:Od7oCSBCTfpRmk73fbNGXA53U1in38iNSklCEwMMZFc= -github.com/aerospike/aerospike-management-lib v0.0.0-20231106202816-b2438dbb7e03/go.mod h1:O4v2oGl4VjG9KwYJoSVEwZXv1PUB4ioKAsrm2tczJPQ= -github.com/bombsimon/logrusr/v4 v4.0.0 h1:Pm0InGphX0wMhPqC02t31onlq9OVyJ98eP/Vh63t1Oo= -github.com/bombsimon/logrusr/v4 v4.0.0/go.mod h1:pjfHC5e59CvjTBIU3V3sGhFWFAnsnhOR03TRc6im0l8= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/aerospike/aerospike-client-go/v6 v6.14.1 h1:1DB9rgbPcCSjR7QS+2CL4MM4atdVcRiWa2AVKO7ydyY= +github.com/aerospike/aerospike-client-go/v6 v6.14.1/go.mod h1:/0Wm81GhMqem+9flWcpazPKoRfjFeG6WrQdXGiMNi0A= +github.com/aerospike/aerospike-management-lib v1.1.0 h1:QchwRIzxxqwnjKHkqbvdDjYl6zHTXiFpmWcN3dSPPuI= +github.com/aerospike/aerospike-management-lib v1.1.0/go.mod h1:NwegUX6or8xmwVMIueBmGTW7lfKlZ9XDQoCuhnQKMCA= +github.com/aerospike/tools-common-go v0.0.0-20240119223231-5879272be88e h1:0XSLj7JHg3gXanQwUqE4my2scwrpFor9vJdqHGXvNsI= +github.com/aerospike/tools-common-go v0.0.0-20240119223231-5879272be88e/go.mod h1:JaDR4z9G/GsYx7N33UBmKY0Bm0QlKpCzX23Kub1FOVo= +github.com/bombsimon/logrusr/v4 v4.1.0 h1:uZNPbwusB0eUXlO8hIUwStE6Lr5bLN6IgYgG+75kuh4= +github.com/bombsimon/logrusr/v4 v4.1.0/go.mod h1:pjfHC5e59CvjTBIU3V3sGhFWFAnsnhOR03TRc6im0l8= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= -github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v24.0.6+incompatible h1:hceabKCtUgDqPu+qm0NgsaXf28Ljf4/pWFL7xjWWDgE= -github.com/docker/docker v24.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= +github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= +github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= +github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= +github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM= +github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/onsi/ginkgo/v2 v2.9.2 h1:BA2GMJOtfGAfagzYtrAlufIP0lq6QERkFmHLMLPwFSU= +github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= @@ -40,18 +63,20 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/qdm12/reprint v0.0.0-20200326205758-722754a53494 h1:wSmWgpuccqS2IOfmYrbRiUgv+g37W5suLLLxwwniTSc= github.com/qdm12/reprint v0.0.0-20200326205758-722754a53494/go.mod h1:yipyliwI08eQ6XwDm1fEwKPdF/xdbkiHtrU+1Hg+vc4= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -61,50 +86,63 @@ github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= +github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= -golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= -golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= +golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240108191215-35c7eff3a6b1 h1:gphdwh0npgs8elJ4T6J+DQJHPVF7RsuJHCfwztUb4J4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240108191215-35c7eff3a6b1/go.mod h1:daQN87bsDqDoe316QbbvX60nMoJQa4r6Ds0ZuoAe5yA= +google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= +google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= -gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= -k8s.io/apimachinery v0.27.2 h1:vBjGaKKieaIreI+oQwELalVG4d8f3YAMNpWLzDXkxeg= -k8s.io/apimachinery v0.27.2/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E= diff --git a/integration_test.go b/integration_test.go index c137341..40fa683 100644 --- a/integration_test.go +++ b/integration_test.go @@ -1,5 +1,4 @@ //go:build integration -// +build integration package main @@ -20,9 +19,13 @@ import ( "testing" "time" - "github.com/aerospike/asconfig/asconf/metadata" + as "github.com/aerospike/aerospike-client-go/v6" + mgmtLib "github.com/aerospike/aerospike-management-lib" + mgmtLibInfo "github.com/aerospike/aerospike-management-lib/info" "github.com/aerospike/asconfig/cmd" + "github.com/aerospike/asconfig/conf/metadata" "github.com/aerospike/asconfig/testutils" + "github.com/go-logr/logr" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" @@ -70,10 +73,10 @@ func TestMain(m *testing.M) { panic(err) } - featKeyDir = os.Getenv("FEATKEY") + featKeyDir = os.Getenv("FEATKEY_DIR") fmt.Println(featKeyDir) if featKeyDir == "" { - panic("FEATKEY environement variable must be full path to a directory containing valid aerospike feature key files featuresv1.conf and featuresv2.conf of feature key format 1 and 2 respectively.") + panic("FEATKEY_DIR environement variable must an absolute path to a directory containing valid aerospike feature key files featuresv1.conf and featuresv2.conf of feature key format 1 and 2 respectively.") } code := m.Run() @@ -228,28 +231,6 @@ var testFiles = []testutils.TestData{ }, } -func versionLessThanEqual(l string, r string) bool { - var majorLess, minorLess, patchLess bool - ltok := strings.Split(l, ".") - rtok := strings.Split(r, ".") - - lmaj, _ := strconv.Atoi(ltok[0]) - lmin, _ := strconv.Atoi(ltok[1]) - lpat, _ := strconv.Atoi(ltok[2]) - - rmaj, _ := strconv.Atoi(rtok[0]) - rmin, _ := strconv.Atoi(rtok[1]) - rpat, _ := strconv.Atoi(rtok[2]) - - majorLess = lmaj <= rmaj - minorLess = lmin <= rmin - patchLess = lpat <= rpat - - return (majorLess && minorLess && patchLess) || - (majorLess && minorLess && !patchLess) || - (majorLess && !minorLess && !patchLess) -} - func getVersion(l []string) (v string) { i := testutils.IndexOf(l, "-a") if i >= 0 { @@ -289,7 +270,7 @@ func TestYamlToConf(t *testing.T) { t.Logf("Skipping getExtraTests: %v", err) } - testFiles = append(testFiles, extraTests...) + testFiles = append(extraTests, testFiles...) for i, tf := range testFiles { var err error @@ -334,7 +315,12 @@ func TestYamlToConf(t *testing.T) { // test that the converted config works with an Aerospike server if !tf.SkipServerTest { version := getVersion(tf.Arguments) - runServer(version, confPath, dockerClient, t, tf) + id, _ := runServer(version, tf.ServerImage, confPath, tf.DockerAuth, dockerClient, t) + + time.Sleep(time.Second * 3) // need this to allow logs to accumulate + checkContainerLogs(id, t, tf, tmpServerLogPath) + + stopServer(id, dockerClient) } // cleanup the destination file @@ -400,10 +386,10 @@ func getDockerAuthFromEnv(auth testutils.DockerAuth) (string, error) { return authStr, nil } -func runServer(version string, confPath string, dockerClient *client.Client, t *testing.T, td testutils.TestData) { +func runServer(version string, serverVersion string, confPath string, auth testutils.DockerAuth, dockerClient *client.Client, t *testing.T) (string, string) { containerName := "aerospike:" + version - if td.ServerImage != "" { - containerName = td.ServerImage + if serverVersion != "" { + containerName = serverVersion } var err error @@ -437,7 +423,10 @@ func runServer(version string, confPath string, dockerClient *client.Client, t * featureKeyPath := filepath.Join(featKeyDir, "featuresv2.conf") lastServerWithFeatureKeyVersion1 := "5.3.0" - if versionLessThanEqual(strings.TrimPrefix(version, "ee-"), lastServerWithFeatureKeyVersion1) { + + if val, err := mgmtLib.CompareVersionsIgnoreRevision(strings.TrimPrefix(version, "ee-"), lastServerWithFeatureKeyVersion1); err != nil { + t.Error(err) + } else if val <= 0 { featureKeyPath = filepath.Join(featKeyDir, "featuresv1.conf") } @@ -471,7 +460,7 @@ func runServer(version string, confPath string, dockerClient *client.Client, t * }, } - dockerAuth, err := getDockerAuthFromEnv(td.DockerAuth) + dockerAuth, err := getDockerAuthFromEnv(auth) if err != nil { t.Error(err) } @@ -485,28 +474,31 @@ func runServer(version string, confPath string, dockerClient *client.Client, t * id, err := testutils.CreateAerospikeContainer(containerName, containerConf, containerHostConf, imagePullOptions, dockerClient) if err != nil { t.Error(err) + return "", "" } - // cleanup container - defer func() { - err = testutils.RemoveAerospikeContainer(id, dockerClient) - if err != nil { - t.Error(err) - } - }() + ctx := context.Background() err = testutils.StartAerospikeContainer(id, dockerClient) if err != nil { t.Error(err) + return "", "" } - // need this to allow logs to accumulate - time.Sleep(time.Second * 3) - - err = testutils.StopAerospikeContainer(id, dockerClient) + resp, err := dockerClient.ContainerInspect(ctx, id) if err != nil { t.Error(err) + return "", "" + } + + return id, resp.NetworkSettings.IPAddress +} + +func stopServer(id string, dockerClient *client.Client) error { + err := testutils.StopAerospikeContainer(id, dockerClient) + if err != nil { + return err } // time for Aerospike to close @@ -518,57 +510,17 @@ func runServer(version string, confPath string, dockerClient *client.Client, t * select { case err := <-errCh: if err != nil { - t.Error(err) + return err // TODO: Check if I need to do something to shutdown the channels. } case <-statusCh: } - data, err := docker("logs", id) + err = testutils.RemoveAerospikeContainer(id, dockerClient) if err != nil { - t.Error(err) - } - - var containerOut string - containerOut = string(data) - - // containerOut := string(logs) - // if the server container logs are empty - // the server may have been configured to log to - // /var/log/aerospike/aerospike.log which is mapped - // to absTmpLog - if len(containerOut) == 0 { - data, err := os.ReadFile(absTmpLog) - if err != nil { - t.Error(err) - } - containerOut = string(data) - } - - // if the logs are still empty, the server logged somewhere else - // or there is a problem, fail in this case - if len(containerOut) == 0 { - t.Errorf("suite: %+v\nAerospike container logs are empty", td) + return err } - // some tests use aerospike versions from when no enterprise container was published - // these will fail with "'x feature' is enterprise-only" - // always ignore this failure - td.ServerErrorAllowList = append(td.ServerErrorAllowList, "' is enterprise-only") - - reg := regexp.MustCompile(`CRITICAL \(config\):.*`) - configErrors := reg.FindAllString(containerOut, -1) - for _, cfgErr := range configErrors { - exempted := false - for _, exemption := range td.ServerErrorAllowList { - if strings.Contains(cfgErr, exemption) { - exempted = true - } - } - - if !exempted { - t.Errorf("suite: %+v\nAerospike encountered a configuration error...\n%s", td, containerOut) - } - } + return nil } var confToYamlTests = []testutils.TestData{ @@ -773,7 +725,12 @@ func TestConfToYaml(t *testing.T) { // test that the converted config works with an Aerospike server if !tf.SkipServerTest { version := getVersion(tf.Arguments) - runServer(version, finalConfPath, dockerClient, t, tf) + id, _ := runServer(version, tf.ServerImage, finalConfPath, tf.DockerAuth, dockerClient, t) + + time.Sleep(time.Second * 3) // need this to allow logs to accumulate + checkContainerLogs(id, t, tf, tmpServerLogPath) + + stopServer(id, dockerClient) } // cleanup the destination files @@ -796,6 +753,59 @@ func docker(args ...string) ([]byte, error) { return out, err } +func getContainerLogs(id string) ([]byte, error) { + return docker("logs", id) +} + +func checkContainerLogs(id string, t *testing.T, td testutils.TestData, absTmpLog string) { + data, err := getContainerLogs(id) + if err != nil { + t.Error(err) + } + + var containerOut string + containerOut = string(data) + + // containerOut := string(logs) + // if the server container logs are empty + // the server may have been configured to log to + // /var/log/aerospike/aerospike.log which is mapped + // to absTmpLog + if len(containerOut) == 0 { + data, err := os.ReadFile(absTmpLog) + if err != nil { + t.Error(err) + } + containerOut = string(data) + } + + // if the logs are still empty, the server logged somewhere else + // or there is a problem, fail in this case + if len(containerOut) == 0 { + t.Errorf("suite: %+v\nAerospike container logs are empty", td) + } + + // some tests use aerospike versions from when no enterprise container was published + // these will fail with "'x feature' is enterprise-only" + // always ignore this failure + td.ServerErrorAllowList = append(td.ServerErrorAllowList, "' is enterprise-only") + + reg := regexp.MustCompile(`CRITICAL \(config\):.*`) + configErrors := reg.FindAllString(containerOut, -1) + for _, cfgErr := range configErrors { + exempted := false + for _, exemption := range td.ServerErrorAllowList { + if strings.Contains(cfgErr, exemption) { + exempted = true + } + } + + if !exempted { + t.Errorf("suite: %+v\nAerospike encountered a configuration error...\n%s", td, containerOut) + } + } +} + func diff(args ...string) ([]byte, error) { args = append([]string{"diff"}, args...) com := exec.Command(binPath+"/asconfig.test", args...) @@ -1065,3 +1075,113 @@ func TestStdinValidate(t *testing.T) { } } } + +type generateTC struct { + source string + destination string + arguments []string + version string +} + +var generateTests = []generateTC{ + { + source: filepath.Join(expectedPath, "dim_nostorage_cluster_cr.conf"), + destination: filepath.Join(destinationPath, "dim_nostorage_cluster_cr.conf"), + arguments: []string{"generate", "-h", "", "-Uadmin", "-Padmin", "--format", "asconfig", "-o", filepath.Join(destinationPath, "dim_nostorage_cluster_cr.conf")}, + version: "ee-6.4.0.2", + }, + // Uncomment after https://github.com/aerospike/schemas/pull/6 is merged and + // the schemas submodule is updated + // { + // source: filepath.Join(expectedPath, "hdd_dii_storage_cluster_cr.conf"), + // destination: filepath.Join(destinationPath, "hdd_dii_storage_cluster_cr.conf"), + // arguments: []string{"generate", "-h", "", "-Uadmin", "-Padmin", "--format", "asconfig", "-o", filepath.Join(destinationPath, "hdd_dii_storage_cluster_cr.conf")}, + // version: "ee-6.2.0.2", + // }, + // { + // source: filepath.Join(expectedPath, "hdd_dim_storage_cluster_cr.conf"), + // destination: filepath.Join(destinationPath, "hdd_dim_storage_cluster_cr.conf"), + // arguments: []string{"generate", "-h", "", "-Uadmin", "-Padmin", "--format", "asconfig", "--output", filepath.Join(destinationPath, "hdd_dim_storage_cluster_cr.conf")}, + // version: "ee-6.4.0.2", + // }, + { + source: filepath.Join(expectedPath, "podspec_cr.conf"), + destination: filepath.Join(destinationPath, "podspec_cr.conf"), + arguments: []string{"generate", "-h", "", "-Uadmin", "-Padmin", "--format", "asconfig", "-o", filepath.Join(destinationPath, "podspec_cr.conf")}, + version: "ee-6.4.0.2", + }, + { + source: filepath.Join(expectedPath, "shadow_file_cluster_cr.conf"), + destination: filepath.Join(destinationPath, "shadow_file_cluster_cr.conf"), + arguments: []string{"generate", "-h", "", "-Uadmin", "-Padmin", "--format", "asconfig", "-o", filepath.Join(destinationPath, "shadow_file_cluster_cr.conf")}, + version: "ee-6.4.0.2", + }, +} + +func TestGenerate(t *testing.T) { + dockerClient, _ := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + + for _, tf := range generateTests { + var err error + + id, ip := runServer(tf.version, "", tf.source, testutils.DockerAuth{}, dockerClient, t) + + // Make a copy of tf.firstArgs + firstArgs := make([]string, len(tf.arguments)) + copy(firstArgs, tf.arguments) + + for idx, arg := range firstArgs { + if arg == "" { + firstArgs[idx] = ip + } + } + + time.Sleep(time.Second * 3) // need this to allow aerospike to startup + + test := exec.Command(binPath+"/asconfig.test", firstArgs...) + test.Env = []string{"GOCOVERDIR=" + coveragePath} + _, err = test.Output() + if err != nil { + t.Errorf("\nTESTCASE: %+v\nERR: %+v\n", firstArgs, string(err.(*exec.ExitError).Stderr)) + return + } + + asPolicy := as.NewClientPolicy() + asHost := as.NewHost(ip, 3000) + + asPolicy.User = "admin" + asPolicy.Password = "admin" + + firstConf, err := mgmtLibInfo.NewAsInfo(logr.Logger{}, asHost, asPolicy).AllConfigs() + + if err != nil { + t.Errorf("\nTESTCASE: %+v\nERR: %+v\n", tf, err) + } + + err = stopServer(id, dockerClient) + if err != nil { + t.Errorf("\nTESTCASE: %+v\nERR: %+v\n", tf, string(err.(*exec.ExitError).Stderr)) + } + + id, ip = runServer(tf.version, "", tf.destination, testutils.DockerAuth{}, dockerClient, t) + + time.Sleep(time.Second * 3) // need this to allow aerospike to startup + + asHost = as.NewHost(ip, 3000) + secondConf, err := mgmtLibInfo.NewAsInfo(logr.Logger{}, asHost, asPolicy).AllConfigs() + + if err != nil { + t.Errorf("\nTESTCASE: %+v\nERR: %+v\n", tf, err) + } + + err = stopServer(id, dockerClient) + if err != nil { + t.Errorf("\nTESTCASE: %+v\nERR: %+v\n", tf, string(err.(*exec.ExitError).Stderr)) + } + + // Compare the config of the two servers + if !reflect.DeepEqual(firstConf, secondConf) { + t.Errorf("\nTESTCASE: %+v\ndiff: %+v, %+v\n", tf, firstConf, secondConf) + } + } +} diff --git a/testdata/bin/.gitignore b/testdata/bin/.gitignore deleted file mode 100644 index 86d0cb2..0000000 --- a/testdata/bin/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore \ No newline at end of file diff --git a/testdata/cases/server70/conf-tests.json b/testdata/cases/server70/conf-tests.json index 345ee51..3d29c7b 100644 --- a/testdata/cases/server70/conf-tests.json +++ b/testdata/cases/server70/conf-tests.json @@ -1 +1,10 @@ -[{"Source":"testdata/cases/server70/server70.conf","Destination":"testdata/cases/server70/server70-res-.yaml","Expected":"testdata/cases/server70/server70.yaml","Arguments":["convert","--aerospike-version","7.0.0","--format","asconfig","--output","testdata/cases/server70/server70-res-.yaml"],"SkipServerTest":false,"ServerErrorAllowList":null,"ServerImage":"aerospike.jfrog.io/docker-remote/aerospike/aerospike-server-enterprise-rc:7.0.0.0-rc3","DockerAuth":{"Username":"ASC_DOCKER_USER","Password":"ASC_DOCKER_PASS"}}] \ No newline at end of file +[ + { + "Source":"testdata/cases/server70/server70.conf", + "Destination":"testdata/cases/server70/server70-res-.yaml", + "Expected":"testdata/cases/server70/server70.yaml", + "Arguments":["convert","--aerospike-version","7.0.0.3","--format","asconfig","--output","testdata/cases/server70/server70-res-.yaml"], + "SkipServerTest":false, + "ServerErrorAllowList":null + } +] \ No newline at end of file diff --git a/testdata/cases/server70/yaml-tests.json b/testdata/cases/server70/yaml-tests.json index 772cb64..6937529 100644 --- a/testdata/cases/server70/yaml-tests.json +++ b/testdata/cases/server70/yaml-tests.json @@ -1 +1,10 @@ -[{"Source":"testdata/cases/server70/server70.yaml","Destination":"testdata/cases/server70/server70-res-.conf","Expected":"testdata/cases/server70/server70.conf","Arguments":["convert","--aerospike-version","7.0.0","--format","yaml","--output","testdata/cases/server70/server70-res-.conf"],"SkipServerTest":false,"ServerErrorAllowList":null,"ServerImage":"aerospike.jfrog.io/docker-remote/aerospike/aerospike-server-enterprise-rc:7.0.0.0-rc3","DockerAuth":{"Username":"ASC_DOCKER_USER","Password":"ASC_DOCKER_PASS"}}] \ No newline at end of file +[ + { + "Source":"testdata/cases/server70/server70.yaml", + "Destination":"testdata/cases/server70/server70-res-.conf", + "Expected":"testdata/cases/server70/server70.conf", + "Arguments":["convert","--aerospike-version","7.0.0.2","--format","yaml","--output","testdata/cases/server70/server70-res-.conf"], + "SkipServerTest":false, + "ServerErrorAllowList":null + } +] \ No newline at end of file diff --git a/testdata/expected/dim_nostorage_cluster_cr.conf b/testdata/expected/dim_nostorage_cluster_cr.conf index 159ddfe..20340d4 100644 --- a/testdata/expected/dim_nostorage_cluster_cr.conf +++ b/testdata/expected/dim_nostorage_cluster_cr.conf @@ -24,6 +24,7 @@ network { heartbeat { mode multicast + multicast-group 239.1.99.222 port 3002 } diff --git a/testdata/expected/hdd_dii_storage_cluster_cr.conf b/testdata/expected/hdd_dii_storage_cluster_cr.conf index f74b048..d2cd6a1 100644 --- a/testdata/expected/hdd_dii_storage_cluster_cr.conf +++ b/testdata/expected/hdd_dii_storage_cluster_cr.conf @@ -14,7 +14,7 @@ namespace test { storage-engine device { data-in-memory true - file /opt/aerospike/data/test/test.dat + file /opt/aerospike/data/test.dat filesize 2000000000 } } @@ -27,7 +27,7 @@ namespace bar { storage-engine device { data-in-memory true - file /opt/aerospike/data/bar/bar.dat + file /opt/aerospike/data/bar.dat filesize 2000000000 } } @@ -40,6 +40,7 @@ network { heartbeat { mode multicast + multicast-group 239.1.99.222 port 3002 } diff --git a/testdata/expected/hdd_dim_storage_cluster_cr.conf b/testdata/expected/hdd_dim_storage_cluster_cr.conf index 2da659e..0165fe9 100644 --- a/testdata/expected/hdd_dim_storage_cluster_cr.conf +++ b/testdata/expected/hdd_dim_storage_cluster_cr.conf @@ -25,6 +25,7 @@ network { heartbeat { mode multicast + multicast-group 239.1.99.222 port 3002 } diff --git a/testdata/expected/host_network_cluster_cr.conf b/testdata/expected/host_network_cluster_cr.conf index bbf54c8..c5fc97b 100644 --- a/testdata/expected/host_network_cluster_cr.conf +++ b/testdata/expected/host_network_cluster_cr.conf @@ -23,6 +23,7 @@ network { heartbeat { mode multicast + multicast-group 239.1.99.222 port 3002 } diff --git a/testdata/expected/podspec_cr.conf b/testdata/expected/podspec_cr.conf index d8a86fe..eddd2d7 100644 --- a/testdata/expected/podspec_cr.conf +++ b/testdata/expected/podspec_cr.conf @@ -20,6 +20,7 @@ network { heartbeat { mode multicast + multicast-group 239.1.99.222 port 3002 } diff --git a/testdata/expected/rack_enabled_cluster_cr.conf b/testdata/expected/rack_enabled_cluster_cr.conf index 5c04fea..3bc06ca 100644 --- a/testdata/expected/rack_enabled_cluster_cr.conf +++ b/testdata/expected/rack_enabled_cluster_cr.conf @@ -29,6 +29,7 @@ network { heartbeat { mode multicast + multicast-group 239.1.99.222 port 3002 } diff --git a/testdata/expected/sc_mode_cluster_cr.conf b/testdata/expected/sc_mode_cluster_cr.conf index 8e8aea0..acb0bbf 100644 --- a/testdata/expected/sc_mode_cluster_cr.conf +++ b/testdata/expected/sc_mode_cluster_cr.conf @@ -24,6 +24,7 @@ network { heartbeat { mode multicast + multicast-group 239.1.99.222 port 3002 } diff --git a/testdata/expected/shadow_device_cluster_cr.conf b/testdata/expected/shadow_device_cluster_cr.conf index 0d02a60..0233b74 100644 --- a/testdata/expected/shadow_device_cluster_cr.conf +++ b/testdata/expected/shadow_device_cluster_cr.conf @@ -23,6 +23,7 @@ network { heartbeat { mode multicast + multicast-group 239.1.99.222 port 3002 } diff --git a/testdata/expected/shadow_file_cluster_cr.conf b/testdata/expected/shadow_file_cluster_cr.conf index cf324e0..d487e06 100644 --- a/testdata/expected/shadow_file_cluster_cr.conf +++ b/testdata/expected/shadow_file_cluster_cr.conf @@ -11,7 +11,7 @@ namespace test { replication-factor 2 storage-engine device { - file /local/test.dat /remote/test.dat + file /opt/aerospike/data/test.dat /opt/aerospike/data/shadow-test.dat filesize 2000000000 } } @@ -24,6 +24,7 @@ network { heartbeat { mode multicast + multicast-group 239.1.99.222 port 3002 } diff --git a/testdata/sources/dim_nostorage_cluster_cr.yaml b/testdata/sources/dim_nostorage_cluster_cr.yaml index b97cc66..8e95c78 100644 --- a/testdata/sources/dim_nostorage_cluster_cr.yaml +++ b/testdata/sources/dim_nostorage_cluster_cr.yaml @@ -14,6 +14,7 @@ network: port: 3001 heartbeat: mode: multicast + multicast-groups: 239.1.99.222 port: 3002 service: port: 3000 diff --git a/testdata/sources/hdd_dii_storage_cluster_cr.yaml b/testdata/sources/hdd_dii_storage_cluster_cr.yaml index 97da7e5..fc5b43c 100644 --- a/testdata/sources/hdd_dii_storage_cluster_cr.yaml +++ b/testdata/sources/hdd_dii_storage_cluster_cr.yaml @@ -10,7 +10,7 @@ namespaces: storage-engine: data-in-memory: true files: - - /opt/aerospike/data/test/test.dat + - /opt/aerospike/data/test.dat filesize: 2000000000 type: device - data-in-index: true @@ -21,7 +21,7 @@ namespaces: storage-engine: data-in-memory: true files: - - /opt/aerospike/data/bar/bar.dat + - /opt/aerospike/data/bar.dat filesize: 2000000000 type: device network: @@ -29,6 +29,7 @@ network: port: 3001 heartbeat: mode: multicast + multicast-groups: 239.1.99.222 port: 3002 service: port: 3000 diff --git a/testdata/sources/hdd_dim_storage_cluster_cr.yaml b/testdata/sources/hdd_dim_storage_cluster_cr.yaml index 9f2a98a..bf8c9de 100644 --- a/testdata/sources/hdd_dim_storage_cluster_cr.yaml +++ b/testdata/sources/hdd_dim_storage_cluster_cr.yaml @@ -16,6 +16,7 @@ network: port: 3001 heartbeat: mode: multicast + multicast-groups: 239.1.99.222 port: 3002 service: port: 3000 diff --git a/testdata/sources/host_network_cluster_cr.yaml b/testdata/sources/host_network_cluster_cr.yaml index 53a7e6b..1edebbc 100644 --- a/testdata/sources/host_network_cluster_cr.yaml +++ b/testdata/sources/host_network_cluster_cr.yaml @@ -14,6 +14,7 @@ network: port: 3001 heartbeat: mode: multicast + multicast-groups: 239.1.99.222 port: 3002 service: port: 3000 diff --git a/testdata/sources/podspec_cr.yaml b/testdata/sources/podspec_cr.yaml index e4cc01e..b3bae07 100644 --- a/testdata/sources/podspec_cr.yaml +++ b/testdata/sources/podspec_cr.yaml @@ -12,6 +12,7 @@ network: port: 3001 heartbeat: mode: multicast + multicast-groups: 239.1.99.222 port: 3002 service: port: 3000 diff --git a/testdata/sources/rack_enabled_cluster_cr.yaml b/testdata/sources/rack_enabled_cluster_cr.yaml index a5002c5..a82ae5f 100644 --- a/testdata/sources/rack_enabled_cluster_cr.yaml +++ b/testdata/sources/rack_enabled_cluster_cr.yaml @@ -19,6 +19,7 @@ network: port: 3001 heartbeat: mode: multicast + multicast-groups: 239.1.99.222 port: 3002 service: port: 3000 diff --git a/testdata/sources/sc_mode_cluster_cr.yaml b/testdata/sources/sc_mode_cluster_cr.yaml index b369252..93d5118 100644 --- a/testdata/sources/sc_mode_cluster_cr.yaml +++ b/testdata/sources/sc_mode_cluster_cr.yaml @@ -15,6 +15,7 @@ network: port: 3001 heartbeat: mode: multicast + multicast-groups: 239.1.99.222 port: 3002 service: port: 3000 diff --git a/testdata/sources/shadow_device_cluster_cr.yaml b/testdata/sources/shadow_device_cluster_cr.yaml index f19e18e..ddab067 100644 --- a/testdata/sources/shadow_device_cluster_cr.yaml +++ b/testdata/sources/shadow_device_cluster_cr.yaml @@ -14,6 +14,7 @@ network: port: 3001 heartbeat: mode: multicast + multicast-groups: 239.1.99.222 port: 3002 service: port: 3000 diff --git a/testdata/sources/shadow_file_cluster_cr.yaml b/testdata/sources/shadow_file_cluster_cr.yaml index 5780d54..d82efcd 100644 --- a/testdata/sources/shadow_file_cluster_cr.yaml +++ b/testdata/sources/shadow_file_cluster_cr.yaml @@ -7,7 +7,7 @@ namespaces: replication-factor: 2 storage-engine: files: - - /local/test.dat /remote/test.dat + - /opt/aerospike/data/test.dat /opt/aerospike/data/shadow-test.dat filesize: 2000000000 type: device network: @@ -15,6 +15,7 @@ network: port: 3001 heartbeat: mode: multicast + multicast-groups: 239.1.99.222 port: 3002 service: port: 3000