diff --git a/asconf/metadata/metadata.go b/asconf/metadata/metadata.go new file mode 100644 index 0000000..0b6ef1c --- /dev/null +++ b/asconf/metadata/metadata.go @@ -0,0 +1,60 @@ +package metadata + +import ( + "fmt" + "regexp" + "sort" +) + +var commentChar string +var findComments *regexp.Regexp + +func init() { + commentChar = "#" + // findComments matches text of the form ` : ` + // for example, parsing... + // # comment about metadata + // # a: b + // other data + // matches + // # a: b + findComments = regexp.MustCompile(commentChar + `(?m)\s*(.+):\s*(.+)\s*$`) +} + +func Unmarshal(src []byte, dst map[string]string) error { + matches := findComments.FindAllSubmatch(src, -1) + + for _, match := range matches { + // 0 index is entire line + k := match[1] + v := match[2] + // only save the first occurrence of k + if _, ok := dst[string(k)]; !ok { + dst[string(k)] = string(v) + } + } + + return nil +} + +func formatLine(k string, v any) string { + fmtStr := "%s %s: %v" + return fmt.Sprintf(fmtStr, commentChar, k, v) +} + +func Marshal(src map[string]string) ([]byte, error) { + res := []byte{} + lines := make([]string, len(src)) + + for k, v := range src { + lines = append(lines, formatLine(k, v)+"\n") + } + + sort.Strings(lines) + + for _, v := range lines { + res = append(res, []byte(v)...) + } + + return res, nil +} diff --git a/asconf/metadata/metadata_test.go b/asconf/metadata/metadata_test.go new file mode 100644 index 0000000..8ceb001 --- /dev/null +++ b/asconf/metadata/metadata_test.go @@ -0,0 +1,193 @@ +//go:build unit +// +build unit + +package metadata_test + +import ( + "reflect" + "testing" + + "github.com/aerospike/asconfig/asconf/metadata" +) + +var testBasic = ` +# comment about metadata +# a: b +other data +` + +var testConf = ` +# comment about metadata +# aerospike-server-version: 6.4.0.1 +# asconfig-version: 0.12.0 +# asadm-version: 2.20.0 + +# + +logging { + + file /dummy/file/path2 { + context any info # aerospike-server-version: collide + } +} +` + +var testConfNoMeta = ` +namespace ns2 { + replication-factor 2 + memory-size 8G + index-type shmem # comment mid config + sindex-type shmem + storage-engine memory +} +# comment +` + +var testConfPartialMeta = ` +namespace ns1 { + replication-factor 2 + memory-size 4G + + index-type flash { + mount /dummy/mount/point1 /test/mount2 + mounts-high-water-pct 30 + mounts-size-limit 10G + } + + # comment about metadata + # aerospike-server-version: 6.4.0.1 + # other-item: a long value +` + +func TestUnmarshal(t *testing.T) { + type args struct { + src []byte + dst map[string]string + } + tests := []struct { + name string + args args + want map[string]string + wantErr bool + }{ + { + name: "t1", + args: args{ + src: []byte(testConf), + dst: map[string]string{}, + }, + want: map[string]string{ + "aerospike-server-version": "6.4.0.1", + "asadm-version": "2.20.0", + "asconfig-version": "0.12.0", + }, + wantErr: false, + }, + { + name: "t2", + args: args{ + src: []byte(testConfNoMeta), + dst: map[string]string{}, + }, + want: map[string]string{}, + wantErr: false, + }, + { + name: "t3", + args: args{ + src: []byte(testConfPartialMeta), + dst: map[string]string{}, + }, + want: map[string]string{ + "aerospike-server-version": "6.4.0.1", + "other-item": "a long value", + }, + wantErr: false, + }, + { + name: "t4", + args: args{ + src: []byte(testBasic), + dst: map[string]string{}, + }, + want: map[string]string{ + "a": "b", + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := metadata.Unmarshal(tt.args.src, tt.args.dst); (err != nil) != tt.wantErr { + t.Errorf("Unmarshal() error = %v, wantErr %v", err, tt.wantErr) + } + if !reflect.DeepEqual(tt.args.dst, tt.want) { + t.Errorf("Unmarshal() = %v, want %v", tt.args.dst, tt.want) + } + }) + } +} + +var testMarshalMetaComplete = `# aerospike-server-version: 7.0.0.0 +# asadm-version: 2.20.0 +# asconfig-version: 0.12.0 +` + +var testMarshalMetaNone = "" + +var testMarshalMetaPartial = `# aerospike-server-version: 6.4.0 +` + +func TestMarshalText(t *testing.T) { + type args struct { + src map[string]string + } + tests := []struct { + name string + args args + want []byte + wantErr bool + }{ + { + name: "t1", + args: args{ + src: map[string]string{ + "aerospike-server-version": "7.0.0.0", + "asadm-version": "2.20.0", + "asconfig-version": "0.12.0", + }, + }, + want: []byte(testMarshalMetaComplete), + wantErr: false, + }, + { + name: "t2", + args: args{ + src: map[string]string{}, + }, + want: []byte(testMarshalMetaNone), + wantErr: false, + }, + { + name: "t3", + args: args{ + src: map[string]string{ + "aerospike-server-version": "6.4.0", + }, + }, + want: []byte(testMarshalMetaPartial), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := metadata.Marshal(tt.args.src) + if (err != nil) != tt.wantErr { + t.Errorf("Marshal() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Marshal() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cmd/convert.go b/cmd/convert.go index 7f879e8..9edfba4 100644 --- a/cmd/convert.go +++ b/cmd/convert.go @@ -9,6 +9,7 @@ import ( "github.com/aerospike/aerospike-management-lib/asconfig" "github.com/aerospike/asconfig/asconf" + "github.com/aerospike/asconfig/asconf/metadata" "github.com/spf13/cobra" ) @@ -40,8 +41,9 @@ func newConvertCmd() *cobra.Command { Long: `Convert is used to convert between yaml and aerospike configuration files. Input files are converted to their opposite format, yaml -> conf, conf -> yaml. The convert command validates the configuration file for compatibility with the Aerospike - version passed to the --aerospike-version option, unless the --force option is used. - Specifying the server version that will use the aerospike.conf is required. + version passed to the --aerospike-version option, unless + version metadata is present in the file or the --force option is used. + Aerospike Database metadata is written to files generated by asconfig. Usage examples... convert local file "aerospike.yaml" to aerospike config format for version 6.2.0 and write it to local file "aerospike.conf." @@ -53,7 +55,10 @@ func newConvertCmd() *cobra.Command { Normally the file format is inferred from file extensions ".yaml" ".conf" etc. Source format can be forced with the --format flag. Ex: asconfig convert -a "6.2.0" --format yaml example_file - If a file path is not provided, asconfig reads the file contents from stdin.`, + If a file path is not provided, asconfig reads the file contents from stdin. + Ex: asconfig convert -a "6.4.0" + If the file has been converted by asconfig before, the --aerospike-version option is not needed. + Ex: asconfig convert -a "6.4.0" aerospike.yaml | asconfig convert --format conf`, RunE: func(cmd *cobra.Command, args []string) error { logger.Debug("Running convert command") @@ -101,6 +106,15 @@ func newConvertCmd() *cobra.Command { return fmt.Errorf("%w: %s", errInvalidFormat, srcFormat) } + // if the version option is empty, + // try populating from the metadata + if version == "" { + version, err = getMetaDataItem(fdata, metaKeyAerospikeVersion) + if err != nil { + return errors.Join(errMissingAerospikeVersion, err) + } + } + conf, err := asconf.NewAsconf( fdata, srcFormat, @@ -126,6 +140,17 @@ func newConvertCmd() *cobra.Command { return err } + // prepend metadata to the config output + mtext, err := genMetaDataText(metaDataArgs{ + src: fdata, + aerospikeVersion: version, + asconfigVersion: VERSION, + }) + if err != nil { + return err + } + out = append(mtext, out...) + outputPath, err := cmd.Flags().GetString("output") if err != nil { return err @@ -190,8 +215,22 @@ func newConvertCmd() *cobra.Command { return err } - if !force { - cmd.MarkFlagRequired("aerospike-version") + cfgData, err := os.ReadFile(args[0]) + if err != nil { + return err + } + + metaData := map[string]string{} + metadata.Unmarshal(cfgData, metaData) + + // if the aerospike server version is in the cfg file's + // metadata, don't mark --aerospike-version as required + var aeroVersionRequired bool + if _, ok := metaData[metaKeyAerospikeVersion]; !ok { + if !force { + cmd.MarkFlagRequired("aerospike-version") + aeroVersionRequired = true + } } av, err := cmd.Flags().GetString("aerospike-version") @@ -199,7 +238,7 @@ func newConvertCmd() *cobra.Command { return err } - if !force { + if aeroVersionRequired { if av == "" { return errors.Join(errMissingAerospikeVersion, err) } diff --git a/cmd/utils.go b/cmd/utils.go index e65639f..22286e7 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -2,15 +2,83 @@ package cmd import ( "errors" + "fmt" "path/filepath" "strings" "github.com/aerospike/asconfig/asconf" + "github.com/aerospike/asconfig/asconf/metadata" "github.com/spf13/cobra" "github.com/spf13/pflag" ) +const ( + metaKeyAerospikeVersion = "aerospike-server-version" + metaKeyAsconfigVersion = "asconfig-version" + metaKeyAsadmVersion = "asadm-version" +) + +type metaDataArgs struct { + src []byte + aerospikeVersion string + asconfigVersion string +} + +func genMetaDataText(args metaDataArgs) ([]byte, error) { + + metaHeader := "# *** Aerospike Metadata Generated by Asconfig ***" + + src := args.src + aev := args.aerospikeVersion + asv := args.asconfigVersion + + mdata := map[string]string{} + err := metadata.Unmarshal(src, mdata) + if err != nil { + return nil, err + } + + mdata[metaKeyAerospikeVersion] = aev + mdata[metaKeyAsconfigVersion] = asv + + mtext, err := metadata.Marshal(mdata) + if err != nil { + return nil, err + } + + metaFooter := "# *** End Aerospike Metadata ***" + + mtext = []byte(fmt.Sprintf("%s\n%s%s\n", metaHeader, mtext, metaFooter)) + + return mtext, nil +} + +func getMetaDataItemOptional(src []byte, key string) (string, error) { + mdata := map[string]string{} + err := metadata.Unmarshal(src, mdata) + if err != nil { + return "", err + } + + val := mdata[key] + + return val, nil +} + +func getMetaDataItem(src []byte, key string) (string, error) { + val, err := getMetaDataItemOptional(src, key) + if err != nil { + return "", err + } + + if val == "" { + return "", fmt.Errorf("metadata does not contain %s", key) + } + + return val, nil +} + // common flags func getCommonFlags() *pflag.FlagSet { res := &pflag.FlagSet{} diff --git a/cmd/validate.go b/cmd/validate.go index b193d47..c284f9b 100644 --- a/cmd/validate.go +++ b/cmd/validate.go @@ -48,25 +48,42 @@ func newValidateCmd() *cobra.Command { srcPath = args[0] } - version, err := cmd.Flags().GetString("aerospike-version") + srcFormat, err := getConfFileFormat(srcPath, cmd) if err != nil { return err } - logger.Debugf("Processing flag aerospike-version value=%s", version) + logger.Debugf("Processing flag format value=%v", srcFormat) - srcFormat, err := getConfFileFormat(srcPath, cmd) + fdata, err := os.ReadFile(srcPath) if err != nil { return err } - logger.Debugf("Processing flag format value=%v", srcFormat) + version, err := getMetaDataItemOptional(fdata, metaKeyAerospikeVersion) + if err != nil { + return errors.Join(errMissingAerospikeVersion, err) + } - fdata, err := os.ReadFile(srcPath) + // if the Aerospike server version was not in the file + // metadata, require that it is passed as an argument + if version == "" { + cmd.MarkFlagRequired("aerospike-version") + } + + versionArg, err := cmd.Flags().GetString("aerospike-version") if err != nil { return err } + // the command line --aerospike-version option overrides + // the metadata server version + if versionArg != "" { + version = versionArg + } + + logger.Debugf("Processing flag aerospike-version value=%s", version) + conf, err := asconf.NewAsconf( fdata, srcFormat, @@ -98,9 +115,10 @@ func newValidateCmd() *cobra.Command { } // flags and configuration settings + // --aerospike-version is required unless the server version + // is in the input config file's metadata commonFlags := getCommonFlags() res.Flags().AddFlagSet(commonFlags) - res.MarkFlagRequired("aerospike-version") res.Version = VERSION diff --git a/integration_test.go b/integration_test.go index 1c6108a..096267a 100644 --- a/integration_test.go +++ b/integration_test.go @@ -13,12 +13,15 @@ import ( "os" "os/exec" "path/filepath" + "reflect" "regexp" "strconv" "strings" "testing" "time" + "github.com/aerospike/asconfig/asconf/metadata" + "github.com/aerospike/asconfig/cmd" "github.com/aerospike/asconfig/testutils" "github.com/docker/docker/api/types" @@ -797,7 +800,7 @@ func diff(args ...string) ([]byte, error) { args = append([]string{"diff"}, args...) com := exec.Command(binPath+"/asconfig.test", args...) com.Env = []string{"GOCOVERDIR=" + coveragePath} - diff, err := com.Output() + diff, err := com.CombinedOutput() if err != nil { err = fmt.Errorf("diff failed err: %s, out: %s", err, string(diff)) } @@ -917,6 +920,68 @@ func TestConvertStdin(t *testing.T) { } } +type convertMetaDataTest struct { + expectedMetaData map[string]string + expectedFile string + arguments []string +} + +var convertMetaDataTests = []convertMetaDataTest{ + { + expectedMetaData: map[string]string{ + "aerospike-server-version": "6.2.0.2", + "asconfig-version": cmd.VERSION, + }, + expectedFile: filepath.Join(expectedPath, "all_flash_cluster_cr.conf"), + arguments: []string{"convert", "-a", "6.2.0.2", filepath.Join(sourcePath, "all_flash_cluster_cr.yaml")}, + }, + { + expectedMetaData: map[string]string{ + "aerospike-server-version": "6.4.0.1", + "asconfig-version": cmd.VERSION, + "asadm-version": "2.12.0", + }, + expectedFile: filepath.Join(extraTestPath, "metadata", "metadata.yaml"), + arguments: []string{"convert", filepath.Join(extraTestPath, "metadata", "metadata.conf")}, + }, +} + +func TestConvertMetaData(t *testing.T) { + for _, tf := range convertMetaDataTests { + tmpOutFileName := filepath.Join(destinationPath, "stdinConvertTmp") + + tf.arguments = append(tf.arguments, "-o", tmpOutFileName) + com := exec.Command(binPath+"/asconfig.test", tf.arguments...) + com.Env = []string{"GOCOVERDIR=" + coveragePath} + output, err := com.CombinedOutput() + if err != nil { + t.Errorf("convert failed err: %s, out: %s", err, string(output)) + } + + fileOut, err := os.ReadFile(tmpOutFileName) + if err != nil { + t.Error(err) + } + + metaData := map[string]string{} + err = metadata.Unmarshal(fileOut, metaData) + if err != nil { + t.Errorf("metadata unmarshal err: %s, out: %s", err, string(fileOut)) + } + + if !reflect.DeepEqual(metaData, tf.expectedMetaData) { + t.Errorf("METADATA NOT EQUAL\nCASE: %+v\nACTUAL: %+v\nEXPECTED: %+v\n", tf, metaData, tf.expectedMetaData) + } + + diffFormat := filepath.Ext(tf.expectedFile) + diffFormat = strings.TrimPrefix(diffFormat, ".") + + if _, err := diff(tmpOutFileName, tf.expectedFile, "--format", diffFormat); err != nil { + t.Errorf("\nTESTCASE: %+v\nERR: %+v\n", tf, err) + } + } +} + type validateTest struct { arguments []string source string @@ -932,9 +997,15 @@ var validateTests = []validateTest{ source: filepath.Join(sourcePath, "pmem_cluster_cr.yaml"), }, { - arguments: []string{"validate", "-a", "7.0.0", filepath.Join(extraTestPath, "server64/server64.yaml")}, + arguments: []string{"validate", filepath.Join(extraTestPath, "metadata", "metadata.conf")}, + expectError: false, + expectedResult: "", + source: filepath.Join(extraTestPath, "metadata", "metadata.conf"), + }, + { + arguments: []string{"validate", "-a", "7.0.0", filepath.Join(extraTestPath, "server64", "server64.yaml")}, expectError: true, - source: filepath.Join(extraTestPath, "server64/server64.yaml"), + source: filepath.Join(extraTestPath, "server64", "server64.yaml"), expectedResult: `context: (root).namespaces.0 - description: Additional property memory-size is not allowed, error-type: additional_property_not_allowed context: (root).namespaces.0.storage-engine diff --git a/testdata/cases/metadata/conf-tests.json b/testdata/cases/metadata/conf-tests.json new file mode 100644 index 0000000..6d4dd5e --- /dev/null +++ b/testdata/cases/metadata/conf-tests.json @@ -0,0 +1 @@ +[{"Source":"testdata/cases/metadata/metadata.conf","Destination":"testdata/cases/metadata/metadata-res-.yaml","Expected":"testdata/cases/metadata/metadata.yaml","Arguments":["convert","--aerospike-version","6.4.0.1","--format","asconfig","--output","testdata/cases/metadata/metadata-res-.yaml"],"SkipServerTest":false,"ServerErrorAllowList":null,"ServerImage":"","DockerAuth":{"Username":"","Password":""}}] \ No newline at end of file diff --git a/testdata/cases/metadata/metadata.conf b/testdata/cases/metadata/metadata.conf new file mode 100644 index 0000000..839525c --- /dev/null +++ b/testdata/cases/metadata/metadata.conf @@ -0,0 +1,77 @@ +# *** Aerospike Metadata Generated by Asconfig *** +# aerospike-server-version: 6.4.0.1 +# asconfig-version: 0.12.0 +# asadm-version: 2.12.0 +# *** End Aerospike Metadata *** +service { + user root + group root + pidfile /dummy/file/path1 + proto-fd-max 15000 + + secrets-address-port test_dns_name 4000 127.0.0.1 + secrets-tls-context tlscontext + secrets-uds-path /test/path/socket +} + +logging { + + file /dummy/file/path2 { + context any info + } +} + +network { + service { + address any + port 3000 + } + + heartbeat { + mode multicast + multicast-group 127.0.0.1 + port 9918 + + + + + interval 150 + timeout 10 + } + + fabric { + port 3001 + } + + info { + port 3003 + } +} + +namespace ns1 { + replication-factor 2 + memory-size 4G + + index-type flash { + mount /dummy/mount/point1 /test/mount2 + mounts-high-water-pct 30 + mounts-size-limit 10G + } + + sindex-type flash { + mount /dummy/mount/point3 + mounts-high-water-pct 60 + mounts-size-limit 20000M + } + + storage-engine memory +} + +namespace ns2 { + replication-factor 2 + memory-size 8G + index-type shmem + sindex-type shmem + storage-engine memory +} + diff --git a/testdata/cases/metadata/metadata.yaml b/testdata/cases/metadata/metadata.yaml new file mode 100644 index 0000000..7c5bf29 --- /dev/null +++ b/testdata/cases/metadata/metadata.yaml @@ -0,0 +1,59 @@ +# *** Aerospike Metadata Generated by Asconfig *** +# aerospike-server-version: 6.4.0.1 +# asconfig-version: 0.12.0 +# asadm-version: 2.12.0 +# *** End Aerospike Metadata *** +logging: + - any: info + name: /dummy/file/path2 +namespaces: + - index-type: + type: shmem + memory-size: 8589934592 + name: ns2 + replication-factor: 2 + sindex-type: + type: shmem + storage-engine: + type: memory + - index-type: + mounts: + - /dummy/mount/point1 /test/mount2 + mounts-high-water-pct: 30 + mounts-size-limit: 10737418240 + type: flash + memory-size: 4294967296 + name: ns1 + replication-factor: 2 + sindex-type: + mounts: + - /dummy/mount/point3 + mounts-high-water-pct: 60 + mounts-size-limit: 20971520000 + type: flash + storage-engine: + type: memory +network: + fabric: + port: 3001 + heartbeat: + interval: 150 + mode: multicast + multicast-groups: + - 127.0.0.1 + port: 9918 + timeout: 10 + info: + port: 3003 + service: + addresses: + - any + port: 3000 +service: + group: root + pidfile: /dummy/file/path1 + proto-fd-max: 15000 + secrets-address-port: test_dns_name:4000:127.0.0.1 + secrets-tls-context: tlscontext + secrets-uds-path: /test/path/socket + user: root diff --git a/testdata/cases/metadata/versions.json b/testdata/cases/metadata/versions.json new file mode 100644 index 0000000..e2a14fd --- /dev/null +++ b/testdata/cases/metadata/versions.json @@ -0,0 +1 @@ +{"TestedVersion":"6.4.0.1","OriginallyUsedVersion":"6.4.0.1"} \ No newline at end of file diff --git a/testdata/cases/metadata/yaml-tests.json b/testdata/cases/metadata/yaml-tests.json new file mode 100644 index 0000000..78698a9 --- /dev/null +++ b/testdata/cases/metadata/yaml-tests.json @@ -0,0 +1 @@ +[{"Source":"testdata/cases/metadata/metadata.yaml","Destination":"testdata/cases/metadata/metadata-res-.conf","Expected":"testdata/cases/metadata/metadata.conf","Arguments":["convert","--aerospike-version","6.4.0.1","--format","yaml","--output","testdata/cases/metadata/metadata-res-.conf"],"SkipServerTest":false,"ServerErrorAllowList":null,"ServerImage":"","DockerAuth":{"Username":"","Password":""}}] \ No newline at end of file