From 9c0c75cff7a0d09d47f4d14910852db17cd6a6ae Mon Sep 17 00:00:00 2001 From: Tim Ramlot <42113979+inteon@users.noreply.github.com> Date: Mon, 4 Dec 2023 11:18:01 +0100 Subject: [PATCH] switch to 'yaml.v3' --- fields.go | 151 ++-------------------------------------------- goyaml.v3/yaml.go | 86 +++++++++++++------------- yaml.go | 58 ++++++++++++------ yaml_test.go | 31 +++++----- 4 files changed, 102 insertions(+), 224 deletions(-) diff --git a/fields.go b/fields.go index 0ea28bd..2b91e37 100644 --- a/fields.go +++ b/fields.go @@ -5,7 +5,6 @@ package yaml import ( - "bytes" "encoding" "encoding/json" "reflect" @@ -13,7 +12,6 @@ import ( "strings" "sync" "unicode" - "unicode/utf8" ) // indirect walks down 'value' allocating pointers as needed, @@ -67,9 +65,7 @@ func indirect(value reflect.Value, decodingNull bool) (json.Unmarshaler, encodin // A field represents a single field found in a struct. type field struct { - name string - nameBytes []byte // []byte(name) - equalFold func(s, t []byte) bool // bytes.EqualFold or equivalent + name string tag bool index []int @@ -78,12 +74,6 @@ type field struct { quoted bool } -func fillField(f field) field { - f.nameBytes = []byte(f.name) - f.equalFold = foldFunc(f.nameBytes) - return f -} - // byName sorts field by name, breaking ties with depth, // then breaking ties with "name came from json tag", then // breaking ties with index sequence. @@ -183,14 +173,15 @@ func typeFields(t reflect.Type) []field { if name == "" { name = sf.Name } - fields = append(fields, fillField(field{ + fields = append(fields, field{ name: name, tag: tagged, index: index, typ: ft, omitEmpty: opts.Contains("omitempty"), quoted: opts.Contains("string"), - })) + }) + if count[f.typ] > 1 { // If there were multiple instances, add a second, // so that the annihilation code will see a duplicate. @@ -204,7 +195,7 @@ func typeFields(t reflect.Type) []field { // Record new anonymous struct to explore in next round. nextCount[ft]++ if nextCount[ft] == 1 { - next = append(next, fillField(field{name: ft.Name(), index: index, typ: ft})) + next = append(next, field{name: ft.Name(), index: index, typ: ft}) } } } @@ -333,138 +324,6 @@ func isValidTag(s string) bool { return true } -const ( - caseMask = ^byte(0x20) // Mask to ignore case in ASCII. - kelvin = '\u212a' - smallLongEss = '\u017f' -) - -// foldFunc returns one of four different case folding equivalence -// functions, from most general (and slow) to fastest: -// -// 1) bytes.EqualFold, if the key s contains any non-ASCII UTF-8 -// 2) equalFoldRight, if s contains special folding ASCII ('k', 'K', 's', 'S') -// 3) asciiEqualFold, no special, but includes non-letters (including _) -// 4) simpleLetterEqualFold, no specials, no non-letters. -// -// The letters S and K are special because they map to 3 runes, not just 2: -// - S maps to s and to U+017F 'ſ' Latin small letter long s -// - k maps to K and to U+212A 'K' Kelvin sign -// -// See http://play.golang.org/p/tTxjOc0OGo -// -// The returned function is specialized for matching against s and -// should only be given s. It's not curried for performance reasons. -func foldFunc(s []byte) func(s, t []byte) bool { - nonLetter := false - special := false // special letter - for _, b := range s { - if b >= utf8.RuneSelf { - return bytes.EqualFold - } - upper := b & caseMask - if upper < 'A' || upper > 'Z' { - nonLetter = true - } else if upper == 'K' || upper == 'S' { - // See above for why these letters are special. - special = true - } - } - if special { - return equalFoldRight - } - if nonLetter { - return asciiEqualFold - } - return simpleLetterEqualFold -} - -// equalFoldRight is a specialization of bytes.EqualFold when s is -// known to be all ASCII (including punctuation), but contains an 's', -// 'S', 'k', or 'K', requiring a Unicode fold on the bytes in t. -// See comments on foldFunc. -func equalFoldRight(s, t []byte) bool { - for _, sb := range s { - if len(t) == 0 { - return false - } - tb := t[0] - if tb < utf8.RuneSelf { - if sb != tb { - sbUpper := sb & caseMask - if 'A' <= sbUpper && sbUpper <= 'Z' { - if sbUpper != tb&caseMask { - return false - } - } else { - return false - } - } - t = t[1:] - continue - } - // sb is ASCII and t is not. t must be either kelvin - // sign or long s; sb must be s, S, k, or K. - tr, size := utf8.DecodeRune(t) - switch sb { - case 's', 'S': - if tr != smallLongEss { - return false - } - case 'k', 'K': - if tr != kelvin { - return false - } - default: - return false - } - t = t[size:] - - } - - return len(t) <= 0 -} - -// asciiEqualFold is a specialization of bytes.EqualFold for use when -// s is all ASCII (but may contain non-letters) and contains no -// special-folding letters. -// See comments on foldFunc. -func asciiEqualFold(s, t []byte) bool { - if len(s) != len(t) { - return false - } - for i, sb := range s { - tb := t[i] - if sb == tb { - continue - } - if ('a' <= sb && sb <= 'z') || ('A' <= sb && sb <= 'Z') { - if sb&caseMask != tb&caseMask { - return false - } - } else { - return false - } - } - return true -} - -// simpleLetterEqualFold is a specialization of bytes.EqualFold for -// use when s is all ASCII letters (no underscores, etc) and also -// doesn't contain 'k', 'K', 's', or 'S'. -// See comments on foldFunc. -func simpleLetterEqualFold(s, t []byte) bool { - if len(s) != len(t) { - return false - } - for i, b := range s { - if b&caseMask != t[i]&caseMask { - return false - } - } - return true -} - // tagOptions is the string following a comma in a struct field's "json" // tag, or the empty string. It does not include the leading comma. type tagOptions string diff --git a/goyaml.v3/yaml.go b/goyaml.v3/yaml.go index 8cec6da..068c623 100644 --- a/goyaml.v3/yaml.go +++ b/goyaml.v3/yaml.go @@ -17,8 +17,7 @@ // // Source code and other details for the project are available at GitHub: // -// https://github.com/go-yaml/yaml -// +// https://github.com/go-yaml/yaml package yaml import ( @@ -75,24 +74,24 @@ type Marshaler interface { // // For example: // -// type T struct { -// F int `yaml:"a,omitempty"` -// B int -// } -// var t T -// yaml.Unmarshal([]byte("a: 1\nb: 2"), &t) +// type T struct { +// F int `yaml:"a,omitempty"` +// B int +// } +// var t T +// yaml.Unmarshal([]byte("a: 1\nb: 2"), &t) // // See the documentation of Marshal for the format of tags and a list of // supported tag options. -// func Unmarshal(in []byte, out interface{}) (err error) { return unmarshal(in, out, false) } // A Decoder reads and decodes YAML values from an input stream. type Decoder struct { - parser *parser - knownFields bool + parser *parser + knownFields bool + noUniqueKeys bool } // NewDecoder returns a new decoder that reads from r. @@ -111,6 +110,11 @@ func (dec *Decoder) KnownFields(enable bool) { dec.knownFields = enable } +// UniqueKeys ensures that the keys in the yaml document are unique. +func (dec *Decoder) UniqueKeys(enable bool) { + dec.noUniqueKeys = !enable +} + // Decode reads the next YAML-encoded value from its input // and stores it in the value pointed to by v. // @@ -119,6 +123,7 @@ func (dec *Decoder) KnownFields(enable bool) { func (dec *Decoder) Decode(v interface{}) (err error) { d := newDecoder() d.knownFields = dec.knownFields + d.uniqueKeys = !dec.noUniqueKeys defer handleErr(&err) node := dec.parser.parse() if node == nil { @@ -185,36 +190,35 @@ func unmarshal(in []byte, out interface{}, strict bool) (err error) { // // The field tag format accepted is: // -// `(...) yaml:"[][,[,]]" (...)` +// `(...) yaml:"[][,[,]]" (...)` // // The following flags are currently supported: // -// omitempty Only include the field if it's not set to the zero -// value for the type or to empty slices or maps. -// Zero valued structs will be omitted if all their public -// fields are zero, unless they implement an IsZero -// method (see the IsZeroer interface type), in which -// case the field will be excluded if IsZero returns true. +// omitempty Only include the field if it's not set to the zero +// value for the type or to empty slices or maps. +// Zero valued structs will be omitted if all their public +// fields are zero, unless they implement an IsZero +// method (see the IsZeroer interface type), in which +// case the field will be excluded if IsZero returns true. // -// flow Marshal using a flow style (useful for structs, -// sequences and maps). +// flow Marshal using a flow style (useful for structs, +// sequences and maps). // -// inline Inline the field, which must be a struct or a map, -// causing all of its fields or keys to be processed as if -// they were part of the outer struct. For maps, keys must -// not conflict with the yaml keys of other struct fields. +// inline Inline the field, which must be a struct or a map, +// causing all of its fields or keys to be processed as if +// they were part of the outer struct. For maps, keys must +// not conflict with the yaml keys of other struct fields. // // In addition, if the key is "-", the field is ignored. // // For example: // -// type T struct { -// F int `yaml:"a,omitempty"` -// B int -// } -// yaml.Marshal(&T{B: 2}) // Returns "b: 2\n" -// yaml.Marshal(&T{F: 1}} // Returns "a: 1\nb: 0\n" -// +// type T struct { +// F int `yaml:"a,omitempty"` +// B int +// } +// yaml.Marshal(&T{B: 2}) // Returns "b: 2\n" +// yaml.Marshal(&T{F: 1}} // Returns "a: 1\nb: 0\n" func Marshal(in interface{}) (out []byte, err error) { defer handleErr(&err) e := newEncoder() @@ -358,22 +362,21 @@ const ( // // For example: // -// var person struct { -// Name string -// Address yaml.Node -// } -// err := yaml.Unmarshal(data, &person) -// -// Or by itself: +// var person struct { +// Name string +// Address yaml.Node +// } +// err := yaml.Unmarshal(data, &person) // -// var person Node -// err := yaml.Unmarshal(data, &person) +// Or by itself: // +// var person Node +// err := yaml.Unmarshal(data, &person) type Node struct { // Kind defines whether the node is a document, a mapping, a sequence, // a scalar value, or an alias to another node. The specific data type of // scalar nodes may be obtained via the ShortTag and LongTag methods. - Kind Kind + Kind Kind // Style allows customizing the apperance of the node in the tree. Style Style @@ -421,7 +424,6 @@ func (n *Node) IsZero() bool { n.HeadComment == "" && n.LineComment == "" && n.FootComment == "" && n.Line == 0 && n.Column == 0 } - // LongTag returns the long form of the tag that indicates the data type for // the node. If the Tag field isn't explicitly defined, one will be computed // based on the node properties. diff --git a/yaml.go b/yaml.go index fc10246..ffa7f88 100644 --- a/yaml.go +++ b/yaml.go @@ -24,7 +24,8 @@ import ( "reflect" "strconv" - "sigs.k8s.io/yaml/goyaml.v2" + yamlv2 "sigs.k8s.io/yaml/goyaml.v2" + yamlv3 "sigs.k8s.io/yaml/goyaml.v3" ) // Marshal marshals obj into JSON using stdlib json.Marshal, and then converts JSON to YAML using JSONToYAML (see that method for more reference) @@ -37,6 +38,20 @@ func Marshal(obj interface{}) ([]byte, error) { return JSONToYAML(jsonBytes) } +func yamlv3NonStrict(input []byte, output interface{}) error { + decoder := yamlv3.NewDecoder(bytes.NewReader(input)) + decoder.KnownFields(false) + decoder.UniqueKeys(false) + return decoder.Decode(output) +} + +func yamlv3Strict(input []byte, output interface{}) error { + decoder := yamlv3.NewDecoder(bytes.NewReader(input)) + decoder.KnownFields(true) + decoder.UniqueKeys(true) + return decoder.Decode(output) +} + // JSONOpt is a decoding option for decoding from JSON format. type JSONOpt func(*json.Decoder) *json.Decoder @@ -53,7 +68,7 @@ type JSONOpt func(*json.Decoder) *json.Decoder // - YAML non-string keys, e.g. ints, bools and floats, are converted to strings implicitly during the YAML to JSON conversion process. // - There are no compatibility guarantees for returned error values. func Unmarshal(yamlBytes []byte, obj interface{}, opts ...JSONOpt) error { - return unmarshal(yamlBytes, obj, yaml.Unmarshal, opts...) + return unmarshal(yamlBytes, obj, yamlv3NonStrict, opts...) } // UnmarshalStrict is similar to Unmarshal (please read its documentation for reference), with the following exceptions: @@ -61,7 +76,7 @@ func Unmarshal(yamlBytes []byte, obj interface{}, opts ...JSONOpt) error { // - Duplicate fields in an object yield an error. This is according to the YAML specification. // - If obj, or any of its recursive children, is a struct, presence of fields in the serialized data unknown to the struct will yield an error. func UnmarshalStrict(yamlBytes []byte, obj interface{}, opts ...JSONOpt) error { - return unmarshal(yamlBytes, obj, yaml.UnmarshalStrict, append(opts, DisallowUnknownFields)...) + return unmarshal(yamlBytes, obj, yamlv3Strict, append(opts, DisallowUnknownFields)...) } // unmarshal unmarshals the given YAML byte stream into the given interface, @@ -111,13 +126,13 @@ func JSONToYAML(j []byte) ([]byte, error) { // etc.) when unmarshalling to interface{}, it just picks float64 // universally. go-yaml does go through the effort of picking the right // number type, so we can preserve number type throughout this process. - err := yaml.Unmarshal(j, &jsonObj) + err := yamlv3.Unmarshal(j, &jsonObj) if err != nil { return nil, err } // Marshal this object into YAML. - yamlBytes, err := yaml.Marshal(jsonObj) + yamlBytes, err := yamlv3.Marshal(jsonObj) if err != nil { return nil, err } @@ -144,13 +159,13 @@ func JSONToYAML(j []byte) ([]byte, error) { // - Unlike Unmarshal, all integers, up to 64 bits, are preserved during this round-trip. // - There are no compatibility guarantees for returned error values. func YAMLToJSON(y []byte) ([]byte, error) { - return yamlToJSONTarget(y, nil, yaml.Unmarshal) + return yamlToJSONTarget(y, nil, yamlv3NonStrict) } // YAMLToJSONStrict is like YAMLToJSON but enables strict YAML decoding, // returning an error on any duplicate field names. func YAMLToJSONStrict(y []byte) ([]byte, error) { - return yamlToJSONTarget(y, nil, yaml.UnmarshalStrict) + return yamlToJSONTarget(y, nil, yamlv3Strict) } func yamlToJSONTarget(yamlBytes []byte, jsonTarget *reflect.Value, unmarshalFn func([]byte, interface{}) error) ([]byte, error) { @@ -196,6 +211,15 @@ func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (in } } + // Transform map[string]interface{} into map[interface{}]interface{} + if stringMap, ok := yamlObj.(map[string]interface{}); ok { + interfaceMap := make(map[interface{}]interface{}) + for k, v := range stringMap { + interfaceMap[k] = v + } + yamlObj = interfaceMap + } + // If yamlObj is a number or a boolean, check if jsonTarget is a string - // if so, coerce. Else return normal. // If yamlObj is a map or array, find the field that each key is @@ -227,7 +251,7 @@ func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (in // Stolen from go-yaml to use the same conversion to string as // the go-yaml library uses to convert float to string when // Marshaling. - s := strconv.FormatFloat(typedKey, 'g', -1, 32) + s := strconv.FormatFloat(typedKey, 'g', -1, 64) switch s { case "+Inf": s = ".inf" @@ -256,20 +280,14 @@ func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (in if jsonTarget != nil { t := *jsonTarget if t.Kind() == reflect.Struct { - keyBytes := []byte(keyString) // Find the field that the JSON library would use. var f *field fields := cachedTypeFields(t.Type()) for i := range fields { - ff := &fields[i] - if bytes.Equal(ff.nameBytes, keyBytes) { - f = ff + f = &fields[i] + if f.name == keyString { break } - // Do case-insensitive comparison. - if f == nil && ff.equalFold(ff.nameBytes, keyBytes) { - f = ff - } } if f != nil { // Find the reflect.Value of the most preferential @@ -339,7 +357,7 @@ func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (in case int64: s = strconv.FormatInt(typedVal, 10) case float64: - s = strconv.FormatFloat(typedVal, 'g', -1, 32) + s = strconv.FormatFloat(typedVal, 'g', -1, 64) case uint64: s = strconv.FormatUint(typedVal, 10) case bool: @@ -370,13 +388,13 @@ func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (in // Big int/int64/uint64 do not lose precision as in the json-yaml roundtripping case. // // string, bool and any other types are unchanged. -func JSONObjectToYAMLObject(j map[string]interface{}) yaml.MapSlice { +func JSONObjectToYAMLObject(j map[string]interface{}) yamlv2.MapSlice { if len(j) == 0 { return nil } - ret := make(yaml.MapSlice, 0, len(j)) + ret := make(yamlv2.MapSlice, 0, len(j)) for k, v := range j { - ret = append(ret, yaml.MapItem{Key: k, Value: jsonToYAMLValue(v)}) + ret = append(ret, yamlv2.MapItem{Key: k, Value: jsonToYAMLValue(v)}) } return ret } diff --git a/yaml_test.go b/yaml_test.go index bcabc79..c04810e 100644 --- a/yaml_test.go +++ b/yaml_test.go @@ -175,15 +175,13 @@ func testYAMLToJSON(t *testing.T, f testYAMLToJSONFunc, tests map[string]yamlToJ type MarshalTest struct { A string B int64 - // Would like to test float64, but it's not supported in go-yaml. - // (See https://github.com/go-yaml/yaml/issues/83.) - C float32 + C float64 } func TestMarshal(t *testing.T) { - f32String := strconv.FormatFloat(math.MaxFloat32, 'g', -1, 32) - s := MarshalTest{"a", math.MaxInt64, math.MaxFloat32} - e := []byte(fmt.Sprintf("A: a\nB: %d\nC: %s\n", math.MaxInt64, f32String)) + f64String := strconv.FormatFloat(math.MaxFloat64, 'g', -1, 64) + s := MarshalTest{"a", math.MaxInt64, math.MaxFloat64} + e := []byte(fmt.Sprintf("A: a\nB: %d\nC: %s\n", math.MaxInt64, f64String)) y, err := Marshal(s) if err != nil { @@ -296,12 +294,12 @@ func TestUnmarshal(t *testing.T) { "tagged casematched boolean key (yes)": { encoded: []byte("Yes: test"), decodeInto: new(UnmarshalTaggedStruct), - decoded: UnmarshalTaggedStruct{TrueLower: "test"}, + decoded: UnmarshalTaggedStruct{YesUpper: "test"}, }, "tagged non-casematched boolean key (yes)": { encoded: []byte("yes: test"), decodeInto: new(UnmarshalTaggedStruct), - decoded: UnmarshalTaggedStruct{TrueLower: "test"}, + decoded: UnmarshalTaggedStruct{YesLower: "test"}, }, "tagged integer key": { encoded: []byte("3: test"), @@ -343,7 +341,7 @@ func TestUnmarshal(t *testing.T) { "boolean value (no) into string field": { encoded: []byte("a: no"), decodeInto: new(UnmarshalStruct), - decoded: UnmarshalStruct{A: "false"}, + decoded: UnmarshalStruct{A: "no"}, }, // decode into complex fields @@ -390,7 +388,7 @@ func TestUnmarshal(t *testing.T) { encoded: []byte("Yes:"), decodeInto: new(map[string]struct{}), decoded: map[string]struct{}{ - "true": {}, + "Yes": {}, }, }, "string map: decode integer key": { @@ -639,8 +637,8 @@ func TestYAMLToJSON(t *testing.T) { }, "boolean value (no)": { yaml: "t: no\n", - json: `{"t":false}`, - yamlReverseOverwrite: strPtr("t: false\n"), + json: `{"t":"no"}`, + yamlReverseOverwrite: strPtr("t: \"no\"\n"), }, "integer value (2^53 + 1)": { yaml: "t: 9007199254740993\n", @@ -653,8 +651,9 @@ func TestYAMLToJSON(t *testing.T) { yamlReverseOverwrite: strPtr("t: 1e+36\n"), }, "line-wrapped string value": { - yaml: "t: this is very long line with spaces and it must be longer than 80 so we will repeat\n that it must be longer that 80\n", - json: `{"t":"this is very long line with spaces and it must be longer than 80 so we will repeat that it must be longer that 80"}`, + yaml: "t: this is very long line with spaces and it must be longer than 80 so we will repeat\n that it must be longer that 80\n", + json: `{"t":"this is very long line with spaces and it must be longer than 80 so we will repeat that it must be longer that 80"}`, + yamlReverseOverwrite: strPtr(`t: this is very long line with spaces and it must be longer than 80 so we will repeat that it must be longer that 80` + "\n"), }, "empty yaml value": { yaml: "t: ", @@ -668,8 +667,8 @@ func TestYAMLToJSON(t *testing.T) { }, "boolean key (no)": { yaml: "no: a", - json: `{"false":"a"}`, - yamlReverseOverwrite: strPtr("\"false\": a\n"), + json: `{"no":"a"}`, + yamlReverseOverwrite: strPtr("\"no\": a\n"), }, "integer key": { yaml: "1: a",