From 6cacff13cbd26020835e9e9d82859377fc474856 Mon Sep 17 00:00:00 2001 From: Islam Aliev Date: Thu, 21 Nov 2024 22:39:36 +0100 Subject: [PATCH] refactor: Add unified JSON interface (#3265) ## Relevant issue(s) Resolves #3264 ## Description Introduces a common interface for JSON values that would allow later to do better processing of json objects. --- client/document.go | 70 +-- client/document_test.go | 6 +- client/json.go | 433 +++++++++++++++ client/json_test.go | 519 ++++++++++++++++++ client/normal_new.go | 2 +- client/normal_scalar.go | 17 +- client/normal_value.go | 2 +- client/normal_value_test.go | 55 +- client/normal_void.go | 2 +- .../i3265-unified-json-types.md | 4 + .../field_kinds/field_kind_json_test.go | 20 +- tests/integration/query/json/with_ge_test.go | 4 +- tests/integration/query/json/with_gt_test.go | 2 +- tests/integration/query/json/with_lt_test.go | 2 +- .../integration/query/json/with_nlike_test.go | 8 +- tests/integration/utils.go | 7 +- 16 files changed, 987 insertions(+), 166 deletions(-) create mode 100644 client/json.go create mode 100644 client/json_test.go create mode 100644 docs/data_format_changes/i3265-unified-json-types.md diff --git a/client/document.go b/client/document.go index 4abadcac52..b4ae927522 100644 --- a/client/document.go +++ b/client/document.go @@ -358,11 +358,11 @@ func validateFieldSchema(val any, field FieldDefinition) (NormalValue, error) { return NewNormalNillableIntArray(v), nil case FieldKind_NILLABLE_JSON: - v, err := getJSON(val) + v, err := NewJSON(val) if err != nil { return nil, err } - return NewNormalJSON(&JSON{v}), nil + return NewNormalJSON(v), nil } return nil, NewErrUnhandledType("FieldKind", field.Kind) @@ -438,72 +438,6 @@ func getDateTime(v any) (time.Time, error) { return time.Parse(time.RFC3339, s) } -// getJSON converts the given value to a valid JSON value. -// -// If the value is of type *fastjson.Value it needs to be -// manually parsed. All other values are valid JSON. -func getJSON(v any) (any, error) { - val, ok := v.(*fastjson.Value) - if !ok { - return v, nil - } - switch val.Type() { - case fastjson.TypeArray: - arr, err := val.Array() - if err != nil { - return nil, err - } - out := make([]any, len(arr)) - for i, v := range arr { - c, err := getJSON(v) - if err != nil { - return nil, err - } - out[i] = c - } - return out, nil - - case fastjson.TypeObject: - obj, err := val.Object() - if err != nil { - return nil, err - } - out := make(map[string]any) - obj.Visit(func(key []byte, v *fastjson.Value) { - c, e := getJSON(v) - out[string(key)] = c - err = errors.Join(err, e) - }) - return out, err - - case fastjson.TypeFalse: - return false, nil - - case fastjson.TypeTrue: - return true, nil - - case fastjson.TypeNumber: - out, err := val.Int64() - if err == nil { - return out, nil - } - return val.Float64() - - case fastjson.TypeString: - out, err := val.StringBytes() - if err != nil { - return nil, err - } - return string(out), nil - - case fastjson.TypeNull: - return nil, nil - - default: - return nil, NewErrInvalidJSONPayload(v) - } -} - func getArray[T any]( v any, typeGetter func(any) (T, error), diff --git a/client/document_test.go b/client/document_test.go index b74af54b27..4f4dc9aa48 100644 --- a/client/document_test.go +++ b/client/document_test.go @@ -194,13 +194,13 @@ func TestNewFromJSON_WithValidJSONFieldValue_NoError(t *testing.T) { assert.Equal(t, doc.values[doc.fields["Age"]].IsDocument(), false) assert.Equal(t, doc.values[doc.fields["Custom"]].Value(), map[string]any{ "string": "maple", - "int": int64(260), + "int": float64(260), "float": float64(3.14), "false": false, "true": true, "null": nil, - "array": []any{"one", int64(1)}, - "object": map[string]any{"one": int64(1)}, + "array": []any{"one", float64(1)}, + "object": map[string]any{"one": float64(1)}, }) assert.Equal(t, doc.values[doc.fields["Custom"]].IsDocument(), false) } diff --git a/client/json.go b/client/json.go new file mode 100644 index 0000000000..23a23de2b1 --- /dev/null +++ b/client/json.go @@ -0,0 +1,433 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package client + +import ( + "encoding/json" + "io" + + "github.com/valyala/fastjson" + "golang.org/x/exp/constraints" +) + +// JSON represents a JSON value that can be any valid JSON type: object, array, number, string, boolean, or null. +// It provides type-safe access to the underlying value through various accessor methods. +type JSON interface { + json.Marshaler + // Array returns the value as a JSON array along with a boolean indicating if the value is an array. + // Returns nil and false if the value is not an array. + Array() ([]JSON, bool) + + // Object returns the value as a JSON object along with a boolean indicating if the value is an object. + // Returns nil and false if the value is not an object. + Object() (map[string]JSON, bool) + + // Number returns the value as a number along with a boolean indicating if the value is a number. + // Returns 0 and false if the value is not a number. + Number() (float64, bool) + + // String returns the value as a string along with a boolean indicating if the value is a string. + // Returns empty string and false if the value is not a string. + String() (string, bool) + + // Bool returns the value as a boolean along with a boolean indicating if the value is a boolean. + // Returns false and false if the value is not a boolean. + Bool() (bool, bool) + + // IsNull returns true if the value is null, false otherwise. + IsNull() bool + + // Value returns the value that JSON represents. + // The type will be one of: map[string]JSON, []JSON, float64, string, bool, or nil. + Value() any + + // Unwrap returns the underlying value with all nested JSON values unwrapped. + // For objects and arrays, this recursively unwraps all nested JSON values. + Unwrap() any + + // Marshal writes the JSON value to the writer. + // Returns an error if marshaling fails. + Marshal(w io.Writer) error +} + +type jsonVoid struct{} + +func (v jsonVoid) Object() (map[string]JSON, bool) { + return nil, false +} + +func (v jsonVoid) Array() ([]JSON, bool) { + return nil, false +} + +func (v jsonVoid) Number() (float64, bool) { + return 0, false +} + +func (v jsonVoid) String() (string, bool) { + return "", false +} + +func (v jsonVoid) Bool() (bool, bool) { + return false, false +} + +func (v jsonVoid) IsNull() bool { + return false +} + +type jsonBase[T any] struct { + jsonVoid + val T +} + +func (v jsonBase[T]) Value() any { + return v.val +} + +func (v jsonBase[T]) Unwrap() any { + return v.val +} + +func (v jsonBase[T]) Marshal(w io.Writer) error { + return json.NewEncoder(w).Encode(v.val) +} + +func (v jsonBase[T]) MarshalJSON() ([]byte, error) { + return json.Marshal(v.val) +} + +type jsonObject struct { + jsonBase[map[string]JSON] +} + +var _ JSON = jsonObject{} + +func (obj jsonObject) Object() (map[string]JSON, bool) { + return obj.val, true +} + +func (obj jsonObject) MarshalJSON() ([]byte, error) { + return json.Marshal(obj.val) +} + +func (obj jsonObject) Unwrap() any { + result := make(map[string]any, len(obj.jsonBase.val)) + for k, v := range obj.val { + result[k] = v.Unwrap() + } + return result +} + +type jsonArray struct { + jsonBase[[]JSON] +} + +var _ JSON = jsonArray{} + +func (arr jsonArray) Array() ([]JSON, bool) { + return arr.val, true +} + +func (arr jsonArray) MarshalJSON() ([]byte, error) { + return json.Marshal(arr.val) +} + +func (arr jsonArray) Unwrap() any { + result := make([]any, len(arr.jsonBase.val)) + for i := range arr.val { + result[i] = arr.val[i].Unwrap() + } + return result +} + +type jsonNumber struct { + jsonBase[float64] +} + +var _ JSON = jsonNumber{} + +func (n jsonNumber) Number() (float64, bool) { + return n.val, true +} + +func (n jsonNumber) MarshalJSON() ([]byte, error) { + return json.Marshal(n.val) +} + +type jsonString struct { + jsonBase[string] +} + +var _ JSON = jsonString{} + +func (s jsonString) String() (string, bool) { + return s.val, true +} + +func (s jsonString) MarshalJSON() ([]byte, error) { + return json.Marshal(s.val) +} + +type jsonBool struct { + jsonBase[bool] +} + +var _ JSON = jsonBool{} + +func (b jsonBool) Bool() (bool, bool) { + return b.val, true +} + +func (b jsonBool) MarshalJSON() ([]byte, error) { + return json.Marshal(b.val) +} + +type jsonNull struct { + jsonVoid +} + +var _ JSON = jsonNull{} + +func (n jsonNull) IsNull() bool { + return true +} + +func (n jsonNull) Value() any { + return nil +} + +func (n jsonNull) Unwrap() any { + return nil +} + +func (n jsonNull) Marshal(w io.Writer) error { + return json.NewEncoder(w).Encode(nil) +} + +func (n jsonNull) MarshalJSON() ([]byte, error) { + return json.Marshal(nil) +} + +func newJSONObject(val map[string]JSON) JSON { + return jsonObject{jsonBase[map[string]JSON]{val: val}} +} + +func newJSONArray(val []JSON) JSON { + return jsonArray{jsonBase[[]JSON]{val: val}} +} + +func newJSONNumber(val float64) JSON { + return jsonNumber{jsonBase[float64]{val: val}} +} + +func newJSONString(val string) JSON { + return jsonString{jsonBase[string]{val: val}} +} + +func newJSONBool(val bool) JSON { + return jsonBool{jsonBase[bool]{val: val}} +} + +func newJSONNull() JSON { + return jsonNull{} +} + +// ParseJSONBytes parses the given JSON bytes into a JSON value. +// Returns error if the input is not valid JSON. +func ParseJSONBytes(data []byte) (JSON, error) { + var p fastjson.Parser + v, err := p.ParseBytes(data) + if err != nil { + return nil, err + } + return NewJSONFromFastJSON(v), nil +} + +// ParseJSONString parses the given JSON string into a JSON value. +// Returns error if the input is not valid JSON. +func ParseJSONString(data string) (JSON, error) { + // we could have called ParseJSONBytes([]byte(data), but this would copy the string to a byte slice. + // fastjson.Parser.ParseBytes casts the bytes slice to a string internally, so we can avoid the extra copy. + var p fastjson.Parser + v, err := p.Parse(data) + if err != nil { + return nil, err + } + return NewJSONFromFastJSON(v), nil +} + +// NewJSON creates a JSON value from a Go value. +// The Go value must be one of: +// - nil (becomes JSON null) +// - *fastjson.Value +// - string +// - map[string]any +// - bool +// - numeric types (int8 through int64, uint8 through uint64, float32, float64) +// - slice of any above type +// - []any +// Returns error if the input cannot be converted to JSON. +func NewJSON(v any) (JSON, error) { + if v == nil { + return newJSONNull(), nil + } + switch val := v.(type) { + case *fastjson.Value: + return NewJSONFromFastJSON(val), nil + case string: + return newJSONString(val), nil + case map[string]any: + return NewJSONFromMap(val) + case bool: + return newJSONBool(val), nil + case int8: + return newJSONNumber(float64(val)), nil + case int16: + return newJSONNumber(float64(val)), nil + case int32: + return newJSONNumber(float64(val)), nil + case int64: + return newJSONNumber(float64(val)), nil + case int: + return newJSONNumber(float64(val)), nil + case uint8: + return newJSONNumber(float64(val)), nil + case uint16: + return newJSONNumber(float64(val)), nil + case uint32: + return newJSONNumber(float64(val)), nil + case uint64: + return newJSONNumber(float64(val)), nil + case uint: + return newJSONNumber(float64(val)), nil + case float32: + return newJSONNumber(float64(val)), nil + case float64: + return newJSONNumber(val), nil + + case []bool: + return newJSONBoolArray(val), nil + case []int8: + return newJSONNumberArray(val), nil + case []int16: + return newJSONNumberArray(val), nil + case []int32: + return newJSONNumberArray(val), nil + case []int64: + return newJSONNumberArray(val), nil + case []int: + return newJSONNumberArray(val), nil + case []uint8: + return newJSONNumberArray(val), nil + case []uint16: + return newJSONNumberArray(val), nil + case []uint32: + return newJSONNumberArray(val), nil + case []uint64: + return newJSONNumberArray(val), nil + case []uint: + return newJSONNumberArray(val), nil + case []float32: + return newJSONNumberArray(val), nil + case []float64: + return newJSONNumberArray(val), nil + case []string: + return newJSONStringArray(val), nil + + case []any: + return newJsonArrayFromAnyArray(val) + } + + return nil, NewErrInvalidJSONPayload(v) +} + +func newJsonArrayFromAnyArray(arr []any) (JSON, error) { + result := make([]JSON, len(arr)) + for i := range arr { + jsonVal, err := NewJSON(arr[i]) + if err != nil { + return nil, err + } + result[i] = jsonVal + } + return newJSONArray(result), nil +} + +func newJSONBoolArray(v []bool) JSON { + arr := make([]JSON, len(v)) + for i := range v { + arr[i] = newJSONBool(v[i]) + } + return newJSONArray(arr) +} + +func newJSONNumberArray[T constraints.Integer | constraints.Float](v []T) JSON { + arr := make([]JSON, len(v)) + for i := range v { + arr[i] = newJSONNumber(float64(v[i])) + } + return newJSONArray(arr) +} + +func newJSONStringArray(v []string) JSON { + arr := make([]JSON, len(v)) + for i := range v { + arr[i] = newJSONString(v[i]) + } + return newJSONArray(arr) +} + +// NewJSONFromFastJSON creates a JSON value from a fastjson.Value. +func NewJSONFromFastJSON(v *fastjson.Value) JSON { + switch v.Type() { + case fastjson.TypeObject: + fastObj := v.GetObject() + obj := make(map[string]JSON, fastObj.Len()) + fastObj.Visit(func(k []byte, v *fastjson.Value) { + obj[string(k)] = NewJSONFromFastJSON(v) + }) + return newJSONObject(obj) + case fastjson.TypeArray: + fastArr := v.GetArray() + arr := make([]JSON, len(fastArr)) + for i := range fastArr { + arr[i] = NewJSONFromFastJSON(fastArr[i]) + } + return newJSONArray(arr) + case fastjson.TypeNumber: + return newJSONNumber(v.GetFloat64()) + case fastjson.TypeString: + return newJSONString(string(v.GetStringBytes())) + case fastjson.TypeTrue: + return newJSONBool(true) + case fastjson.TypeFalse: + return newJSONBool(false) + case fastjson.TypeNull: + return newJSONNull() + } + return nil +} + +// NewJSONFromMap creates a JSON object from a map[string]any. +// The map values must be valid Go values that can be converted to JSON. +// Returns error if any map value cannot be converted to JSON. +func NewJSONFromMap(data map[string]any) (JSON, error) { + obj := make(map[string]JSON, len(data)) + for k, v := range data { + jsonVal, err := NewJSON(v) + if err != nil { + return nil, err + } + obj[k] = jsonVal + } + return newJSONObject(obj), nil +} diff --git a/client/json_test.go b/client/json_test.go new file mode 100644 index 0000000000..9ac4d3b781 --- /dev/null +++ b/client/json_test.go @@ -0,0 +1,519 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package client + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "github.com/valyala/fastjson" +) + +func TestParseJSONAndMarshal_WithValidInput_ShouldMarshal(t *testing.T) { + tests := []struct { + name string + fromFunc func(string) (JSON, error) + }{ + { + name: "FromBytes", + fromFunc: func(data string) (JSON, error) { return ParseJSONBytes([]byte(data)) }, + }, + { + name: "FromString", + fromFunc: ParseJSONString, + }, + { + name: "FromFastJSON", + fromFunc: func(data string) (JSON, error) { + var p fastjson.Parser + v, err := p.Parse(data) + if err != nil { + return nil, err + } + return NewJSONFromFastJSON(v), nil + }, + }, + { + name: "FromMap", + fromFunc: func(data string) (JSON, error) { + var result map[string]any + if err := json.Unmarshal([]byte(data), &result); err != nil { + return nil, err + } + return NewJSONFromMap(result) + }, + }, + } + + data := `{"key1": "value1", "key2": 2, "key3": true, "key4": null, "key5": ["item1", 2, false]}` + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + jsonObj, err := tt.fromFunc(data) + require.NoError(t, err, "fromFunc failed with error %v", err) + + var buf bytes.Buffer + err = jsonObj.Marshal(&buf) + require.NoError(t, err, "jsonObj.Marshal(&buf) failed with error %v", err) + + actualStr := strings.ReplaceAll(buf.String(), "\n", "") + expectedStr := strings.ReplaceAll(data, " ", "") + require.Equal(t, actualStr, expectedStr, "Expected %s, got %s", expectedStr, actualStr) + + rawJSON, err := jsonObj.MarshalJSON() + require.NoError(t, err, "jsonObj.MarshalJSON() failed with error %v", err) + actualStr = strings.ReplaceAll(string(rawJSON), "\n", "") + require.Equal(t, actualStr, expectedStr, "Expected %s, got %s", expectedStr, actualStr) + }) + } +} + +func TestNewJSONAndMarshal_WithInvalidInput_ShouldFail(t *testing.T) { + tests := []struct { + name string + fromFunc func(string) (JSON, error) + }{ + { + name: "FromBytes", + fromFunc: func(data string) (JSON, error) { return ParseJSONBytes([]byte(data)) }, + }, + { + name: "FromString", + fromFunc: ParseJSONString, + }, + { + name: "FromMap", + fromFunc: func(data string) (JSON, error) { + var result map[string]any + if err := json.Unmarshal([]byte(data), &result); err != nil { + return nil, err + } + return NewJSONFromMap(result) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := tt.fromFunc(`{"key1": "value1}`) + require.Error(t, err, "Expected error, but got nil") + }) + } +} + +func TestNewJSONFomString_WithInvalidInput_Error(t *testing.T) { + _, err := ParseJSONString("str") + require.Error(t, err, "Expected error, but got nil") +} + +func TestJSONObject_Methods_ShouldWorkAsExpected(t *testing.T) { + m := map[string]JSON{ + "key": newJSONString("value"), + "nested": newJSONObject(map[string]JSON{ + "inner": newJSONNumber(42), + "array": newJSONArray([]JSON{newJSONString("test"), newJSONBool(true)}), + }), + } + obj := newJSONObject(m) + expectedUnwrapped := map[string]any{ + "key": "value", + "nested": map[string]any{ + "inner": float64(42), + "array": []any{"test", true}, + }, + } + + // Positive tests + val, ok := obj.Object() + require.True(t, ok) + require.Equal(t, m, val) + require.Equal(t, m, obj.Value()) + require.Equal(t, expectedUnwrapped, obj.Unwrap()) + + // Negative tests + _, ok = obj.Array() + require.False(t, ok) + _, ok = obj.Number() + require.False(t, ok) + _, ok = obj.String() + require.False(t, ok) + _, ok = obj.Bool() + require.False(t, ok) + require.False(t, obj.IsNull()) +} + +func TestJSONArray_Methods_ShouldWorkAsExpected(t *testing.T) { + arr := []JSON{ + newJSONString("item1"), + newJSONObject(map[string]JSON{ + "key": newJSONString("value"), + "num": newJSONNumber(42), + }), + newJSONNumber(2), + } + jsonArr := newJSONArray(arr) + expectedUnwrapped := []any{ + "item1", + map[string]any{ + "key": "value", + "num": float64(42), + }, + float64(2), + } + + // Positive tests + val, ok := jsonArr.Array() + require.True(t, ok) + require.Equal(t, arr, val) + require.Equal(t, arr, jsonArr.Value()) + require.Equal(t, expectedUnwrapped, jsonArr.Unwrap()) + + // Negative tests + _, ok = jsonArr.Object() + require.False(t, ok) + _, ok = jsonArr.Number() + require.False(t, ok) + _, ok = jsonArr.String() + require.False(t, ok) + _, ok = jsonArr.Bool() + require.False(t, ok) + require.False(t, jsonArr.IsNull()) +} + +func TestJSONNumber_Methods_ShouldWorkAsExpected(t *testing.T) { + num := newJSONNumber(2.5) + expected := 2.5 + + // Positive tests + val, ok := num.Number() + require.True(t, ok) + require.Equal(t, expected, val) + require.Equal(t, expected, num.Value()) + require.Equal(t, expected, num.Unwrap()) + + // Negative tests + _, ok = num.Object() + require.False(t, ok) + _, ok = num.Array() + require.False(t, ok) + _, ok = num.String() + require.False(t, ok) + _, ok = num.Bool() + require.False(t, ok) + require.False(t, num.IsNull()) +} + +func TestJSONString_Methods_ShouldWorkAsExpected(t *testing.T) { + str := newJSONString("value") + expected := "value" + + // Positive tests + val, ok := str.String() + require.True(t, ok) + require.Equal(t, expected, val) + require.Equal(t, expected, str.Value()) + require.Equal(t, expected, str.Unwrap()) + + // Negative tests + _, ok = str.Object() + require.False(t, ok) + _, ok = str.Array() + require.False(t, ok) + _, ok = str.Number() + require.False(t, ok) + _, ok = str.Bool() + require.False(t, ok) + require.False(t, str.IsNull()) +} + +func TestJSONBool_Methods_ShouldWorkAsExpected(t *testing.T) { + b := newJSONBool(true) + expected := true + + // Positive tests + val, ok := b.Bool() + require.True(t, ok) + require.Equal(t, expected, val) + require.Equal(t, expected, b.Value()) + require.Equal(t, expected, b.Unwrap()) + + // Negative tests + _, ok = b.Object() + require.False(t, ok) + _, ok = b.Array() + require.False(t, ok) + _, ok = b.Number() + require.False(t, ok) + _, ok = b.String() + require.False(t, ok) + require.False(t, b.IsNull()) +} + +func TestJSONNull_Methods_ShouldWorkAsExpected(t *testing.T) { + null := newJSONNull() + + // Positive tests + require.True(t, null.IsNull()) + require.Nil(t, null.Value()) + require.Nil(t, null.Unwrap()) + + // Negative tests + _, ok := null.Object() + require.False(t, ok) + _, ok = null.Array() + require.False(t, ok) + _, ok = null.Number() + require.False(t, ok) + _, ok = null.String() + require.False(t, ok) + _, ok = null.Bool() + require.False(t, ok) +} + +func TestNewJSONAndMarshalJSON(t *testing.T) { + tests := []struct { + name string + input any + expected JSON + expectedJSON string + expectError bool + }{ + { + name: "Nil", + input: nil, + expected: newJSONNull(), + expectedJSON: "null", + }, + { + name: "FastJSON", + input: fastjson.MustParse(`{"key": "value"}`), + expected: newJSONObject(map[string]JSON{"key": newJSONString("value")}), + expectedJSON: `{"key":"value"}`, + }, + { + name: "Map", + input: map[string]any{"key": "value"}, + expected: newJSONObject(map[string]JSON{"key": newJSONString("value")}), + expectedJSON: `{"key":"value"}`, + }, + { + name: "Bool", + input: true, + expected: newJSONBool(true), + expectedJSON: "true", + }, + { + name: "String", + input: "str", + expected: newJSONString("str"), + expectedJSON: `"str"`, + }, + { + name: "Int8", + input: int8(42), + expected: newJSONNumber(42), + expectedJSON: "42", + }, + { + name: "Int16", + input: int16(42), + expected: newJSONNumber(42), + expectedJSON: "42", + }, + { + name: "Int32", + input: int32(42), + expected: newJSONNumber(42), + expectedJSON: "42", + }, + { + name: "Int64", + input: int64(42), + expected: newJSONNumber(42), + expectedJSON: "42", + }, + { + name: "Int", + input: 42, + expected: newJSONNumber(42), + expectedJSON: "42", + }, + { + name: "Uint8", + input: uint8(42), + expected: newJSONNumber(42), + expectedJSON: "42", + }, + { + name: "Uint16", + input: uint16(42), + expected: newJSONNumber(42), + expectedJSON: "42", + }, + { + name: "Uint32", + input: uint32(42), + expected: newJSONNumber(42), + expectedJSON: "42", + }, + { + name: "Uint64", + input: uint64(42), + expected: newJSONNumber(42), + expectedJSON: "42", + }, + { + name: "Uint", + input: uint(42), + expected: newJSONNumber(42), + expectedJSON: "42", + }, + { + name: "Float32", + input: float32(2.5), + expected: newJSONNumber(2.5), + expectedJSON: "2.5", + }, + { + name: "Float64", + input: float64(2.5), + expected: newJSONNumber(2.5), + expectedJSON: "2.5", + }, + { + name: "BoolArray", + input: []bool{true, false}, + expected: newJSONArray([]JSON{newJSONBool(true), newJSONBool(false)}), + expectedJSON: "[true,false]", + }, + { + name: "StringArray", + input: []string{"a", "b", "c"}, + expected: newJSONArray([]JSON{newJSONString("a"), newJSONString("b"), newJSONString("c")}), + expectedJSON: `["a","b","c"]`, + }, + { + name: "AnyArray", + input: []any{"a", 1, true}, + expected: newJSONArray([]JSON{newJSONString("a"), newJSONNumber(1), newJSONBool(true)}), + expectedJSON: `["a",1,true]`, + }, + { + name: "Int8Array", + input: []int8{1, 2, 3}, + expected: newJSONArray([]JSON{newJSONNumber(1), newJSONNumber(2), newJSONNumber(3)}), + expectedJSON: "[1,2,3]", + }, + { + name: "Int16Array", + input: []int16{1, 2, 3}, + expected: newJSONArray([]JSON{newJSONNumber(1), newJSONNumber(2), newJSONNumber(3)}), + expectedJSON: "[1,2,3]", + }, + { + name: "Int32Array", + input: []int32{1, 2, 3}, + expected: newJSONArray([]JSON{newJSONNumber(1), newJSONNumber(2), newJSONNumber(3)}), + expectedJSON: "[1,2,3]", + }, + { + name: "Int64Array", + input: []int64{1, 2, 3}, + expected: newJSONArray([]JSON{newJSONNumber(1), newJSONNumber(2), newJSONNumber(3)}), + expectedJSON: "[1,2,3]", + }, + { + name: "IntArray", + input: []int{1, 2, 3}, + expected: newJSONArray([]JSON{newJSONNumber(1), newJSONNumber(2), newJSONNumber(3)}), + expectedJSON: "[1,2,3]", + }, + { + name: "Uint8Array", + input: []uint8{1, 2, 3}, + expected: newJSONArray([]JSON{newJSONNumber(1), newJSONNumber(2), newJSONNumber(3)}), + expectedJSON: "[1,2,3]", + }, + { + name: "Uint16Array", + input: []uint16{1, 2, 3}, + expected: newJSONArray([]JSON{newJSONNumber(1), newJSONNumber(2), newJSONNumber(3)}), + expectedJSON: "[1,2,3]", + }, + { + name: "Uint32Array", + input: []uint32{1, 2, 3}, + expected: newJSONArray([]JSON{newJSONNumber(1), newJSONNumber(2), newJSONNumber(3)}), + expectedJSON: "[1,2,3]", + }, + { + name: "Uint64Array", + input: []uint64{1, 2, 3}, + expected: newJSONArray([]JSON{newJSONNumber(1), newJSONNumber(2), newJSONNumber(3)}), + expectedJSON: "[1,2,3]", + }, + { + name: "UintArray", + input: []uint{1, 2, 3}, + expected: newJSONArray([]JSON{newJSONNumber(1), newJSONNumber(2), newJSONNumber(3)}), + expectedJSON: "[1,2,3]", + }, + { + name: "Float32Array", + input: []float32{1.0, 2.25, 3.5}, + expected: newJSONArray([]JSON{newJSONNumber(1.0), newJSONNumber(2.25), newJSONNumber(3.5)}), + expectedJSON: "[1,2.25,3.5]", + }, + { + name: "Float64Array", + input: []float64{1.0, 2.25, 3.5}, + expected: newJSONArray([]JSON{newJSONNumber(1.0), newJSONNumber(2.25), newJSONNumber(3.5)}), + expectedJSON: "[1,2.25,3.5]", + }, + { + name: "AnyArrayWithInvalidElement", + input: []any{"valid", make(chan int)}, // channels can't be converted to JSON + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := NewJSON(tt.input) + if tt.expectError { + require.Error(t, err, "Expected error, but got nil") + return + } + require.NoError(t, err, "NewJSON failed with error %v", err) + require.Equal(t, result, tt.expected) + + if !tt.expectError { + jsonBytes, err := result.MarshalJSON() + require.NoError(t, err, "MarshalJSON failed with error %v", err) + require.Equal(t, tt.expectedJSON, string(jsonBytes)) + } + }) + } +} + +func TestNewJSONFromMap_WithInvalidValue_ShouldFail(t *testing.T) { + // Map with an invalid value (channel cannot be converted to JSON) + input := map[string]any{ + "valid": "value", + "invalid": make(chan int), + } + + _, err := NewJSONFromMap(input) + require.Error(t, err) +} diff --git a/client/normal_new.go b/client/normal_new.go index bcd0f00929..8eb1b9f24c 100644 --- a/client/normal_new.go +++ b/client/normal_new.go @@ -64,7 +64,7 @@ func NewNormalValue(val any) (NormalValue, error) { return NewNormalTime(v), nil case *Document: return NewNormalDocument(v), nil - case *JSON: + case JSON: return NewNormalJSON(v), nil case immutable.Option[bool]: diff --git a/client/normal_scalar.go b/client/normal_scalar.go index ae92fbe3a6..cc6d9054db 100644 --- a/client/normal_scalar.go +++ b/client/normal_scalar.go @@ -17,13 +17,6 @@ import ( "golang.org/x/exp/constraints" ) -// JSON contains a valid JSON value. -// -// The inner type can be any valid normal value or normal value array. -type JSON struct { - inner any -} - // NormalValue is dummy implementation of NormalValue to be embedded in other types. type baseNormalValue[T any] struct { NormalVoid @@ -126,15 +119,15 @@ func (v normalDocument) Document() (*Document, bool) { } type normalJSON struct { - baseNormalValue[*JSON] + baseNormalValue[JSON] } -func (v normalJSON) JSON() (*JSON, bool) { +func (v normalJSON) JSON() (JSON, bool) { return v.val, true } func (v normalJSON) Unwrap() any { - return v.val.inner + return v.val.Unwrap() } func newNormalInt(val int64) NormalValue { @@ -181,8 +174,8 @@ func NewNormalDocument(val *Document) NormalValue { } // NewNormalJSON creates a new NormalValue that represents a `JSON` value. -func NewNormalJSON(val *JSON) NormalValue { - return normalJSON{baseNormalValue[*JSON]{val: val}} +func NewNormalJSON(val JSON) NormalValue { + return normalJSON{baseNormalValue[JSON]{val: val}} } func areNormalScalarsEqual[T comparable](val T, f func() (T, bool)) bool { diff --git a/client/normal_value.go b/client/normal_value.go index 081814ffe2..3dc66a83fd 100644 --- a/client/normal_value.go +++ b/client/normal_value.go @@ -64,7 +64,7 @@ type NormalValue interface { Document() (*Document, bool) // JSON returns the value as JSON. The second return flag is true if the value is JSON. // Otherwise it will return nil and false. - JSON() (*JSON, bool) + JSON() (JSON, bool) // NillableBool returns the value as a nillable bool. // The second return flag is true if the value is [immutable.Option[bool]]. diff --git a/client/normal_value_test.go b/client/normal_value_test.go index bcea59e046..773727c72a 100644 --- a/client/normal_value_test.go +++ b/client/normal_value_test.go @@ -78,8 +78,8 @@ const ( // Otherwise, it returns the input itself. func extractValue(input any) any { // unwrap JSON inner values - if v, ok := input.(*JSON); ok { - return v.inner + if v, ok := input.(JSON); ok { + return v.Unwrap() } inputVal := reflect.ValueOf(input) @@ -171,7 +171,7 @@ func TestNormalValue_NewValueAndTypeAssertion(t *testing.T) { BytesType: func(v any) NormalValue { return NewNormalBytes(v.([]byte)) }, TimeType: func(v any) NormalValue { return NewNormalTime(v.(time.Time)) }, DocumentType: func(v any) NormalValue { return NewNormalDocument(v.(*Document)) }, - JSONType: func(v any) NormalValue { return NewNormalJSON(v.(*JSON)) }, + JSONType: func(v any) NormalValue { return NewNormalJSON(v.(JSON)) }, NillableBoolType: func(v any) NormalValue { return NewNormalNillableBool(v.(immutable.Option[bool])) }, NillableIntType: func(v any) NormalValue { return NewNormalNillableInt(v.(immutable.Option[int64])) }, @@ -293,7 +293,7 @@ func TestNormalValue_NewValueAndTypeAssertion(t *testing.T) { }, { nType: JSONType, - input: &JSON{nil}, + input: newJSONNumber(2), }, { nType: NillableBoolType, @@ -842,53 +842,6 @@ func TestNormalValue_NewNormalValueFromAnyArray(t *testing.T) { } } -func TestNormalValue_NewNormalJSON(t *testing.T) { - var expect *JSON - var actual *JSON - - expect = &JSON{nil} - normal := NewNormalJSON(expect) - - actual, _ = normal.JSON() - assert.Equal(t, expect, actual) - - expect = &JSON{"hello"} - normal = NewNormalJSON(expect) - - actual, _ = normal.JSON() - assert.Equal(t, expect, actual) - - expect = &JSON{true} - normal = NewNormalJSON(expect) - - actual, _ = normal.JSON() - assert.Equal(t, expect, actual) - - expect = &JSON{int64(10)} - normal = NewNormalJSON(expect) - - actual, _ = normal.JSON() - assert.Equal(t, expect, actual) - - expect = &JSON{float64(3.14)} - normal = NewNormalJSON(expect) - - actual, _ = normal.JSON() - assert.Equal(t, expect, actual) - - expect = &JSON{map[string]any{"one": 1}} - normal = NewNormalJSON(expect) - - actual, _ = normal.JSON() - assert.Equal(t, expect, actual) - - expect = &JSON{[]any{1, "two"}} - normal = NewNormalJSON(expect) - - actual, _ = normal.JSON() - assert.Equal(t, expect, actual) -} - func TestNormalValue_NewNormalInt(t *testing.T) { i64 := int64(2) v := NewNormalInt(i64) diff --git a/client/normal_void.go b/client/normal_void.go index 3238a25ad2..a9078e5328 100644 --- a/client/normal_void.go +++ b/client/normal_void.go @@ -65,7 +65,7 @@ func (NormalVoid) Document() (*Document, bool) { return nil, false } -func (NormalVoid) JSON() (*JSON, bool) { +func (NormalVoid) JSON() (JSON, bool) { return nil, false } diff --git a/docs/data_format_changes/i3265-unified-json-types.md b/docs/data_format_changes/i3265-unified-json-types.md new file mode 100644 index 0000000000..979f75d869 --- /dev/null +++ b/docs/data_format_changes/i3265-unified-json-types.md @@ -0,0 +1,4 @@ +# Unified JSON Types + +Applied a common interface to all JSON types which made it use float64 for all numbers. +This in turned caused encoded data to change because CBOR encoding of float64 is different from int64. diff --git a/tests/integration/mutation/create/field_kinds/field_kind_json_test.go b/tests/integration/mutation/create/field_kinds/field_kind_json_test.go index b578bf3928..5cb6fdd966 100644 --- a/tests/integration/mutation/create/field_kinds/field_kind_json_test.go +++ b/tests/integration/mutation/create/field_kinds/field_kind_json_test.go @@ -39,7 +39,6 @@ func TestMutationCreate_WithJSONFieldGivenObjectValue_Succeeds(t *testing.T) { testUtils.Request{ Request: `query { Users { - _docID name custom } @@ -47,10 +46,9 @@ func TestMutationCreate_WithJSONFieldGivenObjectValue_Succeeds(t *testing.T) { Results: map[string]any{ "Users": []map[string]any{ { - "_docID": "bae-a948a3b2-3e89-5654-b0f0-71685a66b4d7", "custom": map[string]any{ "tree": "maple", - "age": uint64(250), + "age": float64(250), }, "name": "John", }, @@ -84,7 +82,6 @@ func TestMutationCreate_WithJSONFieldGivenListOfScalarsValue_Succeeds(t *testing testUtils.Request{ Request: `query { Users { - _docID name custom } @@ -92,8 +89,7 @@ func TestMutationCreate_WithJSONFieldGivenListOfScalarsValue_Succeeds(t *testing Results: map[string]any{ "Users": []map[string]any{ { - "_docID": "bae-90fd8b1b-bd11-56b5-a78c-2fb6f7b4dca0", - "custom": []any{"maple", uint64(250)}, + "custom": []any{"maple", float64(250)}, "name": "John", }, }, @@ -129,7 +125,6 @@ func TestMutationCreate_WithJSONFieldGivenListOfObjectsValue_Succeeds(t *testing testUtils.Request{ Request: `query { Users { - _docID name custom } @@ -137,7 +132,6 @@ func TestMutationCreate_WithJSONFieldGivenListOfObjectsValue_Succeeds(t *testing Results: map[string]any{ "Users": []map[string]any{ { - "_docID": "bae-dd7c12f5-a7c5-55c6-8b35-ece853ae7f9e", "custom": []any{ map[string]any{"tree": "maple"}, map[string]any{"tree": "oak"}, @@ -174,7 +168,6 @@ func TestMutationCreate_WithJSONFieldGivenIntValue_Succeeds(t *testing.T) { testUtils.Request{ Request: `query { Users { - _docID name custom } @@ -182,8 +175,7 @@ func TestMutationCreate_WithJSONFieldGivenIntValue_Succeeds(t *testing.T) { Results: map[string]any{ "Users": []map[string]any{ { - "_docID": "bae-59731737-8793-5794-a9a5-0ed0ad696d5c", - "custom": uint64(250), + "custom": float64(250), "name": "John", }, }, @@ -216,7 +208,6 @@ func TestMutationCreate_WithJSONFieldGivenStringValue_Succeeds(t *testing.T) { testUtils.Request{ Request: `query { Users { - _docID name custom } @@ -224,7 +215,6 @@ func TestMutationCreate_WithJSONFieldGivenStringValue_Succeeds(t *testing.T) { Results: map[string]any{ "Users": []map[string]any{ { - "_docID": "bae-608582c3-979e-5f34-80f8-a70fce875d05", "custom": "hello", "name": "John", }, @@ -258,7 +248,6 @@ func TestMutationCreate_WithJSONFieldGivenBooleanValue_Succeeds(t *testing.T) { testUtils.Request{ Request: `query { Users { - _docID name custom } @@ -266,7 +255,6 @@ func TestMutationCreate_WithJSONFieldGivenBooleanValue_Succeeds(t *testing.T) { Results: map[string]any{ "Users": []map[string]any{ { - "_docID": "bae-0c4b39cf-433c-5a9c-9bed-1e2796c35d14", "custom": true, "name": "John", }, @@ -300,7 +288,6 @@ func TestMutationCreate_WithJSONFieldGivenNullValue_Succeeds(t *testing.T) { testUtils.Request{ Request: `query { Users { - _docID name custom } @@ -308,7 +295,6 @@ func TestMutationCreate_WithJSONFieldGivenNullValue_Succeeds(t *testing.T) { Results: map[string]any{ "Users": []map[string]any{ { - "_docID": "bae-f405f600-56d9-5de4-8d02-75fdced35e3b", "custom": nil, "name": "John", }, diff --git a/tests/integration/query/json/with_ge_test.go b/tests/integration/query/json/with_ge_test.go index bfb574170e..4a9afc403e 100644 --- a/tests/integration/query/json/with_ge_test.go +++ b/tests/integration/query/json/with_ge_test.go @@ -270,10 +270,10 @@ func TestQueryJSON_WithGreaterEqualFilterWithNestedNullValue_ShouldFilter(t *tes Results: map[string]any{ "Users": []map[string]any{ { - "Name": "John", + "Name": "David", }, { - "Name": "David", + "Name": "John", }, }, }, diff --git a/tests/integration/query/json/with_gt_test.go b/tests/integration/query/json/with_gt_test.go index 3a2972320b..07d08ce7ca 100644 --- a/tests/integration/query/json/with_gt_test.go +++ b/tests/integration/query/json/with_gt_test.go @@ -182,7 +182,7 @@ func TestQueryJSON_WithGreaterThanFilterBlockWithNestedGreaterValue_ShouldFilter { "Name": "John", "Custom": map[string]any{ - "age": uint64(21), + "age": float64(21), }, }, }, diff --git a/tests/integration/query/json/with_lt_test.go b/tests/integration/query/json/with_lt_test.go index 14a422d5ad..636139c05d 100644 --- a/tests/integration/query/json/with_lt_test.go +++ b/tests/integration/query/json/with_lt_test.go @@ -178,7 +178,7 @@ func TestQueryJSON_WithLesserThanFilterBlockWithNestedGreaterValue_ShouldFilter( { "Name": "Bob", "Custom": map[string]any{ - "age": uint64(19), + "age": float64(19), }, }, }, diff --git a/tests/integration/query/json/with_nlike_test.go b/tests/integration/query/json/with_nlike_test.go index db0615b2ca..6de741f61b 100644 --- a/tests/integration/query/json/with_nlike_test.go +++ b/tests/integration/query/json/with_nlike_test.go @@ -65,16 +65,16 @@ func TestQueryJSON_WithNotLikeFilter_ShouldFilter(t *testing.T) { Results: map[string]any{ "Users": []map[string]any{ { - "custom": uint64(32), + "custom": map[string]any{"one": float64(1)}, }, { - "custom": "Viserys I Targaryen, King of the Andals", + "custom": float64(32), }, { - "custom": map[string]any{"one": uint64(1)}, + "custom": []any{float64(1), float64(2)}, }, { - "custom": []any{uint64(1), uint64(2)}, + "custom": "Viserys I Targaryen, King of the Andals", }, { "custom": false, diff --git a/tests/integration/utils.go b/tests/integration/utils.go index 3c0e9baffd..dfd3096bd3 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -114,7 +114,6 @@ func init() { // mutation type. mutationType = CollectionSaveMutationType } - mutationType = GQLRequestMutationType if value, ok := os.LookupEnv(viewTypeEnvName); ok { viewType = ViewType(value) @@ -1987,7 +1986,7 @@ func assertRequestResultDocs( ) bool { // compare results require.Equal(s.t, len(expectedResults), len(actualResults), - s.testCase.Description+" \n(number of results don't match)") + s.testCase.Description+" \n(number of results don't match for %s)", stack) for actualDocIndex, actualDoc := range actualResults { stack.pushArray(actualDocIndex) @@ -1998,9 +1997,9 @@ func assertRequestResultDocs( len(expectedDoc), len(actualDoc), fmt.Sprintf( - "%s \n(number of properties for item at index %v don't match)", + "%s \n(number of properties don't match for %s)", s.testCase.Description, - actualDocIndex, + stack, ), )