From d53425526693699288a38007d78a2af1d7a331e0 Mon Sep 17 00:00:00 2001 From: Jeff Thompson Date: Mon, 9 Oct 2023 15:14:59 +0200 Subject: [PATCH 1/5] chore: Add ToLowerSnakeCase, use it for Protobuf field names Signed-off-by: Jeff Thompson --- tm2/pkg/amino/genproto/genproto.go | 2 +- tm2/pkg/amino/genproto/snakecase.go | 335 ++++++++++++++++++++++++++++ 2 files changed, 336 insertions(+), 1 deletion(-) create mode 100644 tm2/pkg/amino/genproto/snakecase.go diff --git a/tm2/pkg/amino/genproto/genproto.go b/tm2/pkg/amino/genproto/genproto.go index 8af73b2690a..347b460b168 100644 --- a/tm2/pkg/amino/genproto/genproto.go +++ b/tm2/pkg/amino/genproto/genproto.go @@ -206,7 +206,7 @@ func (p3c *P3Context) GenerateProto3MessagePartial(p3doc *P3Doc, rt reflect.Type p3Field := P3Field{ Repeated: fp3IsRepeated, Type: fp3, - Name: field.Name, + Name: ToLowerSnakeCase(field.Name), Number: field.FieldOptions.BinFieldNum, } p3msg.Fields = append(p3msg.Fields, p3Field) diff --git a/tm2/pkg/amino/genproto/snakecase.go b/tm2/pkg/amino/genproto/snakecase.go new file mode 100644 index 00000000000..4884dd93b85 --- /dev/null +++ b/tm2/pkg/amino/genproto/snakecase.go @@ -0,0 +1,335 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file has ToLowerSnakeCase which the Protobuf linter uses to make +// the expected spelling of a message field name. It is a copy of: +// https://github.com/bufbuild/buf/blob/main/private/pkg/stringutil/stringutil.go + +package genproto + +import ( + "sort" + "strings" + "unicode" +) + +// TrimLines splits the output into individual lines and trims the spaces from each line. +// +// This also trims the start and end spaces from the original output. +func TrimLines(output string) string { + return strings.TrimSpace(strings.Join(SplitTrimLines(output), "\n")) +} + +// SplitTrimLines splits the output into individual lines and trims the spaces from each line. +func SplitTrimLines(output string) []string { + // this should work for windows as well as \r will be trimmed + split := strings.Split(output, "\n") + lines := make([]string, len(split)) + for i, line := range split { + lines[i] = strings.TrimSpace(line) + } + return lines +} + +// SplitTrimLinesNoEmpty splits the output into individual lines and trims the spaces from each line. +// +// This removes any empty lines. +func SplitTrimLinesNoEmpty(output string) []string { + // this should work for windows as well as \r will be trimmed + split := strings.Split(output, "\n") + lines := make([]string, 0, len(split)) + for _, line := range split { + line = strings.TrimSpace(line) + if line != "" { + lines = append(lines, line) + } + } + return lines +} + +// MapToSortedSlice transforms m to a sorted slice. +func MapToSortedSlice(m map[string]struct{}) []string { + s := MapToSlice(m) + sort.Strings(s) + return s +} + +// MapToSlice transforms m to a slice. +func MapToSlice(m map[string]struct{}) []string { + s := make([]string, 0, len(m)) + for e := range m { + s = append(s, e) + } + return s +} + +// SliceToMap transforms s to a map. +func SliceToMap(s []string) map[string]struct{} { + m := make(map[string]struct{}, len(s)) + for _, e := range s { + m[e] = struct{}{} + } + return m +} + +// SliceToUniqueSortedSlice returns a sorted copy of s with no duplicates. +func SliceToUniqueSortedSlice(s []string) []string { + return MapToSortedSlice(SliceToMap(s)) +} + +// SliceToUniqueSortedSliceFilterEmptyStrings returns a sorted copy of s with no duplicates and no empty strings. +// +// Strings with only spaces are considered empty. +func SliceToUniqueSortedSliceFilterEmptyStrings(s []string) []string { + m := SliceToMap(s) + for key := range m { + if strings.TrimSpace(key) == "" { + delete(m, key) + } + } + return MapToSortedSlice(m) +} + +// SliceToChunks splits s into chunks of the given chunk size. +// +// If s is nil or empty, returns empty. +// If chunkSize is <=0, returns [][]string{s}. +func SliceToChunks(s []string, chunkSize int) [][]string { + var chunks [][]string + if len(s) == 0 { + return chunks + } + if chunkSize <= 0 { + return [][]string{s} + } + c := make([]string, len(s)) + copy(c, s) + // https://github.com/golang/go/wiki/SliceTricks#batching-with-minimal-allocation + for chunkSize < len(c) { + c, chunks = c[chunkSize:], append(chunks, c[0:chunkSize:chunkSize]) + } + return append(chunks, c) +} + +// SliceElementsEqual returns true if the two slices have equal elements. +// +// Nil and empty slices are treated as equals. +func SliceElementsEqual(one []string, two []string) bool { + if len(one) != len(two) { + return false + } + for i, elem := range one { + if two[i] != elem { + return false + } + } + return true +} + +// SliceElementsContained returns true if superset contains subset. +// +// Nil and empty slices are treated as equals. +func SliceElementsContained(superset []string, subset []string) bool { + m := SliceToMap(superset) + for _, elem := range subset { + if _, ok := m[elem]; !ok { + return false + } + } + return true +} + +// JoinSliceQuoted joins the slice with quotes. +func JoinSliceQuoted(s []string, sep string) string { + if len(s) == 0 { + return "" + } + return `"` + strings.Join(s, `"`+sep+`"`) + `"` +} + +// SliceToString prints the slice as [e1,e2]. +func SliceToString(s []string) string { + if len(s) == 0 { + return "" + } + return "[" + strings.Join(s, ",") + "]" +} + +// SliceToHumanString prints the slice as "e1, e2, and e3". +func SliceToHumanString(s []string) string { + switch len(s) { + case 0: + return "" + case 1: + return s[0] + case 2: + return s[0] + " and " + s[1] + default: + return strings.Join(s[:len(s)-1], ", ") + ", and " + s[len(s)-1] + } +} + +// SliceToHumanStringQuoted prints the slice as `"e1", "e2", and "e3"`. +func SliceToHumanStringQuoted(s []string) string { + switch len(s) { + case 0: + return "" + case 1: + return `"` + s[0] + `"` + case 2: + return `"` + s[0] + `" and "` + s[1] + `"` + default: + return `"` + strings.Join(s[:len(s)-1], `", "`) + `", and "` + s[len(s)-1] + `"` + } +} + +// SliceToHumanStringOr prints the slice as "e1, e2, or e3". +func SliceToHumanStringOr(s []string) string { + switch len(s) { + case 0: + return "" + case 1: + return s[0] + case 2: + return s[0] + " or " + s[1] + default: + return strings.Join(s[:len(s)-1], ", ") + ", or " + s[len(s)-1] + } +} + +// SliceToHumanStringOrQuoted prints the slice as `"e1", "e2", or "e3"`. +func SliceToHumanStringOrQuoted(s []string) string { + switch len(s) { + case 0: + return "" + case 1: + return `"` + s[0] + `"` + case 2: + return `"` + s[0] + `" or "` + s[1] + `"` + default: + return `"` + strings.Join(s[:len(s)-1], `", "`) + `", or "` + s[len(s)-1] + `"` + } +} + +// SnakeCaseOption is an option for snake_case conversions. +type SnakeCaseOption func(*snakeCaseOptions) + +// SnakeCaseWithNewWordOnDigits is a SnakeCaseOption that signifies +// to split on digits, ie foo_bar_1 instead of foo_bar1. +func SnakeCaseWithNewWordOnDigits() SnakeCaseOption { + return func(snakeCaseOptions *snakeCaseOptions) { + snakeCaseOptions.newWordOnDigits = true + } +} + +// ToLowerSnakeCase transforms s to lower_snake_case. +func ToLowerSnakeCase(s string, options ...SnakeCaseOption) string { + return strings.ToLower(toSnakeCase(s, options...)) +} + +// ToUpperSnakeCase transforms s to UPPER_SNAKE_CASE. +func ToUpperSnakeCase(s string, options ...SnakeCaseOption) string { + return strings.ToUpper(toSnakeCase(s, options...)) +} + +// ToPascalCase converts s to PascalCase. +// +// Splits on '-', '_', ' ', '\t', '\n', '\r'. +// Uppercase letters will stay uppercase, +func ToPascalCase(s string) string { + output := "" + var previous rune + for i, c := range strings.TrimSpace(s) { + if !isDelimiter(c) { + if i == 0 || isDelimiter(previous) || unicode.IsUpper(c) { + output += string(unicode.ToUpper(c)) + } else { + output += string(unicode.ToLower(c)) + } + } + previous = c + } + return output +} + +// IsAlphanumeric returns true for [0-9a-zA-Z]. +func IsAlphanumeric(r rune) bool { + return IsNumeric(r) || IsAlpha(r) +} + +// IsAlpha returns true for [a-zA-Z]. +func IsAlpha(r rune) bool { + return IsLowerAlpha(r) || IsUpperAlpha(r) +} + +// IsLowerAlpha returns true for [a-z]. +func IsLowerAlpha(r rune) bool { + return 'a' <= r && r <= 'z' +} + +// IsUpperAlpha returns true for [A-Z]. +func IsUpperAlpha(r rune) bool { + return 'A' <= r && r <= 'Z' +} + +// IsNumeric returns true for [0-9]. +func IsNumeric(r rune) bool { + return '0' <= r && r <= '9' +} + +// IsLowerAlphanumeric returns true for [0-9a-z]. +func IsLowerAlphanumeric(r rune) bool { + return IsNumeric(r) || IsLowerAlpha(r) +} + +func toSnakeCase(s string, options ...SnakeCaseOption) string { + snakeCaseOptions := &snakeCaseOptions{} + for _, option := range options { + option(snakeCaseOptions) + } + output := "" + s = strings.TrimFunc(s, isDelimiter) + for i, c := range s { + if isDelimiter(c) { + c = '_' + } + if i == 0 { + output += string(c) + } else if isSnakeCaseNewWord(c, snakeCaseOptions.newWordOnDigits) && + output[len(output)-1] != '_' && + ((i < len(s)-1 && !isSnakeCaseNewWord(rune(s[i+1]), true) && !isDelimiter(rune(s[i+1]))) || + (snakeCaseOptions.newWordOnDigits && unicode.IsDigit(c)) || + (unicode.IsLower(rune(s[i-1])))) { + output += "_" + string(c) + } else if !(isDelimiter(c) && output[len(output)-1] == '_') { + output += string(c) + } + } + return output +} + +func isSnakeCaseNewWord(r rune, newWordOnDigits bool) bool { + if newWordOnDigits { + return unicode.IsUpper(r) || unicode.IsDigit(r) + } + return unicode.IsUpper(r) +} + +func isDelimiter(r rune) bool { + return r == '.' || r == '-' || r == '_' || r == ' ' || r == '\t' || r == '\n' || r == '\r' +} + +type snakeCaseOptions struct { + newWordOnDigits bool +} From b125b882bea79ba65e6b9204ec0ec3bc3302e537 Mon Sep 17 00:00:00 2001 From: Jeff Thompson Date: Tue, 10 Oct 2023 09:20:30 +0200 Subject: [PATCH 2/5] chore: genproto_test: Use lower_snake_case in expected proto output. Signed-off-by: Jeff Thompson --- tm2/pkg/amino/genproto/genproto_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tm2/pkg/amino/genproto/genproto_test.go b/tm2/pkg/amino/genproto/genproto_test.go index 20a45361519..c1999c09401 100644 --- a/tm2/pkg/amino/genproto/genproto_test.go +++ b/tm2/pkg/amino/genproto/genproto_test.go @@ -15,9 +15,9 @@ func TestBasic(t *testing.T) { obj := sm1.StructSM{} p3message := p3c.GenerateProto3MessagePartial(&p3doc, reflect.TypeOf(obj)) assert.Equal(t, p3message.Print(), `message StructSM { - sint64 FieldA = 1; - string FieldB = 2; - submodule2.StructSM2 FieldC = 3; + sint64 field_a = 1; + string field_b = 2; + submodule2.StructSM2 field_c = 3; } `) @@ -38,8 +38,8 @@ import "github.com/gnolang/gno/tm2/pkg/amino/genproto/example/submodule2/submodu // messages message StructSM { - sint64 FieldA = 1; - string FieldB = 2; - submodule2.StructSM2 FieldC = 3; + sint64 field_a = 1; + string field_b = 2; + submodule2.StructSM2 field_c = 3; }`) } From 0b2c7b65363ea03e1d9b7fd8e88e907662741e98 Mon Sep 17 00:00:00 2001 From: Jeff Thompson Date: Wed, 11 Oct 2023 18:22:56 +0200 Subject: [PATCH 3/5] chore: Move snakecase.go to stringutil subfolder. Signed-off-by: Jeff Thompson --- tm2/pkg/amino/genproto/genproto.go | 3 ++- tm2/pkg/amino/genproto/{ => stringutil}/snakecase.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) rename tm2/pkg/amino/genproto/{ => stringutil}/snakecase.go (99%) diff --git a/tm2/pkg/amino/genproto/genproto.go b/tm2/pkg/amino/genproto/genproto.go index 347b460b168..ae4fdb5e169 100644 --- a/tm2/pkg/amino/genproto/genproto.go +++ b/tm2/pkg/amino/genproto/genproto.go @@ -16,6 +16,7 @@ import ( "time" "github.com/gnolang/gno/tm2/pkg/amino" + "github.com/gnolang/gno/tm2/pkg/amino/genproto/stringutil" "github.com/gnolang/gno/tm2/pkg/amino/pkg" ) @@ -206,7 +207,7 @@ func (p3c *P3Context) GenerateProto3MessagePartial(p3doc *P3Doc, rt reflect.Type p3Field := P3Field{ Repeated: fp3IsRepeated, Type: fp3, - Name: ToLowerSnakeCase(field.Name), + Name: stringutil.ToLowerSnakeCase(field.Name), Number: field.FieldOptions.BinFieldNum, } p3msg.Fields = append(p3msg.Fields, p3Field) diff --git a/tm2/pkg/amino/genproto/snakecase.go b/tm2/pkg/amino/genproto/stringutil/snakecase.go similarity index 99% rename from tm2/pkg/amino/genproto/snakecase.go rename to tm2/pkg/amino/genproto/stringutil/snakecase.go index 4884dd93b85..b7eb8c987da 100644 --- a/tm2/pkg/amino/genproto/snakecase.go +++ b/tm2/pkg/amino/genproto/stringutil/snakecase.go @@ -16,7 +16,7 @@ // the expected spelling of a message field name. It is a copy of: // https://github.com/bufbuild/buf/blob/main/private/pkg/stringutil/stringutil.go -package genproto +package stringutil import ( "sort" From c087fe31de22202b7ed8216903575aa604f61f3a Mon Sep 17 00:00:00 2001 From: Jeff Thompson Date: Wed, 11 Oct 2023 11:22:05 +0200 Subject: [PATCH 4/5] chore: In amino/pkg, add optional WithComments, use them in GenerateProto3MessagePartial Signed-off-by: Jeff Thompson --- tm2/pkg/amino/genproto/comments_test.go | 61 ++++++++++++++++++++++++ tm2/pkg/amino/genproto/genproto.go | 10 ++++ tm2/pkg/amino/pkg/pkg.go | 62 ++++++++++++++++++++++++- 3 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 tm2/pkg/amino/genproto/comments_test.go diff --git a/tm2/pkg/amino/genproto/comments_test.go b/tm2/pkg/amino/genproto/comments_test.go new file mode 100644 index 00000000000..2294dfe42de --- /dev/null +++ b/tm2/pkg/amino/genproto/comments_test.go @@ -0,0 +1,61 @@ +package genproto + +import ( + "path" + "testing" + + "github.com/gnolang/gno/tm2/pkg/amino" + "github.com/jaekwon/testify/assert" +) + +// message comment +type TestMessageName struct { + // field comment 1 + FieldName1 string + // field comment 2 + FieldName2 []uint64 +} + +// message comment 2 +type TestMessageName2 struct { + // another field comment + FieldName string +} + +func TestComments(t *testing.T) { + pkg := amino.RegisterPackage( + amino.NewPackage( + "github.com/gnolang/gno/tm2/pkg/amino/genproto", + "amino_test", + amino.GetCallersDirname(), + ).WithTypes( + &TestMessageName{}, + &TestMessageName2{}, + // Add comments from this same source file. + ).WithComments(path.Join(amino.GetCallersDirname(), "comments_test.go"))) + + p3c := NewP3Context() + p3c.RegisterPackage(pkg) + p3c.ValidateBasic() + p3doc := p3c.GenerateProto3SchemaForTypes(pkg, pkg.ReflectTypes()...) + proto3Schema := p3doc.Print() + assert.Equal(t, proto3Schema, `syntax = "proto3"; +package amino_test; + +option go_package = "github.com/gnolang/gno/tm2/pkg/amino/genproto/pb"; + +// messages +// message comment +message TestMessageName { + // field comment 1 + string FieldName1 = 1; + // field comment 2 + repeated uint64 FieldName2 = 2; +} + +// message comment 2 +message TestMessageName2 { + // another field comment + string FieldName = 1; +}`) +} diff --git a/tm2/pkg/amino/genproto/genproto.go b/tm2/pkg/amino/genproto/genproto.go index ae4fdb5e169..8f936f8d4f5 100644 --- a/tm2/pkg/amino/genproto/genproto.go +++ b/tm2/pkg/amino/genproto/genproto.go @@ -192,6 +192,13 @@ func (p3c *P3Context) GenerateProto3MessagePartial(p3doc *P3Doc, rt reflect.Type p3msg.Name = info.Name // not rinfo. + var fieldComments map[string]string + if pkgType, ok := rinfo.Package.GetType(rt); ok { + p3msg.Comment = pkgType.Comment + // We will check for optional field comments below. + fieldComments = pkgType.FieldComments + } + // Append to p3msg.Fields, fields of the struct. for _, field := range rsfields { // rinfo. fp3, fp3IsRepeated, implicit := typeToP3Type(info.Package, field.TypeInfo, field.FieldOptions) @@ -210,6 +217,9 @@ func (p3c *P3Context) GenerateProto3MessagePartial(p3doc *P3Doc, rt reflect.Type Name: stringutil.ToLowerSnakeCase(field.Name), Number: field.FieldOptions.BinFieldNum, } + if fieldComments != nil { + p3Field.Comment = fieldComments[field.Name] + } p3msg.Fields = append(p3msg.Fields, p3Field) } diff --git a/tm2/pkg/amino/pkg/pkg.go b/tm2/pkg/amino/pkg/pkg.go index fad46c4cad7..5ce0a0ef7ea 100644 --- a/tm2/pkg/amino/pkg/pkg.go +++ b/tm2/pkg/amino/pkg/pkg.go @@ -2,6 +2,9 @@ package pkg import ( "fmt" + "go/ast" + "go/parser" + "go/token" "path/filepath" "reflect" "regexp" @@ -11,8 +14,10 @@ import ( type Type struct { Type reflect.Type - Name string // proto3 name (override) - PointerPreferred bool // whether pointer is preferred for decoding interface. + Name string // proto3 name (override) + PointerPreferred bool // whether pointer is preferred for decoding interface. + Comment string // optional doc comment for the type + FieldComments map[string]string // If not nil, the optional doc comment for each field name } func (t *Type) FullName(pkg *Package) string { @@ -196,6 +201,59 @@ func (pkg *Package) WithP3SchemaFile(file string) *Package { return pkg } +// Parse the Go code in filename and scan the AST looking for struct doc comments. +// Find the Type in pkg.Types and set its Comment and FieldComments, which are +// used by genproto.GenerateProto3MessagePartial to set the Comment in the P3Doc +// and related P3Field objects. +func (pkg *Package) WithComments(filename string) *Package { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, filename, nil, parser.ParseComments) + if err != nil { + panic(err) + } + + ast.Inspect(f, func(node ast.Node) bool { + if genDecl, ok := node.(*ast.GenDecl); ok { + for _, spec := range genDecl.Specs { + if typeSpec, ok := spec.(*ast.TypeSpec); ok { + if pkgType := pkg.getTypeByName(typeSpec.Name.Name); pkgType != nil { + if genDecl.Doc != nil { + // Set the type comment. + pkgType.Comment = strings.TrimSpace(genDecl.Doc.Text()) + } + if structType, ok := typeSpec.Type.(*ast.StructType); ok { + for _, field := range structType.Fields.List { + if field.Names != nil && len(field.Names) == 1 && field.Doc != nil { + // Set the field comment. + if pkgType.FieldComments == nil { + pkgType.FieldComments = make(map[string]string) + } + + pkgType.FieldComments[field.Names[0].Name] = strings.TrimSpace(field.Doc.Text()) + } + } + } + } + } + } + } + return true + }) + + return pkg +} + +// Get the Type by name. If not found, return nil. +func (pkg *Package) getTypeByName(name string) *Type { + for _, t := range pkg.Types { + if t.Name == name { + return t + } + } + + return nil +} + // Result cannot be modified. func (pkg *Package) GetType(rt reflect.Type) (t Type, ok bool) { if rt.Kind() == reflect.Ptr { From 29b0f81bd5bd8b1056f141df90e3a5eb9bf70bbc Mon Sep 17 00:00:00 2001 From: Jeff Thompson Date: Mon, 16 Oct 2023 09:41:54 +0200 Subject: [PATCH 5/5] chore: In comments_test.go, expect snake_case field names. Signed-off-by: Jeff Thompson --- tm2/pkg/amino/genproto/comments_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tm2/pkg/amino/genproto/comments_test.go b/tm2/pkg/amino/genproto/comments_test.go index 2294dfe42de..5e87663fdc9 100644 --- a/tm2/pkg/amino/genproto/comments_test.go +++ b/tm2/pkg/amino/genproto/comments_test.go @@ -48,14 +48,14 @@ option go_package = "github.com/gnolang/gno/tm2/pkg/amino/genproto/pb"; // message comment message TestMessageName { // field comment 1 - string FieldName1 = 1; + string field_name1 = 1; // field comment 2 - repeated uint64 FieldName2 = 2; + repeated uint64 field_name2 = 2; } // message comment 2 message TestMessageName2 { // another field comment - string FieldName = 1; + string field_name = 1; }`) }