diff --git a/lexer.go b/lexer.go index 44784c0..b8011f4 100644 --- a/lexer.go +++ b/lexer.go @@ -28,17 +28,23 @@ const ( tokenNakedVal // naked value (string like, without double quote) // keywords - tokenModule // module - tokenRequire // require - tokenExclude // exclude - tokenReplace // replace + tokenModule // module + tokenRequire // require + tokenExclude // exclude + tokenReplace // replace + tokenGo // go + tokenComment // // + tokenIndirectComment // indirect comment ) var key = map[string]tokenKind{ - "module": tokenModule, - "require": tokenRequire, - "exclude": tokenExclude, - "replace": tokenReplace, + "module": tokenModule, + "require": tokenRequire, + "exclude": tokenExclude, + "replace": tokenReplace, + "go": tokenGo, + "//": tokenComment, + "indirect": tokenIndirectComment, } type token struct { @@ -163,6 +169,12 @@ func lexFile(l *lexer) lexFn { return lexFile case isAlpha(r): return lexKeywordOrNakedVal + case r >= '0' && r <= '9': + return lexKeywordOrNakedVal + case r == '_': + return lexKeywordOrNakedVal + case r == '/': + return lexComment case r == eof: l.ignore() l.emit(tokenEOF) @@ -173,10 +185,41 @@ func lexFile(l *lexer) lexFn { } } +func lexComment(l *lexer) lexFn { + r := l.next() + if r != '/' { + l.backup() + return lexFile + } + l.emit(tokenComment) + l.ignore() + + comment := "" + for { + r := l.next() + if isWhiteSpace(r) { + comment += l.val() + l.ignore() + continue + } + if r == '\n' { + l.backup() + comment += l.val() + // log.Printf("comment: '%s'", comment) + comment = strings.TrimSpace(comment) + if kind, ok := key[comment]; ok { + l.emit(kind) + } + l.ignore() + return lexFile + } + } +} + func lexKeywordOrNakedVal(l *lexer) lexFn { for { switch r := l.next(); { - case unicode.IsLetter(r), unicode.IsDigit(r), strings.ContainsRune("+-./", r): + case unicode.IsLetter(r), unicode.IsDigit(r), strings.ContainsRune("+-./_", r): // absorb default: l.backup() diff --git a/lexer_test.go b/lexer_test.go index 81d6962..ed6367c 100644 --- a/lexer_test.go +++ b/lexer_test.go @@ -7,21 +7,30 @@ import ( func TestLex(t *testing.T) { input := ` module my/thing + go 1.12 require other/thing v1.0.2 require new/thing v2.3.4 exclude old/thing v1.2.3 require ( future/thing v2.3.4 great/thing v1.2.3 + indirect/thing v5.6.7 // indirect ) replace bad/thing v1.4.5 => good/thing v1.4.5 + // refer: https://github.com/gin-gonic/gin/issues/1673 ` + // input := ` + // // refer: https://github.com/gin-gonic/gin/issues/1673 + // ` expects := []token{ tokNewline(), tokModule(), tokNakedVal("my/thing"), tokNewline(), + tokGo(), + tokNakedVal("1.12"), tokNewline(), + tokRequire(), tokNakedVal("other/thing"), tokNakedVal("v1.0.2"), tokNewline(), @@ -35,6 +44,7 @@ func TestLex(t *testing.T) { tokLeftParen(), tokNewline(), tokNakedVal("future/thing"), tokNakedVal("v2.3.4"), tokNewline(), tokNakedVal("great/thing"), tokNakedVal("v1.2.3"), tokNewline(), + tokNakedVal("indirect/thing"), tokNakedVal("v5.6.7"), tokComment(), tokIndirectComment(), tokNewline(), tokRightParen(), tokNewline(), tokReplace(), @@ -42,13 +52,22 @@ func TestLex(t *testing.T) { tokArrowFun(), tokNakedVal("good/thing"), tokNakedVal("v1.4.5"), tokNewline(), + tokComment(), tokNewline(), + tokEOF(), } + // expects := []token{ + // tokNewline(), + // tokComment(), tokNewline(), + // tokEOF(), + // } l := lexInString(input) for i, e := range expects { v := l.nextToken() if got, want := v, e; got != want { + t.Errorf("got %d %s", got.kind, got.val) + t.Errorf("want %d %s", want.kind, want.val) t.Error("got:", got, "want:", want, "i:", i) } } @@ -62,6 +81,10 @@ func tokModule() token { return token{kind: tokenModule, val: "module"} } +func tokGo() token { + return token{kind: tokenGo, val: "go"} +} + func tokRequire() token { return token{kind: tokenRequire, val: "require"} } @@ -86,6 +109,14 @@ func tokRightParen() token { return token{kind: tokenRightParen, val: ")"} } +func tokComment() token { + return token{kind: tokenComment, val: "//"} +} + +func tokIndirectComment() token { + return token{kind: tokenIndirectComment, val: "indirect"} +} + func tokNakedVal(s string) token { return token{kind: tokenNakedVal, val: s} } diff --git a/parser.go b/parser.go index 35e6bc7..911e171 100644 --- a/parser.go +++ b/parser.go @@ -6,10 +6,11 @@ import ( // Module represents the mod file. type Module struct { - Name string // Name of module - Requires []Package // Require declaration - Excludes []Package // Exclude declaration - Replaces []PackageMap // Replace declaration + GoVersion string // Go version + Name string // Name of module + Requires []Package // Require declaration + Excludes []Package // Exclude declaration + Replaces []PackageMap // Replace declaration } // PackageMap package mapping definition. @@ -20,8 +21,9 @@ type PackageMap struct { // Package represents the package info. type Package struct { - Path string // Import path - Version string // Version (semver) + Path string // Import path + Version string // Version (semver) + Indirect bool // Indirect } // Parse module file from given b. @@ -78,6 +80,14 @@ func (p *parser) errorf(format string, args ...interface{}) parseFn { return p.error(fmt.Errorf(format, args...)) } +func (p *parser) goVersion(v string) { + p.file.GoVersion = v +} + +func (p *parser) comment(v string) { + // do nothing +} + func (p *parser) requirePkg(pkg Package) { p.file.Requires = append(p.file.Requires, pkg) } @@ -130,6 +140,10 @@ func parseVerb(p *parser) parseFn { return parsePkgList(p.excludePkg) case tokenReplace: return parsePkgMapList(p.replacePkg) + case tokenGo: + return parseGoVersion(p.goVersion) + case tokenComment: + return parseComment(p.comment) case tokenNewline: // ignore return parseVerb @@ -140,6 +154,26 @@ func parseVerb(p *parser) parseFn { } } +func parseGoVersion(add func(pkg string)) parseFn { + return func(p *parser) parseFn { + t := p.nextToken() + add(t.val) + return parseVerb + } +} + +func parseComment(add func(pkg string)) parseFn { + return func(p *parser) parseFn { + for { + t := p.nextToken() + if t.kind == tokenNewline { + break + } + } + return parseVerb + } +} + func parsePkgList(add func(pkg Package)) parseFn { return func(p *parser) parseFn { t := p.nextToken() @@ -157,7 +191,12 @@ func parsePkgList(add func(pkg Package)) parseFn { } if t = p.nextToken(); t.kind != tokenNewline { - return p.errorf("expect newline, got %s", t) + if t.kind != tokenComment { + return p.errorf("expect newline, got %s", t) + } + if t = p.nextToken(); t.kind == tokenIndirectComment { + pkg.Indirect = true + } } add(*pkg) @@ -170,7 +209,7 @@ func parsePkgListElem(add func(pkg Package)) parseFn { t := p.skipNewline() if t.kind == tokenRightParen { if t = p.nextToken(); t.kind != tokenNewline { - return p.errorf("expect newline, got %s", t) + return p.errorf("2 expect newline, got %s", t) } return parseVerb @@ -182,7 +221,12 @@ func parsePkgListElem(add func(pkg Package)) parseFn { } if t = p.nextToken(); t.kind != tokenNewline { - return p.errorf("expect newline, got %s", t) + if t.kind != tokenComment { + return p.errorf("3 expect newline, got %s", t) + } + if t = p.nextToken(); t.kind == tokenIndirectComment { + pkg.Indirect = true + } } add(*pkg) diff --git a/parser_test.go b/parser_test.go index f2b46ce..83b518a 100644 --- a/parser_test.go +++ b/parser_test.go @@ -14,6 +14,7 @@ func TestParse(t *testing.T) { require ( new/thing v2.3.4 other/new/thing v1.2.3 + indirect/thing v5.6.7 // indirect ) exclude old/thing v1.2.3 exclude ( @@ -25,12 +26,14 @@ func TestParse(t *testing.T) { bad/thing v1.0.0 => good/thing v1.0.0 new/bad/thing v2.2.3 => new/good/thing v2.2.3 ) + // refer: https://github.com/gin-gonic/gin/issues/1673 ` expectReqs := []module.Package{ {Path: "other/thing", Version: "v1.0.2"}, {Path: "new/thing", Version: "v2.3.4"}, {Path: "other/new/thing", Version: "v1.2.3"}, + {Path: "indirect/thing", Version: "v5.6.7", Indirect: true}, } expectExcl := []module.Package{