Add supports for go version and indirect comment

lexer.go

@@ -28,17 +28,23 @@ const (
 	tokenNakedVal // naked value (string like, without double quote)
 
 	// keywords
-	tokenModule  // module
-	tokenRequire // require
-	tokenExclude // exclude
-	tokenReplace // replace
+	tokenModule          // module
+	tokenRequire         // require
+	tokenExclude         // exclude
+	tokenReplace         // replace
+	tokenGo              // go
+	tokenComment         // //
+	tokenIndirectComment // indirect comment
 )
 
 var key = map[string]tokenKind{
-	"module":  tokenModule,
-	"require": tokenRequire,
-	"exclude": tokenExclude,
-	"replace": tokenReplace,
+	"module":   tokenModule,
+	"require":  tokenRequire,
+	"exclude":  tokenExclude,
+	"replace":  tokenReplace,
+	"go":       tokenGo,
+	"//":       tokenComment,
+	"indirect": tokenIndirectComment,
 }
 
 type token struct {
@@ -163,6 +169,12 @@ func lexFile(l *lexer) lexFn {
 			return lexFile
 		case isAlpha(r):
 			return lexKeywordOrNakedVal
+		case r >= '0' && r <= '9':
+			return lexKeywordOrNakedVal
+		case r == '_':
+			return lexKeywordOrNakedVal
+		case r == '/':
+			return lexComment
 		case r == eof:
 			l.ignore()
 			l.emit(tokenEOF)
@@ -173,10 +185,41 @@ func lexFile(l *lexer) lexFn {
 	}
 }
 
+func lexComment(l *lexer) lexFn {
+	r := l.next()
+	if r != '/' {
+		l.backup()
+		return lexFile
+	}
+	l.emit(tokenComment)
+	l.ignore()
+
+	comment := ""
+	for {
+		r := l.next()
+		if isWhiteSpace(r) {
+			comment += l.val()
+			l.ignore()
+			continue
+		}
+		if r == '\n' {
+			l.backup()
+			comment += l.val()
+			// log.Printf("comment: '%s'", comment)
+			comment = strings.TrimSpace(comment)
+			if kind, ok := key[comment]; ok {
+				l.emit(kind)
+			}
+			l.ignore()
+			return lexFile
+		}
+	}
+}
+
 func lexKeywordOrNakedVal(l *lexer) lexFn {
 	for {
 		switch r := l.next(); {
-		case unicode.IsLetter(r), unicode.IsDigit(r), strings.ContainsRune("+-./", r):
+		case unicode.IsLetter(r), unicode.IsDigit(r), strings.ContainsRune("+-./_", r):
 			// absorb
 		default:
 			l.backup()

lexer_test.go

@@ -7,21 +7,30 @@ import (
 func TestLex(t *testing.T) {
 	input := `
 		module my/thing
+		go 1.12
 		require other/thing v1.0.2
 		require new/thing v2.3.4
 		exclude old/thing v1.2.3
 		require (
 			future/thing v2.3.4
 			great/thing v1.2.3
+			indirect/thing v5.6.7 // indirect
 		)
 		replace bad/thing v1.4.5 => good/thing v1.4.5
+		// refer: https://github.com/gin-gonic/gin/issues/1673
 	`
+	// input := `
+	// 	// refer: https://github.com/gin-gonic/gin/issues/1673
+	// `
 	expects := []token{
 		tokNewline(),
 
 		tokModule(),
 		tokNakedVal("my/thing"), tokNewline(),
 
+		tokGo(),
+		tokNakedVal("1.12"), tokNewline(),
+
 		tokRequire(),
 		tokNakedVal("other/thing"), tokNakedVal("v1.0.2"), tokNewline(),
 
@@ -35,20 +44,30 @@ func TestLex(t *testing.T) {
 		tokLeftParen(), tokNewline(),
 		tokNakedVal("future/thing"), tokNakedVal("v2.3.4"), tokNewline(),
 		tokNakedVal("great/thing"), tokNakedVal("v1.2.3"), tokNewline(),
+		tokNakedVal("indirect/thing"), tokNakedVal("v5.6.7"), tokComment(), tokIndirectComment(), tokNewline(),
 		tokRightParen(), tokNewline(),
 
 		tokReplace(),
 		tokNakedVal("bad/thing"), tokNakedVal("v1.4.5"),
 		tokArrowFun(),
 		tokNakedVal("good/thing"), tokNakedVal("v1.4.5"), tokNewline(),
 
+		tokComment(), tokNewline(),
+
 		tokEOF(),
 	}
+	// expects := []token{
+	// 	tokNewline(),
+	// 	tokComment(), tokNewline(),
+	// 	tokEOF(),
+	// }
 
 	l := lexInString(input)
 	for i, e := range expects {
 		v := l.nextToken()
 		if got, want := v, e; got != want {
+			t.Errorf("got %d %s", got.kind, got.val)
+			t.Errorf("want %d %s", want.kind, want.val)
 			t.Error("got:", got, "want:", want, "i:", i)
 		}
 	}
@@ -62,6 +81,10 @@ func tokModule() token {
 	return token{kind: tokenModule, val: "module"}
 }
 
+func tokGo() token {
+	return token{kind: tokenGo, val: "go"}
+}
+
 func tokRequire() token {
 	return token{kind: tokenRequire, val: "require"}
 }
@@ -86,6 +109,14 @@ func tokRightParen() token {
 	return token{kind: tokenRightParen, val: ")"}
 }
 
+func tokComment() token {
+	return token{kind: tokenComment, val: "//"}
+}
+
+func tokIndirectComment() token {
+	return token{kind: tokenIndirectComment, val: "indirect"}
+}
+
 func tokNakedVal(s string) token {
 	return token{kind: tokenNakedVal, val: s}
 }

parser.go

@@ -6,10 +6,11 @@ import (
 
 // Module represents the mod file.
 type Module struct {
-	Name     string       // Name of module
-	Requires []Package    // Require declaration
-	Excludes []Package    // Exclude declaration
-	Replaces []PackageMap // Replace declaration
+	GoVersion string       // Go version
+	Name      string       // Name of module
+	Requires  []Package    // Require declaration
+	Excludes  []Package    // Exclude declaration
+	Replaces  []PackageMap // Replace declaration
 }
 
 // PackageMap package mapping definition.
@@ -20,8 +21,9 @@ type PackageMap struct {
 
 // Package represents the package info.
 type Package struct {
-	Path    string // Import path
-	Version string // Version (semver)
+	Path     string // Import path
+	Version  string // Version (semver)
+	Indirect bool   // Indirect
 }
 
 // Parse module file from given b.
@@ -78,6 +80,14 @@ func (p *parser) errorf(format string, args ...interface{}) parseFn {
 	return p.error(fmt.Errorf(format, args...))
 }
 
+func (p *parser) goVersion(v string) {
+	p.file.GoVersion = v
+}
+
+func (p *parser) comment(v string) {
+	// do nothing
+}
+
 func (p *parser) requirePkg(pkg Package) {
 	p.file.Requires = append(p.file.Requires, pkg)
 }
@@ -130,6 +140,10 @@ func parseVerb(p *parser) parseFn {
 		return parsePkgList(p.excludePkg)
 	case tokenReplace:
 		return parsePkgMapList(p.replacePkg)
+	case tokenGo:
+		return parseGoVersion(p.goVersion)
+	case tokenComment:
+		return parseComment(p.comment)
 	case tokenNewline:
 		// ignore
 		return parseVerb
@@ -140,6 +154,26 @@ func parseVerb(p *parser) parseFn {
 	}
 }
 
+func parseGoVersion(add func(pkg string)) parseFn {
+	return func(p *parser) parseFn {
+		t := p.nextToken()
+		add(t.val)
+		return parseVerb
+	}
+}
+
+func parseComment(add func(pkg string)) parseFn {
+	return func(p *parser) parseFn {
+		for {
+			t := p.nextToken()
+			if t.kind == tokenNewline {
+				break
+			}
+		}
+		return parseVerb
+	}
+}
+
 func parsePkgList(add func(pkg Package)) parseFn {
 	return func(p *parser) parseFn {
 		t := p.nextToken()
@@ -157,7 +191,12 @@ func parsePkgList(add func(pkg Package)) parseFn {
 		}
 
 		if t = p.nextToken(); t.kind != tokenNewline {
-			return p.errorf("expect newline, got %s", t)
+			if t.kind != tokenComment {
+				return p.errorf("expect newline, got %s", t)
+			}
+			if t = p.nextToken(); t.kind == tokenIndirectComment {
+				pkg.Indirect = true
+			}
 		}
 
 		add(*pkg)
@@ -170,7 +209,7 @@ func parsePkgListElem(add func(pkg Package)) parseFn {
 		t := p.skipNewline()
 		if t.kind == tokenRightParen {
 			if t = p.nextToken(); t.kind != tokenNewline {
-				return p.errorf("expect newline, got %s", t)
+				return p.errorf("2 expect newline, got %s", t)
 			}
 
 			return parseVerb
@@ -182,7 +221,12 @@ func parsePkgListElem(add func(pkg Package)) parseFn {
 		}
 
 		if t = p.nextToken(); t.kind != tokenNewline {
-			return p.errorf("expect newline, got %s", t)
+			if t.kind != tokenComment {
+				return p.errorf("3 expect newline, got %s", t)
+			}
+			if t = p.nextToken(); t.kind == tokenIndirectComment {
+				pkg.Indirect = true
+			}
 		}
 
 		add(*pkg)

parser_test.go

@@ -14,6 +14,7 @@ func TestParse(t *testing.T) {
 		require (
 			new/thing v2.3.4
 			other/new/thing v1.2.3
+			indirect/thing v5.6.7 // indirect
 		)
 		exclude old/thing v1.2.3
 		exclude (
@@ -25,12 +26,14 @@ func TestParse(t *testing.T) {
 			bad/thing v1.0.0 => good/thing v1.0.0
 			new/bad/thing v2.2.3 => new/good/thing v2.2.3
 		)
+		// refer: https://github.com/gin-gonic/gin/issues/1673
 	`
 
 	expectReqs := []module.Package{
 		{Path: "other/thing", Version: "v1.0.2"},
 		{Path: "new/thing", Version: "v2.3.4"},
 		{Path: "other/new/thing", Version: "v1.2.3"},
+		{Path: "indirect/thing", Version: "v5.6.7", Indirect: true},
 	}
 
 	expectExcl := []module.Package{