diff --git a/go.mod b/go.mod index 997b687..2909cce 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.18 require ( github.com/JoshVarga/svgparser v0.0.0-20200804023048-5eaba627a7d1 github.com/adrg/frontmatter v0.2.0 - github.com/antchfx/jsonquery v1.1.5 github.com/antchfx/xmlquery v1.3.3 github.com/bmatcuk/doublestar/v4 v4.0.2 github.com/cheggaaa/pb/v3 v3.0.8 @@ -16,11 +15,10 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/rogpeppe/go-internal v1.10.0 github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 - github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/shopspring/decimal v1.3.1 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 - github.com/xeipuuv/gojsonschema v1.2.0 github.com/zyxar/image2ascii v0.0.0-20180912034614-460a04e371ae golang.org/x/exp v0.0.0-20230321023759-10a507213a29 golang.org/x/net v0.14.0 @@ -45,8 +43,6 @@ require ( github.com/rivo/uniseg v0.4.2 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/stretchr/testify v1.8.4 // indirect - github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect - github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect golang.org/x/sys v0.11.0 // indirect golang.org/x/text v0.12.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect diff --git a/go.sum b/go.sum index d0f8639..c622f77 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,6 @@ github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdc github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/adrg/frontmatter v0.2.0 h1:/DgnNe82o03riBd1S+ZDjd43wAmC6W35q67NHeLkPd4= github.com/adrg/frontmatter v0.2.0/go.mod h1:93rQCj3z3ZlwyxxpQioRKC1wDLto4aXHrbqIsnH9wmE= -github.com/antchfx/jsonquery v1.1.5 h1:1YWrNFYCcIuJPIjFeOP5b6TXbLSUYY8qqxWbuZOB1qE= -github.com/antchfx/jsonquery v1.1.5/go.mod h1:RtMzTHohKaAerkfslTNjr3Y9MdxjKlSgIgaVjVKNiug= github.com/antchfx/xmlquery v1.3.3 h1:HYmadPG0uz8CySdL68rB4DCLKXz2PurCjS3mnkVF4CQ= github.com/antchfx/xmlquery v1.3.3/go.mod h1:64w0Xesg2sTaawIdNqMB+7qaW/bSqkQm+ssPaCMWNnc= github.com/antchfx/xpath v1.1.10/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= @@ -81,8 +79,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI= github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs= -github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 h1:TToq11gyfNlrMFZiYujSekIsPd9AmsA2Bj/iv+s4JHE= -github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= @@ -91,17 +89,9 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= -github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/zyxar/image2ascii v0.0.0-20180912034614-460a04e371ae h1:EiqxsQwk1eimsz+ncJrsMMMwnkYTGiVOrLe5lGxL9cs= github.com/zyxar/image2ascii v0.0.0-20180912034614-460a04e371ae/go.mod h1:Md4Hcw0pmYWDCo1o/fHeOC2Gdhc6oDRwLim8V+SMvI0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/internal/command/frontmatter.go b/internal/command/frontmatter.go index 822fa03..f754c24 100644 --- a/internal/command/frontmatter.go +++ b/internal/command/frontmatter.go @@ -12,20 +12,21 @@ import ( ) var ( - fmStrict bool - fmStrictSet map[string]bool - fmSorted bool - fmRequired []string - fmOptional []string - fmForbidden []string - fmDelimiters []string + fmSchemaOptions shared.SchemaOptions + fmStrict bool + fmStrictSet map[string]bool + fmSorted bool + fmRequired []string + fmOptional []string + fmForbidden []string + fmDelimiters []string ) var frontmatterCmd = &cobra.Command{ Args: cobra.MinimumNArgs(1), Use: "frontmatter [options] files...", Short: "Validate frontmatter", Long: `Checks that the frontmatter in your files is valid`, - PreRunE: frontmatterPrepare, + PreRunE: frontmatterInit, RunE: shared.MakeFileCommand(frontmatterCheck), } @@ -39,8 +40,9 @@ func AddFrontmatterCommand(rootCmd *cobra.Command) { frontmatterCmd.Flags().BoolVar(&fmSorted, "sorted", false, "Keys need to be in alphabetical order") frontmatterCmd.Flags().StringSliceVar(&fmDelimiters, "delimiters", []string{}, "Custom delimiters (if other than `---`, `+++` and `;;;`)") + fmSchemaOptions.AddFlags(frontmatterCmd) + //LATER: report - //LATER: schema } func frontmatterCheck(f *shared.FileContext) { @@ -53,7 +55,7 @@ func frontmatterCheck(f *shared.FileContext) { return } - yamlData := make(map[interface{}]interface{}) + yamlData := make(map[string]any) var formats []*frontmatter.Format @@ -68,6 +70,7 @@ func frontmatterCheck(f *shared.FileContext) { } } + //LATER: maybe flag to require contents? _, parseErr := frontmatter.MustParse(bytes.NewReader(data), &yamlData, formats...) f.RecordResult("frontmatterParse", parseErr == nil, map[string]interface{}{ @@ -97,17 +100,9 @@ func frontmatterCheck(f *shared.FileContext) { if fmStrict { for key := range yamlData { - keyStr, strErr := key.(string) - if !strErr { - f.RecordResult("frontmatterStrictParse", false, map[string]interface{}{ - "err": "key is not a string", - "key": fmt.Sprintf("%v", key), - }) - continue - } - _, ok := fmStrictSet[keyStr] + _, ok := fmStrictSet[key] f.RecordResult("frontmatterStrict", ok, map[string]interface{}{ - "key": keyStr, + "key": key, }) } } @@ -133,9 +128,11 @@ func frontmatterCheck(f *shared.FileContext) { previousKey = currentKey } } + + fmSchemaOptions.Validate(f, yamlData) } -func frontmatterPrepare(cmd *cobra.Command, args []string) error { +func frontmatterInit(cmd *cobra.Command, args []string) error { if fmStrict { fmStrictSet = make(map[string]bool) for _, key := range fmRequired { @@ -150,5 +147,11 @@ func frontmatterPrepare(cmd *cobra.Command, args []string) error { fmt.Fprintf(os.Stderr, "ERROR: delimiter count must be <=2 (passed %d)", len(fmDelimiters)) os.Exit(7) } + + schemaPrepErr := fmSchemaOptions.Prepare() + if schemaPrepErr != nil { + return schemaPrepErr + } + return nil } diff --git a/internal/command/json.go b/internal/command/json.go index 5aa5d58..e58eab8 100644 --- a/internal/command/json.go +++ b/internal/command/json.go @@ -1,21 +1,16 @@ package command import ( - "bytes" - "fmt" - "net/url" - "os" - "path/filepath" + "encoding/json" "github.com/FileFormatInfo/fflint/internal/shared" - "github.com/antchfx/jsonquery" + //"github.com/antchfx/jsonquery" "github.com/spf13/cobra" - "github.com/xeipuuv/gojsonschema" ) var ( - jsonSchemaLocation string - jsonSchema *gojsonschema.Schema + jsonSchemaValidator shared.SchemaOptions + jsonSchemaLocation string ) // jsonCmd represents the json command @@ -31,10 +26,8 @@ var jsonCmd = &cobra.Command{ func AddJsonCommand(rootCmd *cobra.Command) { rootCmd.AddCommand(jsonCmd) - jsonCmd.Flags().StringVar(&jsonSchemaLocation, "schema", "", "JSON Schema to validate against") //LATER: link to docs about embedded ones - + jsonSchemaValidator.AddFlags(jsonCmd) //LATER: whitespace: canonical/none/any - //LATER: schema (https://github.com/xeipuuv/gojsonschema) } func jsonCheck(f *shared.FileContext) { @@ -47,59 +40,25 @@ func jsonCheck(f *shared.FileContext) { return } - _, parseErr := jsonquery.Parse(bytes.NewReader(data)) + var jsonData any + parseErr := json.Unmarshal(data, &jsonData) + f.RecordResult("jsonParse", parseErr == nil, map[string]interface{}{ + "error": parseErr, + }) if parseErr != nil { - f.RecordResult("jsonParse", false, map[string]interface{}{ - "error": parseErr, - }) return } - if jsonSchema != nil { - result, validateErr := jsonSchema.Validate(gojsonschema.NewStringLoader(string(data))) - if validateErr != nil { - f.RecordResult("jsonSchemaRun", false, map[string]interface{}{ - "error": validateErr.Error(), - }) - } else { - f.RecordResult("jsonSchemaValidate", result.Valid(), map[string]interface{}{ - "errors": result.Errors(), - }) - } - } - + jsonSchemaValidator.Validate(f, jsonData) } func jsonInit(cmd *cobra.Command, args []string) error { - if jsonSchemaLocation == "" { - return nil - } - - // work with local file urls - jsonUrl, urlParseErr := url.Parse(jsonSchemaLocation) - if urlParseErr != nil { - return urlParseErr - } - - // allow relative local file schemas - if jsonUrl.Scheme == "" { - jsonUrl.Scheme = "file" - jsonPath, pathErr := filepath.Abs(jsonUrl.Path) - if pathErr != nil { - return pathErr - } - jsonUrl.Path = jsonPath - newLocation := jsonUrl.String() - if shared.Debug { - fmt.Fprintf(os.Stderr, "DEBUG: canonicalizing schema path from '%s' to '%s'\n", jsonSchemaLocation, newLocation) - } - jsonSchemaLocation = newLocation + prepErr := jsonSchemaValidator.Prepare() + if prepErr != nil { + return prepErr } - jsonSchemaLoader := gojsonschema.NewReferenceLoader(jsonSchemaLocation) - var schemaErr error - jsonSchema, schemaErr = gojsonschema.NewSchema(jsonSchemaLoader) - return schemaErr + return nil } diff --git a/internal/shared/schema.go b/internal/shared/schema.go index 179ef4d..55bb523 100644 --- a/internal/shared/schema.go +++ b/internal/shared/schema.go @@ -1,6 +1,9 @@ package shared import ( + "fmt" + "os" + "github.com/santhosh-tekuri/jsonschema/v5" "github.com/spf13/cobra" ) @@ -11,20 +14,43 @@ type SchemaOptions struct { } func (so *SchemaOptions) AddFlags(theCmd *cobra.Command) { - theCmd.Flags().StringVar(&so.src, "schema", "", "JSON schema path (or URL)") } func (so *SchemaOptions) Prepare() error { + if Debug { + fmt.Fprintf(os.Stderr, "DEBUG: trying to compile schema: %s\n", so.src) + } + if so.src == "" { + return nil + } compiled, err := jsonschema.Compile(so.src) if err != nil { + if Debug { + fmt.Fprintf(os.Stderr, "DEBUG: schema compilation error: %v\n", err) + } return err } + if Debug { + fmt.Fprintf(os.Stderr, "DEBUG: successfully compiled schema from %s\n", so.src) + } + so.compiled = compiled return nil } -func (so *SchemaOptions) Validate(data interface{}) error { - return so.compiled.Validate(data) +func (so *SchemaOptions) Validate(f *FileContext, data any) error { + if so.compiled == nil { + return nil + } + validationErr := so.compiled.Validate(data) + if Debug { + fmt.Fprintf(os.Stderr, "DEBUG: schema validation error: %v\n", validationErr) + } + f.RecordResult("jsonSchemaValidatation", validationErr == nil, map[string]interface{}{ + "error": validationErr, + }) + + return validationErr } diff --git a/run.sh b/run.sh index 306a6cc..e1c89e6 100755 --- a/run.sh +++ b/run.sh @@ -1,14 +1,29 @@ #!/usr/bin/env bash # -# run locally +# run tests locally # set -o errexit set -o pipefail set -o nounset -rm -rf ./fflint +if [ -f "./fflint" ]; then + echo "INFO: removing old build of fflint" + rm ./fflint +fi + +echo "INFO: building new fflint" go build -o ./fflint cmd/fflint/main.go -export PATH=$PATH:$(pwd) -fflint version -go test -timeout 30s -run "^TestFflint$" github.com/FileFormatInfo/fflint/cmd/fflint \ No newline at end of file +if [ ! -f "./fflint" ]; then + echo "ERROR: failed to build fflint" + exit 1 +fi + + +export PATH=$(pwd):$PATH +echo "INFO: running fflint version $(fflint version)" + +echo "INFO: running tests" +go test -timeout 30s -run "^TestFflint$" github.com/FileFormatInfo/fflint/cmd/fflint + +echo "INFO: complete at $(date -u +%Y-%m-%dT%H:%M:%SZ)" \ No newline at end of file diff --git a/testdata/frontmatter.txt b/testdata/frontmatter.txt index f153d13..2a4013f 100644 --- a/testdata/frontmatter.txt +++ b/testdata/frontmatter.txt @@ -7,6 +7,21 @@ exec fflint frontmatter --show-tests=all test.md exec fflint frontmatter --delimiters=/***,***/ --debug --show-tests=all test.sql +# anyobject schema +exec fflint frontmatter --glob=none --show-tests=all --debug --schema=anyobject.schema.json test.md +exec fflint frontmatter --glob=none --schema=anyobject.schema.json emptytitle.md +exec fflint frontmatter --glob=none --schema=anyobject.schema.json notitle.md + +# title schema +exec fflint frontmatter --glob=none --schema=title.schema.json test.md +! exec fflint frontmatter --show-tests=all --debug --glob=none --schema=title.schema.json emptytitle.md +! exec fflint frontmatter --glob=none --schema=title.schema.json notitle.md + +# title-optional schema +exec fflint frontmatter --glob=none --schema=title-optional.schema.json test.md +exec fflint frontmatter --glob=none --schema=title-optional.schema.json emptytitle.md +exec fflint frontmatter --glob=none --schema=title-optional.schema.json notitle.md + -- test.md -- --- title: Markdown with frontmatter @@ -22,3 +37,56 @@ title: SQL with custom delimiters ***/ SELECT * FROM examples +-- emptytitle.md -- +--- +title: '' +--- +The content. + +-- notitle.md -- +--- +notitle: '' +--- +The content. + +-- anyobject.schema.json -- +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/product.schema.json", + "title": "Any", + "description": "Any object in frontmatter", + "type": "object" +} + +-- title.schema.json -- +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/product.schema.json", + "title": "Title in frontmatter", + "description": "Frontmatter must have title", + "type": "object", + "properties": { + "title": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "title" + ] +} + +-- title-optional.schema.json -- +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/product.schema.json", + "title": "Title in frontmatter", + "description": "Frontmatter must have title", + "type": "object", + "properties": { + "title": { + "type": "string" + } + } +} + diff --git a/testdata/json.txt b/testdata/json.txt index 26d00b3..60248be 100644 --- a/testdata/json.txt +++ b/testdata/json.txt @@ -6,9 +6,23 @@ exec fflint json object.json exec fflint json something.json ! exec fflint json trailingcomma.json -exec fflint json --schema=object.schema.json --show-tests=all --debug object.json +# object schema tests +exec fflint json --schema=object.schema.json --show-tests=all --debug object.json ! exec fflint json --schema=object.schema.json --show-tests=all --debug string.json +! exec fflint json --schema=object.schema.json --show-tests=all --debug number.json +exec fflint json --schema=object.schema.json --show-tests=all --debug something.json +# string schema tests +! exec fflint json --schema=string.schema.json --show-tests=all --debug object.json +exec fflint json --schema=string.schema.json --show-tests=all --debug string.json +! exec fflint json --schema=string.schema.json --show-tests=all --debug number.json +! exec fflint json --schema=string.schema.json --show-tests=all --debug something.json + +# number schema tests +! exec fflint json --schema=number.schema.json --show-tests=all --debug object.json +! exec fflint json --schema=number.schema.json --show-tests=all --debug string.json +exec fflint json --schema=number.schema.json --show-tests=all --debug number.json +! exec fflint json --schema=number.schema.json --show-tests=all --debug something.json -- toosmall.json -- { @@ -31,6 +45,9 @@ exec fflint json --schema=object.schema.json --show-tests=all --debug object.jso "cd": false, } +-- number.json -- +123 + -- object.schema.json -- { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -38,4 +55,22 @@ exec fflint json --schema=object.schema.json --show-tests=all --debug object.jso "title": "Product", "description": "A product in the catalog", "type": "object" +} + +-- string.schema.json -- +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/product.schema.json", + "title": "Strings", + "description": "A string", + "type": "string" +} + +-- number.schema.json -- +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/product.schema.json", + "title": "Numbers", + "description": "A number", + "type": "number" } \ No newline at end of file