From 02e96220b63574327fbec420fa7ec5a63d369af5 Mon Sep 17 00:00:00 2001 From: Tigran Najaryan <4194920+tigrannajaryan@users.noreply.github.com> Date: Wed, 16 Feb 2022 16:37:44 -0500 Subject: [PATCH] Add schema file tool (#83) This can be used in specification repo to validate the schema files. --- .github/workflows/schema_tools.yml | 41 +++++++ schemas/Dockerfile | 25 ++++ schemas/README.md | 35 ++++++ schemas/go.mod | 16 +++ schemas/go.sum | 17 +++ schemas/main.go | 108 ++++++++++++++++++ schemas/main_test.go | 32 ++++++ schemas/testdata/1.9.0 | 15 +++ .../testdata/invalid_fileformatnumber.yaml | 15 +++ schemas/testdata/invalid_host.yaml | 15 +++ schemas/testdata/invalid_missingver.yaml | 14 +++ schemas/testdata/invalid_scheme.yaml | 15 +++ schemas/testdata/invalid_url.yaml | 15 +++ schemas/testdata/invalid_ver_too_new.yaml | 16 +++ schemas/testdata/invalid_verinurl.yaml | 15 +++ 15 files changed, 394 insertions(+) create mode 100644 .github/workflows/schema_tools.yml create mode 100644 schemas/Dockerfile create mode 100644 schemas/README.md create mode 100644 schemas/go.mod create mode 100644 schemas/go.sum create mode 100644 schemas/main.go create mode 100644 schemas/main_test.go create mode 100644 schemas/testdata/1.9.0 create mode 100644 schemas/testdata/invalid_fileformatnumber.yaml create mode 100644 schemas/testdata/invalid_host.yaml create mode 100644 schemas/testdata/invalid_missingver.yaml create mode 100644 schemas/testdata/invalid_scheme.yaml create mode 100644 schemas/testdata/invalid_url.yaml create mode 100644 schemas/testdata/invalid_ver_too_new.yaml create mode 100644 schemas/testdata/invalid_verinurl.yaml diff --git a/.github/workflows/schema_tools.yml b/.github/workflows/schema_tools.yml new file mode 100644 index 00000000..53b1ef69 --- /dev/null +++ b/.github/workflows/schema_tools.yml @@ -0,0 +1,41 @@ +name: Schema File Tools Docker Image +on: + push: + tags: [ '**' ] + branches: [ main ] + pull_request: + branches: [ main ] + paths: + - .github/workflows/schema_tools.yml + - schemas/* + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + + - name: Build the Docker image + run: docker build schemas/. -t build-tool-schemas + + - name: Login to GitHub Package Registry + if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/') + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Push the Docker image + if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/') + run: | + function tag_and_push { + docker tag build-tool-schemas "otel/build-tool-schemas:${1}" && docker push "otel/build-tool-schemas:${1}" + } + if [[ "${GITHUB_REF}" == "refs/heads/main" ]]; then + tag_and_push "latest" + elif [[ "${GITHUB_REF}" =~ refs/tags/v[0-9]+\.[0-9]+\.[0-9]+ ]]; then + TAG="${GITHUB_REF#"refs/tags/v"}" + tag_and_push "${TAG}" + else + tag_and_push "${GITHUB_REF#"refs/tags/"}" + fi diff --git a/schemas/Dockerfile b/schemas/Dockerfile new file mode 100644 index 00000000..10f0d443 --- /dev/null +++ b/schemas/Dockerfile @@ -0,0 +1,25 @@ +## Build +## +FROM golang:1.17-alpine AS build + +WORKDIR /app + +COPY go.mod ./ +COPY go.sum ./ +COPY *.go ./ +COPY testdata/* ./testdata/ + +RUN CGO_ENABLED=0 go test ./... +RUN go build -o /schemas-tool + +## Run +## +FROM alpine:3.13 + +WORKDIR / + +COPY --from=build /schemas-tool /schemas-tool + +USER nonroot:nonroot + +ENTRYPOINT ["/schemas-tool"] \ No newline at end of file diff --git a/schemas/README.md b/schemas/README.md new file mode 100644 index 00000000..2362d956 --- /dev/null +++ b/schemas/README.md @@ -0,0 +1,35 @@ +# Schema File Tool + +A lightweight schema tool Docker image, published as otel/build-tool-schemas to Docker Hub. + +## Usage + +Command line options: +--file path to the schema file to check +--version expected schema version number. Optional. If provided the schema version is checked inside the file. + +To check that a file is a valid Schema file do this: + +```bash +docker run --rm -v: -w otel/build-tool-schemas [OPTION] --file=/ [other options] +``` + +For help try: + +```bash +docker run --rm otel/build-tool-schemas --help +``` + +## Contributing + +To build the Docker image locally run: + +```bash +docker build schemas/. -t build-tool-schemas +``` + +To run the Docker image locally and check schema file version 1.9.0 do this: + +```bash +docker run -v=/your-path-to-spec-repo/opentelemetry-specification/schemas/:/schemas build-tool-schemas --file /schemas/1.9.0 --version=1.9.0 +``` diff --git a/schemas/go.mod b/schemas/go.mod new file mode 100644 index 00000000..ed912669 --- /dev/null +++ b/schemas/go.mod @@ -0,0 +1,16 @@ +module github.com/open-telemetry/build-tools/schemas + +go 1.17 + +require ( + github.com/Masterminds/semver/v3 v3.1.1 + github.com/stretchr/testify v1.7.0 + go.opentelemetry.io/otel/schema v0.0.2 +) + +require ( + github.com/davecgh/go-spew v1.1.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect +) diff --git a/schemas/go.sum b/schemas/go.sum new file mode 100644 index 00000000..ecde536b --- /dev/null +++ b/schemas/go.sum @@ -0,0 +1,17 @@ +github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +go.opentelemetry.io/otel/schema v0.0.2 h1:PqadNg9PGGI2/9F9yuvOglu868Hded24v9yOJ2dlBWc= +go.opentelemetry.io/otel/schema v0.0.2/go.mod h1:knHeVSWnzKRTcznXuwsuWCvO4sGybDbj/o7IKnG36xQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/schemas/main.go b/schemas/main.go new file mode 100644 index 00000000..a4d770b6 --- /dev/null +++ b/schemas/main.go @@ -0,0 +1,108 @@ +package main + +import ( + "flag" + "fmt" + "net/url" + "os" + + "github.com/Masterminds/semver/v3" + schema "go.opentelemetry.io/otel/schema/v1.0" + "go.opentelemetry.io/otel/schema/v1.0/types" +) + +var schemaFilePath = flag.String("file", "", "Input schema file path") +var schemaVersion = flag.String("version", "", "Expected schema version (optional)") + +func loadSchemaFromFile(schemaFilePath string, schemaVersion string) error { + // Parse the schema file. + telSchema, err := schema.ParseFile(schemaFilePath) + if err != nil { + return err + } + + // We only support a specific format version. + if telSchema.FileFormat != "1.0.0" { + return fmt.Errorf("incorrect schema file format version: %s", telSchema.FileFormat) + } + + // Check the schema URL. + u, err := url.Parse(telSchema.SchemaURL) + if err != nil { + return err + } + + const expectedScheme = "https" + const expectedHost = "opentelemetry.io" + + if u.Scheme != expectedScheme { + return fmt.Errorf("invalid scheme in: %s, expected %s", telSchema.SchemaURL, expectedScheme) + } + + if u.Host != expectedHost { + return fmt.Errorf("invalid host name in: %s, expected %s", telSchema.SchemaURL, expectedHost) + } + + if schemaVersion != "" { + // Version check is requested. + + // Check the URL first. + expectedURL := fmt.Sprintf("https://opentelemetry.io/schemas/%s", schemaVersion) + if telSchema.SchemaURL != expectedURL { + return fmt.Errorf("invalid Schema URL: expected %s, got %s", telSchema.SchemaURL, expectedURL) + } + + // Ensure the version exists in the file. + _, exists := telSchema.Versions[types.TelemetryVersion(schemaVersion)] + if !exists { + return fmt.Errorf("%s does not exist in 'versions' section", schemaVersion) + } + + thisVer, err := semver.StrictNewVersion(schemaVersion) + if err != nil { + return fmt.Errorf( + "invalid version number %s in the schema file: %w", + schemaVersion, err, + ) + } + + // Ensure no other version is newer than the sceham file version. + for ver := range telSchema.Versions { + parsedVer, err := semver.StrictNewVersion(string(ver)) + if err != nil { + return fmt.Errorf( + "invalid version number %s in the schema file: %w", + ver, err, + ) + } + + if parsedVer.GreaterThan(thisVer) { + return fmt.Errorf( + "found version number %s in the schema file which is greater than the schema file version %s", + ver, thisVer.String(), + ) + } + } + } + + return nil +} + +func main() { + flag.Parse() + + if schemaFilePath == nil || *schemaFilePath == "" { + flag.PrintDefaults() + os.Exit(1) + } + + fmt.Printf("Checking schema file %s...", *schemaFilePath) + + err := loadSchemaFromFile(*schemaFilePath, *schemaVersion) + if err != nil { + fmt.Printf("\nSchema file is not valid: %v\n", err) + os.Exit(2) + } + + fmt.Println(" File is valid.") +} diff --git a/schemas/main_test.go b/schemas/main_test.go new file mode 100644 index 00000000..116052f6 --- /dev/null +++ b/schemas/main_test.go @@ -0,0 +1,32 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestValidSchema(t *testing.T) { + err := loadSchemaFromFile("testdata/1.9.0", "") + assert.NoError(t, err) + + err = loadSchemaFromFile("testdata/1.9.0", "1.9.0") + assert.NoError(t, err) +} + +func TestInvalidSchemas(t *testing.T) { + files := []string{ + "invalid_fileformatnumber.yaml", + "invalid_url.yaml", + "invalid_scheme.yaml", + "invalid_host.yaml", + "invalid_verinurl.yaml", + "invalid_missingver.yaml", + "invalid_ver_too_new.yaml", + } + + for _, file := range files { + err := loadSchemaFromFile("testdata/"+file, "1.9.0") + assert.Error(t, err) + } +} diff --git a/schemas/testdata/1.9.0 b/schemas/testdata/1.9.0 new file mode 100644 index 00000000..99b46783 --- /dev/null +++ b/schemas/testdata/1.9.0 @@ -0,0 +1,15 @@ +file_format: 1.0.0 +schema_url: https://opentelemetry.io/schemas/1.9.0 +versions: + 1.9.0: + 1.8.0: + spans: + changes: + - rename_attributes: + attribute_map: + db.cassandra.keyspace: db.name + db.hbase.namespace: db.name + 1.7.0: + 1.6.1: + 1.5.0: + 1.4.0: diff --git a/schemas/testdata/invalid_fileformatnumber.yaml b/schemas/testdata/invalid_fileformatnumber.yaml new file mode 100644 index 00000000..e926b93a --- /dev/null +++ b/schemas/testdata/invalid_fileformatnumber.yaml @@ -0,0 +1,15 @@ +file_format: 2.0.0 +schema_url: https://opentelemetry.io/schemas/1.9.0 +versions: + 1.9.0: + 1.8.0: + spans: + changes: + - rename_attributes: + attribute_map: + db.cassandra.keyspace: db.name + db.hbase.namespace: db.name + 1.7.0: + 1.6.1: + 1.5.0: + 1.4.0: diff --git a/schemas/testdata/invalid_host.yaml b/schemas/testdata/invalid_host.yaml new file mode 100644 index 00000000..ed26c7bf --- /dev/null +++ b/schemas/testdata/invalid_host.yaml @@ -0,0 +1,15 @@ +file_format: 1.0.0 +schema_url: https://opentelemetry.com/schemas/1.9.0 +versions: + 1.9.0: + 1.8.0: + spans: + changes: + - rename_attributes: + attribute_map: + db.cassandra.keyspace: db.name + db.hbase.namespace: db.name + 1.7.0: + 1.6.1: + 1.5.0: + 1.4.0: diff --git a/schemas/testdata/invalid_missingver.yaml b/schemas/testdata/invalid_missingver.yaml new file mode 100644 index 00000000..aee9d166 --- /dev/null +++ b/schemas/testdata/invalid_missingver.yaml @@ -0,0 +1,14 @@ +file_format: 1.0.0 +schema_url: https://opentelemetry.io/schemas/1.9.0 +versions: + 1.8.0: + spans: + changes: + - rename_attributes: + attribute_map: + db.cassandra.keyspace: db.name + db.hbase.namespace: db.name + 1.7.0: + 1.6.1: + 1.5.0: + 1.4.0: diff --git a/schemas/testdata/invalid_scheme.yaml b/schemas/testdata/invalid_scheme.yaml new file mode 100644 index 00000000..852f0692 --- /dev/null +++ b/schemas/testdata/invalid_scheme.yaml @@ -0,0 +1,15 @@ +file_format: 1.0.0 +schema_url: http://opentelemetry.io/schemas/1.9.0 +versions: + 1.9.0: + 1.8.0: + spans: + changes: + - rename_attributes: + attribute_map: + db.cassandra.keyspace: db.name + db.hbase.namespace: db.name + 1.7.0: + 1.6.1: + 1.5.0: + 1.4.0: diff --git a/schemas/testdata/invalid_url.yaml b/schemas/testdata/invalid_url.yaml new file mode 100644 index 00000000..f51f42aa --- /dev/null +++ b/schemas/testdata/invalid_url.yaml @@ -0,0 +1,15 @@ +file_format: 1.0.0 +schema_url: https//opentelemetry.io/schemas/1.9.0 +versions: + 1.9.0: + 1.8.0: + spans: + changes: + - rename_attributes: + attribute_map: + db.cassandra.keyspace: db.name + db.hbase.namespace: db.name + 1.7.0: + 1.6.1: + 1.5.0: + 1.4.0: diff --git a/schemas/testdata/invalid_ver_too_new.yaml b/schemas/testdata/invalid_ver_too_new.yaml new file mode 100644 index 00000000..5cf42d40 --- /dev/null +++ b/schemas/testdata/invalid_ver_too_new.yaml @@ -0,0 +1,16 @@ +file_format: 1.0.0 +schema_url: https://opentelemetry.io/schemas/1.9.0 +versions: + 1.10.0: + 1.9.0: + 1.8.0: + spans: + changes: + - rename_attributes: + attribute_map: + db.cassandra.keyspace: db.name + db.hbase.namespace: db.name + 1.7.0: + 1.6.1: + 1.5.0: + 1.4.0: diff --git a/schemas/testdata/invalid_verinurl.yaml b/schemas/testdata/invalid_verinurl.yaml new file mode 100644 index 00000000..b84f2ca4 --- /dev/null +++ b/schemas/testdata/invalid_verinurl.yaml @@ -0,0 +1,15 @@ +file_format: 1.0.0 +schema_url: https://opentelemetry.io/schemas/1.8.0 +versions: + 1.9.0: + 1.8.0: + spans: + changes: + - rename_attributes: + attribute_map: + db.cassandra.keyspace: db.name + db.hbase.namespace: db.name + 1.7.0: + 1.6.1: + 1.5.0: + 1.4.0: