From 61d376a7abdc83aa76bf1f93e7a99adf64d833d0 Mon Sep 17 00:00:00 2001 From: Aleksandr Razumov Date: Wed, 6 Dec 2023 12:30:39 +0300 Subject: [PATCH 1/5] chore: vendor promcompliance --- .codecov.yml | 2 + .gitmodules | 3 - .golangci.yml | 10 +- dev/local/ch-compliance/README.md | 4 +- dev/local/ch-compliance/compliance | 1 - dev/local/ch-compliance/run.sh | 4 +- .../{test.oteldb.yml => test-oteldb.yml} | 2 +- go.mod | 13 +- go.sum | 8 + internal/promcompliance/.gitignore | 17 ++ internal/promcompliance/Dockerfile | 14 + internal/promcompliance/LICENSE | 201 ++++++++++++++ internal/promcompliance/Makefile | 3 + internal/promcompliance/NOTICE | 1 + internal/promcompliance/README.md | 100 +++++++ .../cmd/promql-compliance-tester/main.go | 185 +++++++++++++ internal/promcompliance/comparer/comparer.go | 193 ++++++++++++++ internal/promcompliance/config/config.go | 89 +++++++ .../promcompliance/output/example-output.html | 60 +++++ internal/promcompliance/output/html.go | 64 +++++ internal/promcompliance/output/json.go | 23 ++ internal/promcompliance/output/outputter.go | 9 + internal/promcompliance/output/text.go | 60 +++++ internal/promcompliance/output/tsv.go | 40 +++ .../promcompliance/prometheus-test-data.yml | 10 + .../promcompliance/promql-test-queries.yml | 245 ++++++++++++++++++ internal/promcompliance/test-amp.yml | 11 + internal/promcompliance/test-chronosphere.yml | 7 + internal/promcompliance/test-cortex.yml | 5 + internal/promcompliance/test-gmp.yml | 15 ++ .../promcompliance/test-grafana-cloud.yml | 7 + internal/promcompliance/test-m3.yml | 5 + internal/promcompliance/test-new-relic.yml | 17 ++ internal/promcompliance/test-promscale.yml | 5 + internal/promcompliance/test-sysdig.yml | 15 ++ internal/promcompliance/test-thanos.yml | 11 + .../promcompliance/test-victoriametrics.yml | 9 + internal/promcompliance/test-wavefront.yml | 11 + internal/promcompliance/testcases/expand.go | 154 +++++++++++ 39 files changed, 1619 insertions(+), 14 deletions(-) delete mode 160000 dev/local/ch-compliance/compliance rename dev/local/ch-compliance/{test.oteldb.yml => test-oteldb.yml} (93%) create mode 100644 internal/promcompliance/.gitignore create mode 100644 internal/promcompliance/Dockerfile create mode 100644 internal/promcompliance/LICENSE create mode 100644 internal/promcompliance/Makefile create mode 100644 internal/promcompliance/NOTICE create mode 100644 internal/promcompliance/README.md create mode 100644 internal/promcompliance/cmd/promql-compliance-tester/main.go create mode 100644 internal/promcompliance/comparer/comparer.go create mode 100644 internal/promcompliance/config/config.go create mode 100644 internal/promcompliance/output/example-output.html create mode 100644 internal/promcompliance/output/html.go create mode 100644 internal/promcompliance/output/json.go create mode 100644 internal/promcompliance/output/outputter.go create mode 100644 internal/promcompliance/output/text.go create mode 100644 internal/promcompliance/output/tsv.go create mode 100644 internal/promcompliance/prometheus-test-data.yml create mode 100644 internal/promcompliance/promql-test-queries.yml create mode 100644 internal/promcompliance/test-amp.yml create mode 100644 internal/promcompliance/test-chronosphere.yml create mode 100644 internal/promcompliance/test-cortex.yml create mode 100644 internal/promcompliance/test-gmp.yml create mode 100644 internal/promcompliance/test-grafana-cloud.yml create mode 100644 internal/promcompliance/test-m3.yml create mode 100644 internal/promcompliance/test-new-relic.yml create mode 100644 internal/promcompliance/test-promscale.yml create mode 100644 internal/promcompliance/test-sysdig.yml create mode 100644 internal/promcompliance/test-thanos.yml create mode 100644 internal/promcompliance/test-victoriametrics.yml create mode 100644 internal/promcompliance/test-wavefront.yml create mode 100644 internal/promcompliance/testcases/expand.go diff --git a/.codecov.yml b/.codecov.yml index 26596983..14a1c21e 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -6,6 +6,8 @@ ignore: # Skip non-production code. - "internal/lokiproxy/" - "internal/pyroproxy/" + # Skip vendored-forked + - "internal/promcompliance/" coverage: status: project: false diff --git a/.gitmodules b/.gitmodules index 49848d4d..8f1342ad 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,6 +4,3 @@ [submodule "dev/local/ch-full/opentelemetry-collector-contrib"] path = dev/local/ch-full/opentelemetry-collector-contrib url = https://github.com/open-telemetry/opentelemetry-collector-contrib.git -[submodule "dev/local/ch-compliance/compliance"] - path = dev/local/ch-compliance/compliance - url = https://github.com/prometheus/compliance.git diff --git a/.golangci.yml b/.golangci.yml index a4600842..bc932d7f 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -116,8 +116,16 @@ issues: text: "ST1003" # underscores lol - path: (internal|cmd) + linters: [revive, stylecheck] + text: "comment" + + - path: 'internal/promcompliance' linters: [revive] - text: "package-comments" + text: "exported:" + + - path: 'internal/promcompliance' + linters: [gocritic] + text: "ifElseChain:" - linters: [revive] text: "comment on exported const .+ should be of the form" diff --git a/dev/local/ch-compliance/README.md b/dev/local/ch-compliance/README.md index 8aec2735..bfc38f61 100644 --- a/dev/local/ch-compliance/README.md +++ b/dev/local/ch-compliance/README.md @@ -9,11 +9,11 @@ cd ./compliance/promql && go install ./cmd/promql-compliance-tester && cd - To run with targets in docker-compose: ```console -promql-compliance-tester -config-file promql-test-queries.yml -config-file test.local.yml +promql-compliance-tester -config-file promql-test-queries.yml -config-file test-oteldb.yml ``` **NOTE:** -Results will be false-positive until enough data (5-10min?) is gathered. +Results will be false-positive until enough data (~20min) is gathered. This check was disabled as being broken on latest prometheus reference: ```yaml diff --git a/dev/local/ch-compliance/compliance b/dev/local/ch-compliance/compliance deleted file mode 160000 index 12cbdf92..00000000 --- a/dev/local/ch-compliance/compliance +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 12cbdf92abf7737531871ab7620a2de965fc5382 diff --git a/dev/local/ch-compliance/run.sh b/dev/local/ch-compliance/run.sh index 546d3b6e..2f9b5819 100755 --- a/dev/local/ch-compliance/run.sh +++ b/dev/local/ch-compliance/run.sh @@ -2,12 +2,12 @@ set -e -x -cd ./compliance/promql && go install ./cmd/promql-compliance-tester && cd - +go install github.com/go-faster/oteldb/internal/promcompliance/cmd/promql-compliance-tester docker compose up -d --remove-orphans --build --force-recreate go run ./cmd/compliance-wait echo ">> Testing oteldb implementation" -promql-compliance-tester -config-file promql-test-queries.yml -config-file test.oteldb.yml | tee result.oteldb.txt || true +promql-compliance-tester -config-file promql-test-queries.yml -config-file test-oteldb.yml | tee result.oteldb.txt || true docker compose down -v diff --git a/dev/local/ch-compliance/test.oteldb.yml b/dev/local/ch-compliance/test-oteldb.yml similarity index 93% rename from dev/local/ch-compliance/test.oteldb.yml rename to dev/local/ch-compliance/test-oteldb.yml index 0cdfc9e6..5bbd833e 100644 --- a/dev/local/ch-compliance/test.oteldb.yml +++ b/dev/local/ch-compliance/test-oteldb.yml @@ -1,4 +1,4 @@ -# promql-compliance-tester -config-file promql-test-queries.yml -config-file test.oteldb.yml +# promql-compliance-tester -config-file promql-test-queries.yml -config-file test-oteldb.yml reference_target_config: query_url: http://localhost:9091 diff --git a/go.mod b/go.mod index 83cdde67..2ccc92d0 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/Masterminds/sprig/v3 v3.2.3 github.com/cenkalti/backoff/v4 v4.2.1 github.com/cespare/xxhash/v2 v2.2.0 + github.com/cheggaaa/pb/v3 v3.1.4 github.com/dustin/go-humanize v1.0.1 github.com/go-faster/errors v0.7.0 github.com/go-faster/jx v1.1.0 @@ -14,12 +15,15 @@ require ( github.com/go-logfmt/logfmt v0.6.0 github.com/gogo/protobuf v1.3.2 github.com/golang/snappy v0.0.4 + github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.4.0 github.com/grafana/pyroscope-go v1.0.4 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 github.com/ogen-go/ogen v0.79.1 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.90.1 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus v0.90.1 + github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.17.0 github.com/prometheus/common v0.45.0 github.com/prometheus/prometheus v0.48.0 github.com/stretchr/testify v1.8.4 @@ -49,6 +53,7 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.21.0 go.opentelemetry.io/otel/trace v1.21.0 go.opentelemetry.io/proto/otlp v1.0.0 + go.uber.org/atomic v1.11.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.26.0 go4.org/netipx v0.0.0-20230303233057-f1b76eb4bb35 @@ -56,6 +61,7 @@ require ( golang.org/x/sync v0.5.0 golang.org/x/tools v0.16.0 google.golang.org/grpc v1.59.0 + gopkg.in/yaml.v2 v2.4.0 sigs.k8s.io/yaml v1.4.0 ) @@ -71,6 +77,7 @@ require ( github.com/Masterminds/semver/v3 v3.2.0 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/Microsoft/hcsshim v0.11.1 // indirect + github.com/VividCortex/ewma v1.2.0 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/aws/aws-sdk-go v1.45.25 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -119,6 +126,7 @@ require ( github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect @@ -138,14 +146,13 @@ require ( github.com/pascaldekloe/name v1.0.1 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect - github.com/prometheus/client_golang v1.17.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common/sigv4 v0.1.0 // indirect github.com/prometheus/procfs v0.11.1 // indirect github.com/prometheus/statsd_exporter v0.22.7 // indirect + github.com/rivo/uniseg v0.2.0 // indirect github.com/rs/cors v1.10.1 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/shirou/gopsutil/v3 v3.23.10 // indirect @@ -185,7 +192,6 @@ require ( go.opentelemetry.io/otel/exporters/prometheus v0.44.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.44.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.21.0 // indirect - go.uber.org/atomic v1.11.0 // indirect go.uber.org/automaxprocs v1.5.3 // indirect go.uber.org/goleak v1.3.0 // indirect golang.org/x/crypto v0.16.0 // indirect @@ -199,7 +205,6 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20231012201019-e917dd12ba7a // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect google.golang.org/protobuf v1.31.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index e1a33a5c..b4816f6c 100644 --- a/go.sum +++ b/go.sum @@ -40,6 +40,8 @@ github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5 github.com/Microsoft/hcsshim v0.11.1 h1:hJ3s7GbWlGK4YVV92sO88BQSyF4ZLVy7/awqOlPxFbA= github.com/Microsoft/hcsshim v0.11.1/go.mod h1:nFJmaO4Zr5Y7eADdFOpYswDDlNVbvcIJJNJLECr5JQg= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -81,6 +83,8 @@ github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= +github.com/cheggaaa/pb/v3 v3.1.4 h1:DN8j4TVVdKu3WxVwcRKu0sG00IIU6FewoABZzXbRQeo= +github.com/cheggaaa/pb/v3 v3.1.4/go.mod h1:6wVjILNBaXMs8c21qRiaUM8BR82erfgau1DQ4iUXmSA= github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -412,6 +416,8 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= +github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= @@ -546,6 +552,8 @@ github.com/prometheus/prometheus v0.48.0/go.mod h1:SRw624aMAxTfryAcP8rOjg4S/sHHa github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT1pX2CziuyQR0= github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI= github.com/rhnvrm/simples3 v0.6.1/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= diff --git a/internal/promcompliance/.gitignore b/internal/promcompliance/.gitignore new file mode 100644 index 00000000..fceeef01 --- /dev/null +++ b/internal/promcompliance/.gitignore @@ -0,0 +1,17 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +/promql-compliance-tester diff --git a/internal/promcompliance/Dockerfile b/internal/promcompliance/Dockerfile new file mode 100644 index 00000000..8da27213 --- /dev/null +++ b/internal/promcompliance/Dockerfile @@ -0,0 +1,14 @@ +FROM golang:1.18 as build-env + +WORKDIR /go/src/promql +COPY . /go/src/promql + +ENV CGO_ENABLED 0 + +RUN go build ./cmd/promql-compliance-tester + +FROM quay.io/prometheus/busybox +COPY --from=build-env /go/src/promql/promql-compliance-tester / +COPY --from=build-env /go/src/promql/promql-test-queries.yml / + +ENTRYPOINT ["/promql-compliance-tester"] diff --git a/internal/promcompliance/LICENSE b/internal/promcompliance/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/internal/promcompliance/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/internal/promcompliance/Makefile b/internal/promcompliance/Makefile new file mode 100644 index 00000000..a7b3db8e --- /dev/null +++ b/internal/promcompliance/Makefile @@ -0,0 +1,3 @@ +.PHONY: docker +docker: + docker build . -t promql-compliance-tester:latest diff --git a/internal/promcompliance/NOTICE b/internal/promcompliance/NOTICE new file mode 100644 index 00000000..9ef83ada --- /dev/null +++ b/internal/promcompliance/NOTICE @@ -0,0 +1 @@ +Forked from https://github.com/prometheus/compliance \ No newline at end of file diff --git a/internal/promcompliance/README.md b/internal/promcompliance/README.md new file mode 100644 index 00000000..1529b3be --- /dev/null +++ b/internal/promcompliance/README.md @@ -0,0 +1,100 @@ +# PromQL Compliance Tester + +The PromQL Compliance Tester is a tool for running comparison tests between native Prometheus and vendor PromQL API implementations. + +The tool was [first published and described](https://promlabs.com/blog/2020/08/06/comparing-promql-correctness-across-vendors) in August 2020. [Test results have been published](https://promlabs.com/promql-compliance-tests) on 2020-08-06 and 2020-12-01. + +## Building via Docker + +If you have docker installed, you can build the tool using docker. + +```bash +make docker +``` + +## Building from source + +### Requirements + +This tool is written in Go and requires a working Go setup to build. Library dependencies are handled via [Go Modules](https://blog.golang.org/using-go-modules). + +### Building + +To build the tool: + +```bash +go build ./cmd/promql-compliance-tester +``` + +## Executing + +The tool allows setting the following flags: + +``` +$ ./promql-compliance-tester -h +Usage of ./promql-compliance-tester: + -config-file value + The path to the configuration file. If repeated, the specified files will be concatenated before YAML parsing. + -output-format string + The comparison output format. Valid values: [text, html, json] (default "text") + -output-html-template string + The HTML template to use when using HTML as the output format. (default "./output/example-output.html") + -output-passing + Whether to also include passing test cases in the output. + -query-parallelism int + Maximum number of comparison queries to run in parallel. (default 20) +``` + +Running the tool will execute all test cases in `-config-file` and compare results between reference and target provided in the same file. + +At the end of the run, the output is provided in the form of the number of executed tests and errors if any. Example output can be seen here: + +```bash +./promql-compliance-tester -config-file config.yaml -config-file ./promql-test-queries.yml +529 / 529 [-----------------------------------------------------------------------------------------------------------] 100.00% 278 p/s +Total: 529 / 529 (100.00%) passed, 0 unsupported +``` + +If all tests were executed correctly and passing the tool returns a 0 exit code, otherwise it returns 1. + +## Configuration + +A standard suite of test cases is defined in the [`promql-test-queries.yml`](./promql-test-queries.yml) file, while separate `test-.yml` config files specify test target configurations and query tweaks for a number of individual projects and vendors. To run the tester tool, you need to specify both the test suite config file as well as a config file for a single vendor. + +For example, to run the tester against Cortex: + +```bash +./promql-compliance-tester -config-file=promql-test-queries.yml -config-file=test-cortex.yml +``` + +Note that some of the vendor-specific configuration files require you to replace certain placeholder values for endpoints and credentials before using them. + +## Testing your implementation for compliance + +We encourage projects and vendors to test their implementations for PromQL compliance. To do this, follow these steps: + +1. Check out this repository: `git clone git@github.com:prometheus/compliance`. +2. Change into the repo's `promql` directory: `cd compliance/promql`. +3. Either edit the appropriate `test-.yml` file for your project or service or create a new test target configuration file to be able to query from both your reference Prometheus server and your PromQL-compatible datasource. +4. Edit `prometheus-test-data.yml` to either add a `remote_write` section for your system or make any other adjustments that are necessary to enable propagation of the scraped data to your system (e.g. adding external labels for Thanos). +5. Run a reference Prometheus server that ingests the expected test data (we assume that you have Prometheus installed): `prometheus --config.file=prometheus-test-data.yml`. +6. Wait for at least one hour for sufficient test data to be ingested into both the reference Prometheus server and the system to be tested. +7. Build the tester tool: `go build ./cmd/promql-compliance-tester`. +8. Run the tester tool (replacing `` as appropriate): `./promql-compliance-tester -config-file=promql-test-queries.yml -config-file=test-.yml`. + +If the tool reports a test score of 100% without any cross-cutting query tweaks, your implementation is PromQL-compliant. + +## Contributing + +Help is wanted to improve the PromQL Compliance Tester. In particular, we would love to add and improve the following points: + +* Test instant queries in addition to range queries. +* Add more variation and configurability to input timestamps. +* Flesh out a more comprehensive (and less overlapping) set of input test queries. +* Automate and integrate data loading into different systems. +* Test more vendor implementations of PromQL. +* Version test results and make pretty output presentations easier. + +**Note:** Many people will be interested in benchmarking performance differences between PromQL implementations. While this is important as well, the PromQL Compliance Tester focuses solely on correctness testing. Please contact the [maintainers](../MAINTAINERS.md) if you want to work on performance testing. + +If you would like to help flesh out the tester, please [file issues](https://github.com/prometheus/compliance/issues) or [pull requests](https://github.com/prometheus/compliance/pulls). diff --git a/internal/promcompliance/cmd/promql-compliance-tester/main.go b/internal/promcompliance/cmd/promql-compliance-tester/main.go new file mode 100644 index 00000000..4dce55c5 --- /dev/null +++ b/internal/promcompliance/cmd/promql-compliance-tester/main.go @@ -0,0 +1,185 @@ +package main + +import ( + "flag" + "log" + "math" + "net/http" + "os" + "strconv" + "strings" + "sync" + "time" + + "github.com/cheggaaa/pb/v3" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/api" + v1 "github.com/prometheus/client_golang/api/prometheus/v1" + "go.uber.org/atomic" + + "github.com/go-faster/oteldb/internal/promcompliance/comparer" + "github.com/go-faster/oteldb/internal/promcompliance/config" + "github.com/go-faster/oteldb/internal/promcompliance/output" + "github.com/go-faster/oteldb/internal/promcompliance/testcases" +) + +func newPromAPI(targetConfig config.TargetConfig) (v1.API, error) { + apiConfig := api.Config{Address: targetConfig.QueryURL} + if len(targetConfig.Headers) > 0 || targetConfig.BasicAuthUser != "" { + apiConfig.RoundTripper = roundTripperWithSettings{headers: targetConfig.Headers, basicAuthUser: targetConfig.BasicAuthUser, basicAuthPass: targetConfig.BasicAuthPass} + } + client, err := api.NewClient(apiConfig) + if err != nil { + return nil, errors.Wrapf(err, "creating Prometheus API client for %q: %v", targetConfig.QueryURL, err) + } + + return v1.NewAPI(client), nil +} + +type roundTripperWithSettings struct { + headers map[string]string + basicAuthUser string + basicAuthPass string +} + +func (rt roundTripperWithSettings) RoundTrip(req *http.Request) (*http.Response, error) { + // Per RoundTrip's documentation, RoundTrip should not modify the request, + // except for consuming and closing the Request's Body. + // TODO: Update the Go Prometheus client code to support adding headers to request. + + if rt.basicAuthUser != "" { + req.SetBasicAuth(rt.basicAuthUser, rt.basicAuthPass) + } + + for key, value := range rt.headers { + if strings.EqualFold(key, "host") { + req.Host = value + } else { + req.Header.Add(key, value) + } + } + return http.DefaultTransport.RoundTrip(req) +} + +type arrayFlags []string + +func (i *arrayFlags) String() string { + return "my string representation" +} + +func (i *arrayFlags) Set(value string) error { + *i = append(*i, value) + return nil +} + +func main() { + var configFiles arrayFlags + flag.Var(&configFiles, "config-file", "The path to the configuration file. If repeated, the specified files will be concatenated before YAML parsing.") + outputFormat := flag.String("output-format", "text", "The comparison output format. Valid values: [text, html, json]") + outputHTMLTemplate := flag.String("output-html-template", "./output/example-output.html", "The HTML template to use when using HTML as the output format.") + outputPassing := flag.Bool("output-passing", false, "Whether to also include passing test cases in the output.") + queryParallelism := flag.Int("query-parallelism", 20, "Maximum number of comparison queries to run in parallel.") + flag.Parse() + + var outp output.Outputter + switch *outputFormat { + case "text": + outp = output.Text + case "html": + var err error + outp, err = output.HTML(*outputHTMLTemplate) + if err != nil { + log.Fatalf("Error reading output HTML template: %v", err) + } + case "json": + outp = output.JSON + case "tsv": + outp = output.TSV + default: + log.Fatalf("Invalid output format %q", *outputFormat) + } + + cfg, err := config.LoadFromFiles(configFiles) + if err != nil { + log.Fatalf("Error loading configuration file: %v", err) + } + refAPI, err := newPromAPI(cfg.ReferenceTargetConfig) + if err != nil { + log.Fatalf("Error creating reference API: %v", err) + } + testAPI, err := newPromAPI(cfg.TestTargetConfig) + if err != nil { + log.Fatalf("Error creating test API: %v", err) + } + + comp := comparer.New(refAPI, testAPI, cfg.QueryTweaks) + + end := getTime(cfg.QueryTimeParameters.EndTime, time.Now().UTC().Add(-12*time.Minute)) + start := end.Add( + -getNonZeroDuration(cfg.QueryTimeParameters.RangeInSeconds, 10*time.Minute)) + resolution := getNonZeroDuration( + cfg.QueryTimeParameters.ResolutionInSeconds, 10*time.Second) + expandedTestCases := testcases.ExpandTestCases(cfg.TestCases, cfg.QueryTweaks, start, end, resolution) + + var wg sync.WaitGroup + results := make([]*comparer.Result, len(expandedTestCases)) + progressBar := pb.StartNew(len(results)) + wg.Add(len(results)) + + workCh := make(chan struct{}, *queryParallelism) + + allSuccess := atomic.NewBool(true) + for i, tc := range expandedTestCases { + workCh <- struct{}{} + + go func(i int, tc *comparer.TestCase) { + res, err := comp.Compare(tc) + if err != nil { + log.Fatalf("Error running comparison: %v", err) + } + results[i] = res + if !res.Success() { + allSuccess.Store(false) + } + progressBar.Increment() + <-workCh + wg.Done() + }(i, tc) + } + + wg.Wait() + progressBar.Finish() + + outp(results, *outputPassing, cfg.QueryTweaks) + + if !allSuccess.Load() { + os.Exit(1) + } +} + +func getTime(timeStr string, defaultTime time.Time) time.Time { + result, err := parseTime(timeStr) + if err != nil { + return defaultTime + } + return result +} + +func getNonZeroDuration( + seconds float64, defaultDuration time.Duration) time.Duration { + if seconds == 0.0 { + return defaultDuration + } + return time.Duration(seconds * float64(time.Second)) +} + +func parseTime(s string) (time.Time, error) { + if t, err := strconv.ParseFloat(s, 64); err == nil { + s, ns := math.Modf(t) + return time.Unix(int64(s), int64(ns*float64(time.Second))).UTC(), nil + } + if t, err := time.Parse(time.RFC3339Nano, s); err == nil { + return t, nil + } + return time.Time{}, errors.Errorf("cannot parse %q to a valid timestamp", s) +} diff --git a/internal/promcompliance/comparer/comparer.go b/internal/promcompliance/comparer/comparer.go new file mode 100644 index 00000000..27849ef5 --- /dev/null +++ b/internal/promcompliance/comparer/comparer.go @@ -0,0 +1,193 @@ +package comparer + +import ( + "context" + "fmt" + "sort" + "strings" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/pkg/errors" + v1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/model" + + "github.com/go-faster/oteldb/internal/promcompliance/config" +) + +const ( + defaultFraction = 0.00001 + defaultMargin = 0.0 +) + +// PromAPI allows running instant and range queries against a Prometheus-compatible API. +type PromAPI interface { + // Query performs a query for the given time. + Query(ctx context.Context, query string, ts time.Time, opts ...v1.Option) (model.Value, v1.Warnings, error) + // QueryRange performs a query for the given range. + QueryRange(ctx context.Context, query string, r v1.Range, opts ...v1.Option) (model.Value, v1.Warnings, error) +} + +// TestCase represents a fully expanded query to be tested. +type TestCase struct { + Query string `json:"query"` + SkipComparison bool `json:"skipComparison"` + ShouldFail bool `json:"shouldFail"` + Start time.Time `json:"start"` + End time.Time `json:"end"` + Resolution time.Duration `json:"resolution"` +} + +// A Comparer allows comparing query results for test cases between a reference API and a test API. +type Comparer struct { + refAPI PromAPI + testAPI PromAPI + queryTweaks []*config.QueryTweak + compareOptions cmp.Options +} + +// New returns a new Comparer. +func New(refAPI, testAPI PromAPI, queryTweaks []*config.QueryTweak) *Comparer { + var options cmp.Options + addFloatCompareOptions(queryTweaks, &options) + addDropResultLabelsOptions(queryTweaks, &options) + addCaseInsensitiveCompareOptions(queryTweaks, &options) + return &Comparer{ + refAPI: refAPI, + testAPI: testAPI, + queryTweaks: queryTweaks, + compareOptions: options, + } +} + +// Result tracks a single test case's query comparison result. +type Result struct { + TestCase *TestCase `json:"testCase"` + Diff string `json:"diff"` + UnexpectedFailure string `json:"unexpectedFailure"` + UnexpectedSuccess bool `json:"unexpectedSuccess"` + Unsupported bool `json:"unsupported"` +} + +// Success returns true if the comparison result was successful. +func (r *Result) Success() bool { + return r.Diff == "" && !r.UnexpectedSuccess && r.UnexpectedFailure == "" +} + +// Compare runs a test case query against the reference API and the test API and compares the results. +func (c *Comparer) Compare(tc *TestCase) (*Result, error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + r := v1.Range{ + Start: tc.Start, + End: tc.End, + Step: tc.Resolution, + } + + // TODO: Handle warnings (second, ignored return value). + refResult, _, refErr := c.refAPI.QueryRange(ctx, tc.Query, r) + testResult, _, testErr := c.testAPI.QueryRange(ctx, tc.Query, r) + + if (refErr != nil) != tc.ShouldFail { + if refErr != nil { + return nil, errors.Wrapf(refErr, "querying reference API for %q", tc.Query) + } + return nil, fmt.Errorf("expected reference API query %q to fail, but succeeded", tc.Query) + } + + if (testErr != nil) != tc.ShouldFail { + if testErr != nil { + return &Result{TestCase: tc, UnexpectedFailure: testErr.Error(), Unsupported: strings.Contains(testErr.Error(), "501")}, nil + } + return &Result{TestCase: tc, UnexpectedSuccess: true}, nil + } + + if tc.SkipComparison || tc.ShouldFail { + return &Result{TestCase: tc}, nil + } + + sort.Sort(testResult.(model.Matrix)) + + for _, qt := range c.queryTweaks { + if qt.IgnoreFirstStep { + for _, r := range refResult.(model.Matrix) { + if len(r.Values) > 0 && r.Values[0].Timestamp.Time().Sub(tc.Start) <= 2*time.Millisecond { + r.Values = r.Values[1:] + } + } + } + } + + return &Result{ + TestCase: tc, + Diff: cmp.Diff(refResult, testResult, c.compareOptions), + }, nil +} + +func addFloatCompareOptions(queryTweaks []*config.QueryTweak, options *cmp.Options) { + fraction := defaultFraction + margin := defaultMargin + for _, rt := range queryTweaks { + if rt.AdjustValueTolerance != nil { + if rt.AdjustValueTolerance.Fraction != nil { + fraction = *rt.AdjustValueTolerance.Fraction + } + if rt.AdjustValueTolerance.Margin != nil { + margin = *rt.AdjustValueTolerance.Margin + } + } + } + *options = append( + *options, + // Translate sample values into float64 so that cmpopts.EquateApprox() works. + cmp.Transformer("TranslateFloat64", func(in model.SampleValue) float64 { + return float64(in) + }), + cmpopts.EquateApprox(fraction, margin), + // A NaN is usually not treated as equal to another NaN, but we want to treat it as such here. + cmpopts.EquateNaNs(), + ) +} + +func addDropResultLabelsOptions(queryTweaks []*config.QueryTweak, options *cmp.Options) { + for _, rt := range queryTweaks { + if len(rt.DropResultLabels) != 0 { + localRt := rt + *options = append( + *options, + cmp.Transformer( + "DropResultLabels", + func(in model.Metric) model.Metric { + m := in.Clone() + for _, ln := range localRt.DropResultLabels { + delete(m, ln) + } + return m + }, + ), + ) + } + } +} + +func addCaseInsensitiveCompareOptions(queryTweaks []*config.QueryTweak, options *cmp.Options) { + for _, rt := range queryTweaks { + if rt.IgnoreCase { + *options = append( + *options, + // Translate metric names and labels into lowercase. + cmp.Transformer("TranslateToLowerCase", + func(in model.Metric) model.Metric { + m := map[model.LabelName]model.LabelValue{} + for key, val := range in { + m[model.LabelName(strings.ToLower(string(key)))] = model.LabelValue(strings.ToLower(string(val))) + } + return m + }, + ), + ) + } + } +} diff --git a/internal/promcompliance/config/config.go b/internal/promcompliance/config/config.go new file mode 100644 index 00000000..70e3e058 --- /dev/null +++ b/internal/promcompliance/config/config.go @@ -0,0 +1,89 @@ +package config + +import ( + "bytes" + "os" + + "github.com/pkg/errors" + "github.com/prometheus/common/model" + "gopkg.in/yaml.v2" +) + +// Config models the main configuration file. +type Config struct { + ReferenceTargetConfig TargetConfig `yaml:"reference_target_config"` + TestTargetConfig TargetConfig `yaml:"test_target_config"` + QueryTweaks []*QueryTweak `yaml:"query_tweaks"` + TestCases []*TestCase `yaml:"test_cases"` + QueryTimeParameters QueryTimeParameters `yaml:"query_time_parameters"` +} + +type QueryTimeParameters struct { + EndTime string `yaml:"end_time"` + RangeInSeconds float64 `yaml:"range_in_seconds"` + ResolutionInSeconds float64 `yaml:"resolution_in_seconds"` +} + +// TargetConfig represents the configuration of a single Prometheus API endpoint. +type TargetConfig struct { + QueryURL string `yaml:"query_url"` + BasicAuthUser string `yaml:"basic_auth_user"` + BasicAuthPass string `yaml:"basic_auth_pass"` + Headers map[string]string `yaml:"headers"` + TSDBPath string `yaml:"tsdb_path"` +} + +// A QueryTweak restricts or modifies a query in certain ways that avoids certain systematic errors and/or later comparison problems. +type QueryTweak struct { + Note string `yaml:"note" json:"note"` + NoBug bool `yaml:"no_bug,omitempty" json:"noBug,omitempty"` + TruncateTimestampsToMS int64 `yaml:"truncate_timestamps_to_ms" json:"truncateTimestampsToMS,omitempty"` + AlignTimestampsToStep bool `yaml:"align_timestamps_to_step" json:"alignTimestampsToStep,omitempty"` + OffsetTimestampsByMS int64 `yaml:"offset_timestamps_by_ms" json:"offsetTimestampsByMS,omitempty"` + DropResultLabels []model.LabelName `yaml:"drop_result_labels" json:"dropResultLabels,omitempty"` + IgnoreFirstStep bool `yaml:"ignore_first_step" json:"ignoreFirstStep,omitempty"` + IgnoreCase bool `yaml:"ignore_case" json:"ignoreCase,omitempty"` + AdjustValueTolerance *AdjustValueTolerance `yaml:"adjust_value_tolerance" json:"adjustValueTolerance,omitempty"` +} + +type AdjustValueTolerance struct { + Fraction *float64 `yaml:"fraction" json:"fraction,omitempty"` + Margin *float64 `yaml:"margin" json:"margin,omitempty"` +} + +// TestCase represents a given query (pattern) to be tested. +type TestCase struct { + Query string `yaml:"query"` + VariantArgs []string `yaml:"variant_args,omitempty"` + SkipComparison bool `yaml:"skip_comparison,omitempty"` + ShouldFail bool `yaml:"should_fail,omitempty"` +} + +// LoadFromFiles parses the given YAML files into a Config. +func LoadFromFiles(filenames []string) (*Config, error) { + var buf bytes.Buffer + for _, f := range filenames { + content, err := os.ReadFile(f) // #nosec G304 + if err != nil { + return nil, errors.Wrapf(err, "reading config file %s", f) + } + if _, err := buf.Write(content); err != nil { + return nil, errors.Wrapf(err, "appending config file %s to buffer", f) + } + } + cfg, err := Load(buf.Bytes()) + if err != nil { + return nil, errors.Wrapf(err, "parsing YAML files %s", filenames) + } + return cfg, nil +} + +// Load parses the YAML input into a Config. +func Load(content []byte) (*Config, error) { + cfg := &Config{} + err := yaml.UnmarshalStrict(content, cfg) + if err != nil { + return nil, err + } + return cfg, nil +} diff --git a/internal/promcompliance/output/example-output.html b/internal/promcompliance/output/example-output.html new file mode 100644 index 00000000..2ed61cb3 --- /dev/null +++ b/internal/promcompliance/output/example-output.html @@ -0,0 +1,60 @@ + + + + + +

Passed: {{ numPassed .Results }} / {{ numResults .Results }} ({{ printf "%.2f" (percent (numPassed .Results) (numResults .Results)) }}%)

+ + + + + + + {{ $includePassing := .IncludePassing }} + {{ range .Results }} + {{ if include $includePassing . }} + + + + + + {{ if .UnexpectedFailure }} + + {{ end }} + {{ if .UnexpectedSuccess }} + + {{ end }} + {{ if .Diff }} + + {{ end }} + {{ end }} + {{ end }} +
QueryOutcome
{{ .TestCase.Query }}
{{ if .Success }}PASS{{ else }}FAIL{{ end }}
The query failed to run against the test target: {{ .UnexpectedFailure }}
The query ran successfully against the test target, but should have failed.
{{ .Diff }}
+ + diff --git a/internal/promcompliance/output/html.go b/internal/promcompliance/output/html.go new file mode 100644 index 00000000..03cd7cf4 --- /dev/null +++ b/internal/promcompliance/output/html.go @@ -0,0 +1,64 @@ +package output + +import ( + "html/template" + "log" + "os" + "path" + + "github.com/pkg/errors" + + "github.com/go-faster/oteldb/internal/promcompliance/comparer" + "github.com/go-faster/oteldb/internal/promcompliance/config" +) + +var funcMap = map[string]interface{}{ + "include": func(includePassing bool, result *comparer.Result) bool { + return includePassing || !result.Success() + }, + "numResults": func(results []*comparer.Result) int { + return len(results) + }, + "numPassed": func(results []*comparer.Result) int { + num := 0 + for _, r := range results { + if r.Success() { + num++ + } + } + return num + }, + "numFailed": func(results []*comparer.Result) int { + num := 0 + for _, r := range results { + if !r.Success() { + num++ + } + } + return num + }, + "percent": func(part, total int) float64 { + return 100 * float64(part) / float64(total) + }, +} + +// HTML produces HTML output for a number of query results. +func HTML(tplFile string) (Outputter, error) { + t, err := template.New(path.Base(tplFile)).Funcs(funcMap).ParseFiles(tplFile) + if err != nil { + return nil, errors.Wrapf(err, "parsing template file %q", tplFile) + } + + return func(results []*comparer.Result, includePassing bool, tweaks []*config.QueryTweak) { + err := t.Execute(os.Stdout, struct { + Results []*comparer.Result + IncludePassing bool + }{ + Results: results, + IncludePassing: includePassing, + }) + if err != nil { + log.Println("executing template:", err) + } + }, nil +} diff --git a/internal/promcompliance/output/json.go b/internal/promcompliance/output/json.go new file mode 100644 index 00000000..3f04170b --- /dev/null +++ b/internal/promcompliance/output/json.go @@ -0,0 +1,23 @@ +package output + +import ( + "encoding/json" + "fmt" + + "github.com/go-faster/oteldb/internal/promcompliance/comparer" + "github.com/go-faster/oteldb/internal/promcompliance/config" +) + +// JSON produces JSON-based output for a number of query results. +func JSON(results []*comparer.Result, includePassing bool, tweaks []*config.QueryTweak) { + buf, err := json.Marshal(map[string]interface{}{ + "totalResults": len(results), // Needed because we may exclude passing results. + "results": results, + "includePassing": includePassing, + "queryTweaks": tweaks, + }) + if err != nil { + panic(err) + } + fmt.Print(string(buf)) +} diff --git a/internal/promcompliance/output/outputter.go b/internal/promcompliance/output/outputter.go new file mode 100644 index 00000000..d9d5f207 --- /dev/null +++ b/internal/promcompliance/output/outputter.go @@ -0,0 +1,9 @@ +package output + +import ( + "github.com/go-faster/oteldb/internal/promcompliance/comparer" + "github.com/go-faster/oteldb/internal/promcompliance/config" +) + +// An Outputter outputs a number of test results. +type Outputter func(results []*comparer.Result, includePassing bool, tweaks []*config.QueryTweak) diff --git a/internal/promcompliance/output/text.go b/internal/promcompliance/output/text.go new file mode 100644 index 00000000..14b505ed --- /dev/null +++ b/internal/promcompliance/output/text.go @@ -0,0 +1,60 @@ +package output + +import ( + "fmt" + "strings" + + "github.com/go-faster/oteldb/internal/promcompliance/comparer" + "github.com/go-faster/oteldb/internal/promcompliance/config" +) + +// Text produces text-based output for a number of query results. +func Text(results []*comparer.Result, includePassing bool, tweaks []*config.QueryTweak) { + successes := 0 + unsupported := 0 + for _, res := range results { + if res.Success() { + successes++ + if !includePassing { + continue + } + } + if res.Unsupported { + unsupported++ + } + + fmt.Println(strings.Repeat("-", 80)) + fmt.Printf("QUERY: %v\n", res.TestCase.Query) + fmt.Printf("START: %v, STOP: %v, STEP: %v\n", res.TestCase.Start, res.TestCase.End, res.TestCase.Resolution) + fmt.Printf("RESULT: ") + if res.Success() { + fmt.Println("PASSED") + } else if res.Unsupported { + fmt.Println("UNSUPPORTED: ") + fmt.Printf("Query is unsupported: %v\n", res.UnexpectedFailure) + } else { + fmt.Printf("FAILED: ") + if res.UnexpectedFailure != "" { + fmt.Printf("Query failed unexpectedly: %v\n", res.UnexpectedFailure) + } + if res.UnexpectedSuccess { + fmt.Println("Query succeeded, but should have failed.") + } + if res.Diff != "" { + fmt.Println("Query returned different results:") + fmt.Println(res.Diff) + } + } + } + + fmt.Println(strings.Repeat("=", 80)) + fmt.Println("General query tweaks:") + if len(tweaks) == 0 { + fmt.Println("None.") + } + for _, t := range tweaks { + fmt.Println("* ", t.Note) + } + fmt.Println(strings.Repeat("=", 80)) + fmt.Printf("Total: %d / %d (%.2f%%) passed, %d unsupported\n", successes, len(results), 100*float64(successes)/float64(len(results)), unsupported) +} diff --git a/internal/promcompliance/output/tsv.go b/internal/promcompliance/output/tsv.go new file mode 100644 index 00000000..daa73c0b --- /dev/null +++ b/internal/promcompliance/output/tsv.go @@ -0,0 +1,40 @@ +package output + +import ( + "fmt" + + "github.com/go-faster/oteldb/internal/promcompliance/comparer" + "github.com/go-faster/oteldb/internal/promcompliance/config" +) + +// TSV produces tab separated values output for a number of query results. +func TSV(results []*comparer.Result, _ bool, _ []*config.QueryTweak) { + successes := 0 + unsupported := 0 + + fmt.Println("QUERY\tSTART\tSTOP\tSTEP\tRESULT") + + for _, res := range results { + if res.Success() { + successes++ + } + if res.Unsupported { + unsupported++ + } + + fmt.Printf("%v\t%v\t%v\t%v\t", res.TestCase.Query, res.TestCase.Start, res.TestCase.End, res.TestCase.Resolution) + if res.Success() { + fmt.Println("PASSED") + } else if res.Unsupported { + fmt.Println("UNSUPPORTED") + } else { + fmt.Println("FAILED") + } + } + totalTestCases := len(results) + totalFailed := totalTestCases - successes - unsupported + fmt.Printf("\n\t\tPASSED\t%v\t%.4f\n", successes, float64(successes)/float64(totalTestCases)) + fmt.Printf("\t\tFAILED\t%v\t%.4f\n", totalFailed, float64(totalFailed)/float64(totalTestCases)) + fmt.Printf("\t\tUNSUPPORTED\t%v\t%.4f\n", unsupported, float64(unsupported)/float64(totalTestCases)) + fmt.Printf("\t\tTOTAL\t%v\t%.4f\n", totalTestCases, float64(1)) +} diff --git a/internal/promcompliance/prometheus-test-data.yml b/internal/promcompliance/prometheus-test-data.yml new file mode 100644 index 00000000..2f19bfff --- /dev/null +++ b/internal/promcompliance/prometheus-test-data.yml @@ -0,0 +1,10 @@ +global: + scrape_interval: 5s + +scrape_configs: +- job_name: 'demo' + static_configs: + - targets: + - 'demo.promlabs.com:10000' + - 'demo.promlabs.com:10001' + - 'demo.promlabs.com:10002' diff --git a/internal/promcompliance/promql-test-queries.yml b/internal/promcompliance/promql-test-queries.yml new file mode 100644 index 00000000..c601cd8d --- /dev/null +++ b/internal/promcompliance/promql-test-queries.yml @@ -0,0 +1,245 @@ +# This set of example queries expects data from the following Prometheus configuration file to have +# been ingested into both a vanilla Prometheus server and the third-party system for several hours, +# so that the tester can compare query results from both systems over a range of time: +# +# ----------- prometheus.yml ----------- +# global: +# scrape_interval: 5s +# +# scrape_configs: +# - job_name: 'demo' +# static_configs: +# - targets: +# - 'demo.promlabs.com:10000' +# - 'demo.promlabs.com:10001' +# - 'demo.promlabs.com:10002' +# -------------------------------------- +# +# You will have to add a "remote_write" section to this configuration to ingest data into the third-party +# system, or in the case of Thanos, add a Thanos sidecar to the Prometheus running with this configuration. +# See https://promlabs.com/blog/2020/08/06/comparing-promql-correctness-across-vendors#first-comparisons for +# more background information. +# +# The demo service instances expose a predictable set of synthetic metrics and are hosted on a best-effort +# basis by PromLabs. If you want to run your own demo service instances instead, you can do so via: +# +# docker run -p 10000:10000 julius/prometheus-demo-service:latest -listen-address=:10000 +# docker run -p 10001:10001 julius/prometheus-demo-service:latest -listen-address=:10001 +# docker run -p 10002:10002 julius/prometheus-demo-service:latest -listen-address=:10002 +# +# You will then also need to replace the host "demo.promlabs.com" in the test queries below with whatever +# host you are running the instances on. +test_cases: + # Scalar literals. + - query: '42' + - query: '1.234' + - query: '.123' + - query: '1.23e-3' + - query: '0x3d' + - query: 'Inf' + - query: '+Inf' + - query: '-Inf' + - query: 'NaN' + + # Vector selectors. + # TODO: Add tests for staleness support. + - query: 'demo_memory_usage_bytes' + - query: '{__name__="demo_memory_usage_bytes"}' + - query: 'demo_memory_usage_bytes{type="free"}' + - query: 'demo_memory_usage_bytes{type!="free"}' + - query: 'demo_memory_usage_bytes{instance=~"demo.promlabs.com:.*"}' + - query: 'demo_memory_usage_bytes{instance=~"host"}' + - query: 'demo_memory_usage_bytes{instance!~".*:10000"}' + - query: 'demo_memory_usage_bytes{type="free", instance!="demo.promlabs.com:10000"}' + - query: '{type="free", instance!="demo.promlabs.com:10000"}' + - query: '{__name__=~".*"}' + should_fail: true + - query: "nonexistent_metric_name" + - query: 'demo_memory_usage_bytes offset {{.offset}}' + variant_args: ['offset'] + - query: 'demo_memory_usage_bytes offset -{{.offset}}' + variant_args: ['offset'] + # Test staleness handling. + - query: demo_intermittent_metric + + # Aggregation operators. + - query: '{{.simpleAggrOp}}(demo_memory_usage_bytes)' + variant_args: ['simpleAggrOp'] + - query: '{{.simpleAggrOp}}(nonexistent_metric_name)' + variant_args: ['simpleAggrOp'] + - query: '{{.simpleAggrOp}} by() (demo_memory_usage_bytes)' + variant_args: ['simpleAggrOp'] + - query: '{{.simpleAggrOp}} by(instance) (demo_memory_usage_bytes)' + variant_args: ['simpleAggrOp'] + - query: '{{.simpleAggrOp}} by(instance, type) (demo_memory_usage_bytes)' + variant_args: ['simpleAggrOp'] + - query: '{{.simpleAggrOp}} by(nonexistent) (demo_memory_usage_bytes)' + variant_args: ['simpleAggrOp'] + - query: '{{.simpleAggrOp}} without() (demo_memory_usage_bytes)' + variant_args: ['simpleAggrOp'] + - query: '{{.simpleAggrOp}} without(instance) (demo_memory_usage_bytes)' + variant_args: ['simpleAggrOp'] + - query: '{{.simpleAggrOp}} without(instance, type) (demo_memory_usage_bytes)' + variant_args: ['simpleAggrOp'] + - query: '{{.simpleAggrOp}} without(nonexistent) (demo_memory_usage_bytes)' + variant_args: ['simpleAggrOp'] + - query: '{{.topBottomOp}} (3, demo_memory_usage_bytes)' + variant_args: ['topBottomOp'] + - query: '{{.topBottomOp}} by(instance) (2, demo_memory_usage_bytes)' + variant_args: ['topBottomOp'] + - query: '{{.topBottomOp}} without(instance) (2, demo_memory_usage_bytes)' + variant_args: ['topBottomOp'] + - query: '{{.topBottomOp}} without() (2, demo_memory_usage_bytes)' + variant_args: ['topBottomOp'] + - query: 'quantile({{.quantile}}, demo_memory_usage_bytes)' + variant_args: ['quantile'] + - query: 'avg(max by(type) (demo_memory_usage_bytes))' + + # Binary operators. + - query: '1 * 2 + 4 / 6 - 10 % 2 ^ 2' + - query: 'demo_num_cpus + (1 {{.compBinOp}} bool 2)' + variant_args: ['compBinOp'] + - query: 'demo_memory_usage_bytes {{.binOp}} 1.2345' + variant_args: ['binOp'] + - query: 'demo_memory_usage_bytes {{.compBinOp}} bool 1.2345' + variant_args: ['compBinOp'] + - query: '1.2345 {{.compBinOp}} bool demo_memory_usage_bytes' + variant_args: ['compBinOp'] + - query: '0.12345 {{.binOp}} demo_memory_usage_bytes' + variant_args: ['binOp'] + - query: '(1 * 2 + 4 / 6 - (10%7)^2) {{.binOp}} demo_memory_usage_bytes' + variant_args: ['binOp'] + - query: 'demo_memory_usage_bytes {{.binOp}} (1 * 2 + 4 / 6 - 10)' + variant_args: ['binOp'] + # Check that vector-scalar binops set output timestamps correctly. + - query: 'timestamp(demo_memory_usage_bytes * 1)' + # Check that unary minus sets timestamps correctly. + # TODO: Check this more systematically for every node type? + - query: 'timestamp(-demo_memory_usage_bytes)' + - query: 'demo_memory_usage_bytes {{.binOp}} on(instance, job, type) demo_memory_usage_bytes' + variant_args: ['binOp'] + - query: 'sum by(instance, type) (demo_memory_usage_bytes) {{.binOp}} on(instance, type) group_left(job) demo_memory_usage_bytes' + variant_args: ['binOp'] + - query: 'demo_memory_usage_bytes {{.compBinOp}} bool on(instance, job, type) demo_memory_usage_bytes' + variant_args: ['compBinOp'] + # Check that __name__ is always dropped, even if it's part of the matching labels. + - query: 'demo_memory_usage_bytes / on(instance, job, type, __name__) demo_memory_usage_bytes' + - query: 'sum without(job) (demo_memory_usage_bytes) / on(instance, type) demo_memory_usage_bytes' + - query: 'sum without(job) (demo_memory_usage_bytes) / on(instance, type) group_left demo_memory_usage_bytes' + - query: 'sum without(job) (demo_memory_usage_bytes) / on(instance, type) group_left(job) demo_memory_usage_bytes' + - query: 'demo_memory_usage_bytes / on(instance, job) group_left demo_num_cpus' + - query: 'demo_memory_usage_bytes / on(instance, type, job, non_existent) demo_memory_usage_bytes' + # TODO: Add non-explicit many-to-one / one-to-many that errors. + # TODO: Add many-to-many match that errors. + + # NaN/Inf/-Inf support. + - query: 'demo_num_cpus * Inf' + - query: 'demo_num_cpus * -Inf' + - query: 'demo_num_cpus * NaN' + + # Unary expressions. + - query: 'demo_memory_usage_bytes + -(1)' + - query: '-demo_memory_usage_bytes' + # Check precedence. + - query: -1 ^ 2 + + # Binops involving non-const scalars. + - query: '1 {{.arithBinOp}} time()' + variant_args: ['arithBinOp'] + - query: 'time() {{.arithBinOp}} 1' + variant_args: ['arithBinOp'] + - query: 'time() {{.compBinOp}} bool 1' + variant_args: ['compBinOp'] + - query: '1 {{.compBinOp}} bool time()' + variant_args: ['compBinOp'] + - query: 'time() {{.arithBinOp}} time()' + variant_args: ['arithBinOp'] + - query: 'time() {{.compBinOp}} bool time()' + variant_args: ['compBinOp'] + - query: 'time() {{.binOp}} demo_memory_usage_bytes' + variant_args: ['binOp'] + - query: 'demo_memory_usage_bytes {{.binOp}} time()' + variant_args: ['binOp'] + + # Functions. + - query: '{{.simpleTimeAggrOp}}_over_time(demo_memory_usage_bytes[{{.range}}])' + variant_args: ['simpleTimeAggrOp', 'range'] + - query: 'quantile_over_time({{.quantile}}, demo_memory_usage_bytes[{{.range}}])' + variant_args: ['quantile', 'range'] + - query: 'timestamp(demo_num_cpus)' + - query: 'timestamp(timestamp(demo_num_cpus))' + - query: '{{.simpleMathFunc}}(demo_memory_usage_bytes)' + variant_args: ['simpleMathFunc'] + - query: '{{.simpleMathFunc}}(-demo_memory_usage_bytes)' + variant_args: ['simpleMathFunc'] + - query: '{{.extrapolatedRateFunc}}(nonexistent_metric[5m])' + variant_args: ['extrapolatedRateFunc'] + - query: '{{.extrapolatedRateFunc}}(demo_cpu_usage_seconds_total[{{.range}}])' + variant_args: ['extrapolatedRateFunc', 'range'] + - query: 'deriv(demo_disk_usage_bytes[{{.range}}])' + variant_args: ['range'] + - query: 'predict_linear(demo_disk_usage_bytes[{{.range}}], 600)' + variant_args: ['range'] + - query: 'time()' + # label_replace does a full-string match and replace. + - query: 'label_replace(demo_num_cpus, "job", "destination-value-$1", "instance", "demo.promlabs.com:(.*)")' + # label_replace does not do a sub-string match. + - query: 'label_replace(demo_num_cpus, "job", "destination-value-$1", "instance", "host:(.*)")' + # label_replace works with multiple capture groups. + - query: 'label_replace(demo_num_cpus, "job", "$1-$2", "instance", "local(.*):(.*)")' + # label_replace does not overwrite the destination label if the source label does not exist. + - query: 'label_replace(demo_num_cpus, "job", "value-$1", "nonexistent-src", "source-value-(.*)")' + # label_replace overwrites the destination label if the source label is empty, but matched. + - query: 'label_replace(demo_num_cpus, "job", "value-$1", "nonexistent-src", "(.*)")' + # label_replace does not overwrite the destination label if the source label is not matched. + - query: 'label_replace(demo_num_cpus, "job", "value-$1", "instance", "non-matching-regex")' + # label_replace drops labels that are set to empty values. + - query: 'label_replace(demo_num_cpus, "job", "", "dst", ".*")' + # label_replace fails when the regex is invalid. + - query: 'label_replace(demo_num_cpus, "job", "value-$1", "src", "(.*")' + should_fail: true + # label_replace fails when the destination label name is not a valid Prometheus label name. + - query: 'label_replace(demo_num_cpus, "~invalid", "", "src", "(.*)")' + should_fail: true + # label_replace fails when there would be duplicated identical output label sets. + - query: 'label_replace(demo_num_cpus, "instance", "", "", "")' + should_fail: true + - query: 'label_join(demo_num_cpus, "new_label", "-", "instance", "job")' + - query: 'label_join(demo_num_cpus, "job", "-", "instance", "job")' + - query: 'label_join(demo_num_cpus, "job", "-", "instance")' + - query: 'label_join(demo_num_cpus, "~invalid", "-", "instance")' + should_fail: true + - query: '{{.dateFunc}}()' + variant_args: ['dateFunc'] + - query: '{{.dateFunc}}(demo_batch_last_success_timestamp_seconds offset {{.offset}})' + variant_args: ['dateFunc', 'offset'] + - query: '{{.instantRateFunc}}(demo_cpu_usage_seconds_total[{{.range}}])' + variant_args: ['instantRateFunc', 'range'] + - query: '{{.clampFunc}}(demo_memory_usage_bytes, 2)' + variant_args: ['clampFunc'] + - query: 'clamp(demo_memory_usage_bytes, 0, 1)' + - query: 'clamp(demo_memory_usage_bytes, 0, 1000000000000)' + - query: 'clamp(demo_memory_usage_bytes, 1000000000000, 0)' + - query: 'clamp(demo_memory_usage_bytes, 1000000000000, 1000000000000)' + - query: 'resets(demo_cpu_usage_seconds_total[{{.range}}])' + variant_args: ['range'] + - query: 'changes(demo_batch_last_success_timestamp_seconds[{{.range}}])' + variant_args: ['range'] + - query: 'vector(1.23)' + - query: 'vector(time())' + - query: 'histogram_quantile({{.quantile}}, rate(demo_api_request_duration_seconds_bucket[1m]))' + variant_args: ['quantile'] + - query: 'histogram_quantile(0.9, nonexistent_metric)' + - # Missing "le" label. + query: 'histogram_quantile(0.9, demo_memory_usage_bytes)' + - # Missing "le" label only in some series of the same grouping. + query: 'histogram_quantile(0.9, {__name__=~"demo_api_request_duration_seconds_.+"})' + - query: 'holt_winters(demo_disk_usage_bytes[10m], {{.smoothingFactor}}, {{.trendFactor}})' + variant_args: ['smoothingFactor', 'trendFactor'] + - query: 'count_values("value", demo_api_request_duration_seconds_bucket)' + - query: 'absent(demo_memory_usage_bytes)' + - query: 'absent(nonexistent_metric_name)' + + # Subqueries. + - query: 'max_over_time((time() - max(demo_batch_last_success_timestamp_seconds) < 1000)[5m:10s] offset 5m)' + - query: 'avg_over_time(rate(demo_cpu_usage_seconds_total[1m])[2m:10s])' diff --git a/internal/promcompliance/test-amp.yml b/internal/promcompliance/test-amp.yml new file mode 100644 index 00000000..29eb2601 --- /dev/null +++ b/internal/promcompliance/test-amp.yml @@ -0,0 +1,11 @@ +reference_target_config: + query_url: 'http://localhost:9090' + +test_target_config: + query_url: 'http://localhost:8080/workspaces/' + headers: + Host: aps-workspaces..amazonaws.com + +query_tweaks: + - note: 'AMP aligns incoming query timestamps to a multiple of the query resolution step to enable caching.' + truncate_timestamps_to_ms: 10000 diff --git a/internal/promcompliance/test-chronosphere.yml b/internal/promcompliance/test-chronosphere.yml new file mode 100644 index 00000000..ea1163b9 --- /dev/null +++ b/internal/promcompliance/test-chronosphere.yml @@ -0,0 +1,7 @@ +reference_target_config: + query_url: 'http://localhost:9090' + +test_target_config: + query_url: 'https://.chronosphere.io/data/metrics' + headers: + Authorization: 'Bearer ' diff --git a/internal/promcompliance/test-cortex.yml b/internal/promcompliance/test-cortex.yml new file mode 100644 index 00000000..6d1d5415 --- /dev/null +++ b/internal/promcompliance/test-cortex.yml @@ -0,0 +1,5 @@ +reference_target_config: + query_url: 'http://localhost:9090' + +test_target_config: + query_url: 'http://localhost:9009/api/prom' diff --git a/internal/promcompliance/test-gmp.yml b/internal/promcompliance/test-gmp.yml new file mode 100644 index 00000000..5da26fb3 --- /dev/null +++ b/internal/promcompliance/test-gmp.yml @@ -0,0 +1,15 @@ +reference_target_config: + query_url: 'http://localhost:9090' + +test_target_config: + query_url: 'https://monitoring.googleapis.com/v1/projects/promql-testing/location/global/prometheus' + headers: + Authorization: 'Bearer ' + X-Goog-User-Project: promql-testing + +query_tweaks: + - note: 'GMP requires adding "external_labels" for the location and project ID that need to be stripped before comparing results.' + no_bug: true + drop_result_labels: + - location + - project_id diff --git a/internal/promcompliance/test-grafana-cloud.yml b/internal/promcompliance/test-grafana-cloud.yml new file mode 100644 index 00000000..66283c56 --- /dev/null +++ b/internal/promcompliance/test-grafana-cloud.yml @@ -0,0 +1,7 @@ +reference_target_config: + query_url: 'http://localhost:9090' + +test_target_config: + query_url: 'https://.grafana.net/api/prom' + basic_auth_user: '' + basic_auth_pass: '' diff --git a/internal/promcompliance/test-m3.yml b/internal/promcompliance/test-m3.yml new file mode 100644 index 00000000..9a7a6733 --- /dev/null +++ b/internal/promcompliance/test-m3.yml @@ -0,0 +1,5 @@ +reference_target_config: + query_url: 'http://localhost:9091' + +test_target_config: + query_url: http://localhost:7201 diff --git a/internal/promcompliance/test-new-relic.yml b/internal/promcompliance/test-new-relic.yml new file mode 100644 index 00000000..3f2665e6 --- /dev/null +++ b/internal/promcompliance/test-new-relic.yml @@ -0,0 +1,17 @@ +reference_target_config: + query_url: 'http://localhost:9090' + +test_target_config: + query_url: https://prometheus-api..newrelic.com + headers: + X-Query-Key: + +query_tweaks: + - note: 'New Relic is sometimes off by 1ms when parsing floating point start/end timestamps.' + truncate_timestamps_to_ms: 1000 + - note: 'New Relic adds a "prometheus_server" label to distinguish Prometheus servers, leading to extra labels in query results. These need to be stripped before comparisons.' + no_bug: true + drop_result_labels: + - prometheus_server + - note: 'New Relic omits the first resolution step in the output.' + ignore_first_step: true diff --git a/internal/promcompliance/test-promscale.yml b/internal/promcompliance/test-promscale.yml new file mode 100644 index 00000000..637e7861 --- /dev/null +++ b/internal/promcompliance/test-promscale.yml @@ -0,0 +1,5 @@ +reference_target_config: + query_url: 'http://localhost:9090' + +test_target_config: + query_url: 'http://localhost:9201' diff --git a/internal/promcompliance/test-sysdig.yml b/internal/promcompliance/test-sysdig.yml new file mode 100644 index 00000000..e0960406 --- /dev/null +++ b/internal/promcompliance/test-sysdig.yml @@ -0,0 +1,15 @@ +reference_target_config: + query_url: 'http://localhost:9090' + +test_target_config: + query_url: https://.app.sysdig.com/prometheus + headers: + Authorization: 'Bearer ' + +query_tweaks: + - note: 'All samples and queries are aligned to a 10-second grid in Sysdig.' + align_timestamps_to_step: true + - note: 'Sysdig adds a "remote_write" label to data coming from Prometheus, which needs to be stripped before comparing results.' + no_bug: true + drop_result_labels: + - remote_write diff --git a/internal/promcompliance/test-thanos.yml b/internal/promcompliance/test-thanos.yml new file mode 100644 index 00000000..c0239c69 --- /dev/null +++ b/internal/promcompliance/test-thanos.yml @@ -0,0 +1,11 @@ +reference_target_config: + query_url: 'http://localhost:9090' + +test_target_config: + query_url: 'http://localhost:20902' + +query_tweaks: + - note: 'Thanos requires adding "external_labels" to distinguish Prometheus servers, leading to extra labels in query results that need to be stripped before comparing results.' + no_bug: true + drop_result_labels: + - prometheus diff --git a/internal/promcompliance/test-victoriametrics.yml b/internal/promcompliance/test-victoriametrics.yml new file mode 100644 index 00000000..90f1621c --- /dev/null +++ b/internal/promcompliance/test-victoriametrics.yml @@ -0,0 +1,9 @@ +reference_target_config: + query_url: 'http://localhost:9090' + +test_target_config: + query_url: 'http://localhost:8428' + +query_tweaks: + - note: 'VictoriaMetrics aligns incoming query timestamps to a multiple of the query resolution step.' + align_timestamps_to_step: true diff --git a/internal/promcompliance/test-wavefront.yml b/internal/promcompliance/test-wavefront.yml new file mode 100644 index 00000000..7a6d5f31 --- /dev/null +++ b/internal/promcompliance/test-wavefront.yml @@ -0,0 +1,11 @@ +reference_target_config: + query_url: 'http://localhost:9090/' + +test_target_config: + query_url: https://tracing.wavefront.com/ + headers: + Authorization: 'Bearer ' + +query_tweaks: + - note: 'Wavefront is sometimes off by 1ms when parsing floating point start/end timestamps.' + truncate_timestamps_to_ms: 1000 diff --git a/internal/promcompliance/testcases/expand.go b/internal/promcompliance/testcases/expand.go new file mode 100644 index 00000000..01606ec1 --- /dev/null +++ b/internal/promcompliance/testcases/expand.go @@ -0,0 +1,154 @@ +// Some of this code has been taken and adapted from InfluxData: +// https://github.com/influxdata/influxdb/blob/26fdb792ffd74f773c253df5d9bebf64ef2b3214/query/promql/internal/promqltests/tests.go +// +// The original copyright notice and license of that code is reproduced here: +// +// ------------------------------------------------------------------------------- +// +// MIT License + +// Copyright (c) 2018 InfluxData + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// ------------------------------------------------------------------------------- + +package testcases + +import ( + "bytes" + "fmt" + "text/template" + "time" + + "github.com/go-faster/oteldb/internal/promcompliance/comparer" + "github.com/go-faster/oteldb/internal/promcompliance/config" +) + +var testVariantArgs = map[string][]string{ + "range": {"1s", "15s", "1m", "5m", "15m", "1h"}, + "offset": {"1m", "5m", "10m"}, + "simpleAggrOp": {"sum", "avg", "max", "min", "count", "stddev", "stdvar"}, + "simpleTimeAggrOp": {"sum", "avg", "max", "min", "count", "stddev", "stdvar", "absent", "last"}, + "topBottomOp": {"topk", "bottomk"}, + "quantile": { + "-0.5", + "0.1", + "0.5", + "0.75", + "0.95", + "0.90", + "0.99", + "1", + "1.5", + }, + "arithBinOp": {"+", "-", "*", "/", "%", "^"}, + "compBinOp": {"==", "!=", "<", ">", "<=", ">="}, + "binOp": {"+", "-", "*", "/", "%", "^", "==", "!=", "<", ">", "<=", ">="}, + "simpleMathFunc": {"abs", "ceil", "floor", "exp", "sqrt", "ln", "log2", "log10", "round"}, + "extrapolatedRateFunc": {"delta", "rate", "increase"}, + "clampFunc": {"clamp_min", "clamp_max"}, + "instantRateFunc": {"idelta", "irate"}, + "dateFunc": {"day_of_month", "day_of_week", "days_in_month", "hour", "minute", "month", "year"}, + "smoothingFactor": {"0.1", "0.5", "0.8"}, + "trendFactor": {"0.1", "0.5", "0.8"}, +} + +// tprintf replaces template arguments in a string with their instantiations from the provided map. +func tprintf(tmpl string, data map[string]string) string { + t := template.Must(template.New("Query").Parse(tmpl)) + buf := &bytes.Buffer{} + if err := t.Execute(buf, data); err != nil { + panic(err) + } + return buf.String() +} + +// getVariants returns every possible combinations (variants) of a template query. +func getVariants(query string, remainingVariantArgs []string, args map[string]string) []string { + // Either this Query had no variants defined to begin with or they have + // been fully filled out in "args" from recursive parent calls. + if len(remainingVariantArgs) == 0 { + return []string{tprintf(query, args)} + } + + // Recursively iterate through the values for each variant arg dimension, + // selecting one dimension (arg) to vary per recursion level and let the + // other recursion levels iterate through the remaining dimensions until + // all args are defined. + var queries []string + vArg := remainingVariantArgs[0] + filteredVArgs := make([]string, 0, len(remainingVariantArgs)-1) + for _, va := range remainingVariantArgs { + if va != vArg { + filteredVArgs = append(filteredVArgs, va) + } + } + + vals := testVariantArgs[vArg] + if len(vals) == 0 { + panic(fmt.Errorf("unknown variant arg %q", vArg)) + } + for _, variantVal := range vals { + args[vArg] = variantVal + qs := getVariants(query, filteredVArgs, args) + queries = append(queries, qs...) + } + return queries +} + +func applyQueryTweaks(tc *comparer.TestCase, tweaks []*config.QueryTweak) *comparer.TestCase { + resTC := *tc + for _, t := range tweaks { + if d := time.Duration(t.TruncateTimestampsToMS) * time.Millisecond; d != 0 { + resTC.Start = resTC.Start.Truncate(d) + resTC.End = resTC.End.Truncate(d) + } + if t.AlignTimestampsToStep { + resTC.Start = resTC.Start.Truncate(resTC.Resolution) + resTC.End = resTC.End.Truncate(resTC.Resolution) + } + if d := time.Duration(t.OffsetTimestampsByMS) * time.Millisecond; d != 0 { + resTC.Start = resTC.Start.Add(d) + resTC.End = resTC.End.Add(d) + } + } + return &resTC +} + +// ExpandTestCases returns the fully expanded test cases for a given set of templates test cases. +func ExpandTestCases(cases []*config.TestCase, tweaks []*config.QueryTweak, start, end time.Time, resolution time.Duration) []*comparer.TestCase { + tcs := make([]*comparer.TestCase, 0) + for _, q := range cases { + vs := getVariants(q.Query, q.VariantArgs, make(map[string]string)) + for _, v := range vs { + tc := &comparer.TestCase{ + Query: v, + SkipComparison: q.SkipComparison, + ShouldFail: q.ShouldFail, + Start: start, + End: end, + Resolution: resolution, + } + + tcs = append(tcs, applyQueryTweaks(tc, tweaks)) + } + } + return tcs +} From 1dc3820bfc2a005c59771526b2335c6e11d6d4f3 Mon Sep 17 00:00:00 2001 From: Aleksandr Razumov Date: Wed, 6 Dec 2023 12:37:33 +0300 Subject: [PATCH 2/5] feat(promcompliance): improve configuration * Add more arguments * Embed default HTML template --- .../cmd/promql-compliance-tester/main.go | 13 +++++++++---- internal/promcompliance/output/html.go | 19 ++++++++++++++----- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/internal/promcompliance/cmd/promql-compliance-tester/main.go b/internal/promcompliance/cmd/promql-compliance-tester/main.go index 4dce55c5..4fae2b85 100644 --- a/internal/promcompliance/cmd/promql-compliance-tester/main.go +++ b/internal/promcompliance/cmd/promql-compliance-tester/main.go @@ -76,9 +76,12 @@ func main() { var configFiles arrayFlags flag.Var(&configFiles, "config-file", "The path to the configuration file. If repeated, the specified files will be concatenated before YAML parsing.") outputFormat := flag.String("output-format", "text", "The comparison output format. Valid values: [text, html, json]") - outputHTMLTemplate := flag.String("output-html-template", "./output/example-output.html", "The HTML template to use when using HTML as the output format.") + outputHTMLTemplate := flag.String("output-html-template", "", "The HTML template to use when using HTML as the output format.") outputPassing := flag.Bool("output-passing", false, "Whether to also include passing test cases in the output.") queryParallelism := flag.Int("query-parallelism", 20, "Maximum number of comparison queries to run in parallel.") + startDelta := flag.Duration("start", 12*time.Minute, "The delta between the start time and current time, negated") + rangeDuration := flag.Duration("range", 10*time.Minute, "The duration of the query range.") + resolutionDuration := flag.Duration("resolution", 10*time.Second, "The resolution of the query.") flag.Parse() var outp output.Outputter @@ -114,11 +117,13 @@ func main() { comp := comparer.New(refAPI, testAPI, cfg.QueryTweaks) - end := getTime(cfg.QueryTimeParameters.EndTime, time.Now().UTC().Add(-12*time.Minute)) + end := getTime(cfg.QueryTimeParameters.EndTime, time.Now().Add(-*startDelta)) start := end.Add( - -getNonZeroDuration(cfg.QueryTimeParameters.RangeInSeconds, 10*time.Minute)) + -getNonZeroDuration(cfg.QueryTimeParameters.RangeInSeconds, *rangeDuration), + ) resolution := getNonZeroDuration( - cfg.QueryTimeParameters.ResolutionInSeconds, 10*time.Second) + cfg.QueryTimeParameters.ResolutionInSeconds, *resolutionDuration, + ) expandedTestCases := testcases.ExpandTestCases(cfg.TestCases, cfg.QueryTweaks, start, end, resolution) var wg sync.WaitGroup diff --git a/internal/promcompliance/output/html.go b/internal/promcompliance/output/html.go index 03cd7cf4..d1e1ba37 100644 --- a/internal/promcompliance/output/html.go +++ b/internal/promcompliance/output/html.go @@ -1,12 +1,10 @@ package output import ( + _ "embed" "html/template" "log" "os" - "path" - - "github.com/pkg/errors" "github.com/go-faster/oteldb/internal/promcompliance/comparer" "github.com/go-faster/oteldb/internal/promcompliance/config" @@ -42,11 +40,22 @@ var funcMap = map[string]interface{}{ }, } +//go:embed example-output.html +var defaultTemplateSource string + +func getTemplate(templatePath string) (*template.Template, error) { + t := template.New("output.html").Funcs(funcMap) + if templatePath == "" { + return t.Parse(defaultTemplateSource) + } + return t.ParseFiles(templatePath) +} + // HTML produces HTML output for a number of query results. func HTML(tplFile string) (Outputter, error) { - t, err := template.New(path.Base(tplFile)).Funcs(funcMap).ParseFiles(tplFile) + t, err := getTemplate(tplFile) if err != nil { - return nil, errors.Wrapf(err, "parsing template file %q", tplFile) + return nil, err } return func(results []*comparer.Result, includePassing bool, tweaks []*config.QueryTweak) { From ad735b8cf656f70ff75317004c6bee4e5d37c04d Mon Sep 17 00:00:00 2001 From: Aleksandr Razumov Date: Wed, 6 Dec 2023 13:02:20 +0300 Subject: [PATCH 3/5] feat(promcompliance): cleanup json --- internal/promcompliance/comparer/comparer.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/promcompliance/comparer/comparer.go b/internal/promcompliance/comparer/comparer.go index 27849ef5..9152f69a 100644 --- a/internal/promcompliance/comparer/comparer.go +++ b/internal/promcompliance/comparer/comparer.go @@ -64,10 +64,10 @@ func New(refAPI, testAPI PromAPI, queryTweaks []*config.QueryTweak) *Comparer { // Result tracks a single test case's query comparison result. type Result struct { TestCase *TestCase `json:"testCase"` - Diff string `json:"diff"` - UnexpectedFailure string `json:"unexpectedFailure"` + Diff string `json:"diff,omitempty"` + UnexpectedFailure string `json:"unexpectedFailure,omitempty"` UnexpectedSuccess bool `json:"unexpectedSuccess"` - Unsupported bool `json:"unsupported"` + Unsupported bool `json:"unsupported,omitempty"` } // Success returns true if the comparison result was successful. From a4a3dc7a35f32e99496f7a5a69809f526322e5c6 Mon Sep 17 00:00:00 2001 From: Aleksandr Razumov Date: Wed, 6 Dec 2023 13:02:28 +0300 Subject: [PATCH 4/5] feat(promcompliance): move command --- .../cmd => cmd}/promql-compliance-tester/main.go | 14 ++++++++++++-- dev/local/ch-compliance/README.md | 2 +- .../ch-compliance/cmd/compliance-wait/main.go | 9 ++++++++- dev/local/ch-compliance/run.sh | 11 +++++++---- internal/promcompliance/Dockerfile | 14 -------------- internal/promcompliance/Makefile | 3 --- 6 files changed, 28 insertions(+), 25 deletions(-) rename {internal/promcompliance/cmd => cmd}/promql-compliance-tester/main.go (91%) delete mode 100644 internal/promcompliance/Dockerfile delete mode 100644 internal/promcompliance/Makefile diff --git a/internal/promcompliance/cmd/promql-compliance-tester/main.go b/cmd/promql-compliance-tester/main.go similarity index 91% rename from internal/promcompliance/cmd/promql-compliance-tester/main.go rename to cmd/promql-compliance-tester/main.go index 4fae2b85..605269de 100644 --- a/internal/promcompliance/cmd/promql-compliance-tester/main.go +++ b/cmd/promql-compliance-tester/main.go @@ -1,3 +1,13 @@ +// Binary promql-compliance-tester performs promql compliance testing based on provided +// configuration, comparing results with reference implementation. +// +// Fork of https://github.com/prometheus/compliance. +// +// Changes: +// - Added more configuration arguments +// - Embedded default HTML template +// - Cleaned up JSON output with omitempty +// - Fixed issues reported by linters package main import ( @@ -79,7 +89,7 @@ func main() { outputHTMLTemplate := flag.String("output-html-template", "", "The HTML template to use when using HTML as the output format.") outputPassing := flag.Bool("output-passing", false, "Whether to also include passing test cases in the output.") queryParallelism := flag.Int("query-parallelism", 20, "Maximum number of comparison queries to run in parallel.") - startDelta := flag.Duration("start", 12*time.Minute, "The delta between the start time and current time, negated") + endDelta := flag.Duration("end", 12*time.Minute, "The delta between the end time and current time, negated") rangeDuration := flag.Duration("range", 10*time.Minute, "The duration of the query range.") resolutionDuration := flag.Duration("resolution", 10*time.Second, "The resolution of the query.") flag.Parse() @@ -117,7 +127,7 @@ func main() { comp := comparer.New(refAPI, testAPI, cfg.QueryTweaks) - end := getTime(cfg.QueryTimeParameters.EndTime, time.Now().Add(-*startDelta)) + end := getTime(cfg.QueryTimeParameters.EndTime, time.Now().Add(-*endDelta)) start := end.Add( -getNonZeroDuration(cfg.QueryTimeParameters.RangeInSeconds, *rangeDuration), ) diff --git a/dev/local/ch-compliance/README.md b/dev/local/ch-compliance/README.md index bfc38f61..94f0f4e4 100644 --- a/dev/local/ch-compliance/README.md +++ b/dev/local/ch-compliance/README.md @@ -4,7 +4,7 @@ https://github.com/prometheus/compliance/tree/main/promql#promql-compliance-test To build and install: ``` -cd ./compliance/promql && go install ./cmd/promql-compliance-tester && cd - +go install github.com/go-faster/oteldb/cmd/promql-compliance-tester ``` To run with targets in docker-compose: diff --git a/dev/local/ch-compliance/cmd/compliance-wait/main.go b/dev/local/ch-compliance/cmd/compliance-wait/main.go index 6ff48644..7523b4ee 100644 --- a/dev/local/ch-compliance/cmd/compliance-wait/main.go +++ b/dev/local/ch-compliance/cmd/compliance-wait/main.go @@ -1,6 +1,7 @@ package main import ( + "flag" "fmt" "net/http" "time" @@ -9,6 +10,12 @@ import ( ) func main() { + var arg struct { + Wait time.Duration + } + flag.DurationVar(&arg.Wait, "wait", time.Second*5, "wait time") + flag.Parse() + fmt.Println(">> waiting for prometheus API") bo := backoff.NewExponentialBackOff() _ = backoff.RetryNotify(func() error { @@ -27,7 +34,7 @@ func main() { fmt.Println(err) }) fmt.Println(">> prometheus api ready") - for i := 0; i < 3; i++ { + for i := 0; i < int(arg.Wait.Seconds()); i++ { fmt.Println(">> waiting for some scrapes") time.Sleep(time.Second * 1) } diff --git a/dev/local/ch-compliance/run.sh b/dev/local/ch-compliance/run.sh index 2f9b5819..70c605f1 100755 --- a/dev/local/ch-compliance/run.sh +++ b/dev/local/ch-compliance/run.sh @@ -2,12 +2,15 @@ set -e -x -go install github.com/go-faster/oteldb/internal/promcompliance/cmd/promql-compliance-tester - docker compose up -d --remove-orphans --build --force-recreate -go run ./cmd/compliance-wait + +go run ./cmd/compliance-wait -wait 10s echo ">> Testing oteldb implementation" -promql-compliance-tester -config-file promql-test-queries.yml -config-file test-oteldb.yml | tee result.oteldb.txt || true +RANGE="1m" +END="1m" +go run github.com/go-faster/oteldb/cmd/promql-compliance-tester \ + -end "${END}" -range "${RANGE}" \ + -config-file promql-test-queries.yml -config-file test-oteldb.yml | tee result.oteldb.txt || true docker compose down -v diff --git a/internal/promcompliance/Dockerfile b/internal/promcompliance/Dockerfile deleted file mode 100644 index 8da27213..00000000 --- a/internal/promcompliance/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM golang:1.18 as build-env - -WORKDIR /go/src/promql -COPY . /go/src/promql - -ENV CGO_ENABLED 0 - -RUN go build ./cmd/promql-compliance-tester - -FROM quay.io/prometheus/busybox -COPY --from=build-env /go/src/promql/promql-compliance-tester / -COPY --from=build-env /go/src/promql/promql-test-queries.yml / - -ENTRYPOINT ["/promql-compliance-tester"] diff --git a/internal/promcompliance/Makefile b/internal/promcompliance/Makefile deleted file mode 100644 index a7b3db8e..00000000 --- a/internal/promcompliance/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -.PHONY: docker -docker: - docker build . -t promql-compliance-tester:latest From 99838ac082d0e899c43fbd5a30cf3739eb692c10 Mon Sep 17 00:00:00 2001 From: Aleksandr Razumov Date: Wed, 6 Dec 2023 13:04:11 +0300 Subject: [PATCH 5/5] chore(compliance): more explicit gitignore --- dev/local/ch-compliance/.gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev/local/ch-compliance/.gitignore b/dev/local/ch-compliance/.gitignore index 75cc5fbc..e5e4a408 100644 --- a/dev/local/ch-compliance/.gitignore +++ b/dev/local/ch-compliance/.gitignore @@ -1 +1,3 @@ -result.* \ No newline at end of file +result.*.txt +result.*.json +result.*.html \ No newline at end of file