feat(promcompliance): vendorfork and improve

.codecov.yml

@@ -6,6 +6,8 @@ ignore:
   # Skip non-production code.
   - "internal/lokiproxy/"
   - "internal/pyroproxy/"
+  # Skip vendored-forked
+  - "internal/promcompliance/"
 coverage:
   status:
     project: false

.gitmodules

@@ -4,6 +4,3 @@
 [submodule "dev/local/ch-full/opentelemetry-collector-contrib"]
 	path = dev/local/ch-full/opentelemetry-collector-contrib
 	url = https://github.com/open-telemetry/opentelemetry-collector-contrib.git
-[submodule "dev/local/ch-compliance/compliance"]
-	path = dev/local/ch-compliance/compliance
-	url = https://github.com/prometheus/compliance.git

.golangci.yml

@@ -116,8 +116,16 @@ issues:
       text: "ST1003" # underscores lol
 
     - path: (internal|cmd)
+      linters: [revive, stylecheck]
+      text: "comment"
+
+    - path: 'internal/promcompliance'
       linters: [revive]
-      text: "package-comments"
+      text: "exported:"
+
+    - path: 'internal/promcompliance'
+      linters: [gocritic]
+      text: "ifElseChain:"
 
     - linters: [revive]
       text: "comment on exported const .+ should be of the form"

cmd/promql-compliance-tester/main.go

@@ -0,0 +1,200 @@
+// Binary promql-compliance-tester performs promql compliance testing based on provided
+// configuration, comparing results with reference implementation.
+//
+// Fork of https://github.com/prometheus/compliance.
+//
+// Changes:
+// - Added more configuration arguments
+// - Embedded default HTML template
+// - Cleaned up JSON output with omitempty
+// - Fixed issues reported by linters
+package main
+
+import (
+	"flag"
+	"log"
+	"math"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/cheggaaa/pb/v3"
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/api"
+	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
+	"go.uber.org/atomic"
+
+	"github.com/go-faster/oteldb/internal/promcompliance/comparer"
+	"github.com/go-faster/oteldb/internal/promcompliance/config"
+	"github.com/go-faster/oteldb/internal/promcompliance/output"
+	"github.com/go-faster/oteldb/internal/promcompliance/testcases"
+)
+
+func newPromAPI(targetConfig config.TargetConfig) (v1.API, error) {
+	apiConfig := api.Config{Address: targetConfig.QueryURL}
+	if len(targetConfig.Headers) > 0 || targetConfig.BasicAuthUser != "" {
+		apiConfig.RoundTripper = roundTripperWithSettings{headers: targetConfig.Headers, basicAuthUser: targetConfig.BasicAuthUser, basicAuthPass: targetConfig.BasicAuthPass}
+	}
+	client, err := api.NewClient(apiConfig)
+	if err != nil {
+		return nil, errors.Wrapf(err, "creating Prometheus API client for %q: %v", targetConfig.QueryURL, err)
+	}
+
+	return v1.NewAPI(client), nil
+}
+
+type roundTripperWithSettings struct {
+	headers       map[string]string
+	basicAuthUser string
+	basicAuthPass string
+}
+
+func (rt roundTripperWithSettings) RoundTrip(req *http.Request) (*http.Response, error) {
+	// Per RoundTrip's documentation, RoundTrip should not modify the request,
+	// except for consuming and closing the Request's Body.
+	// TODO: Update the Go Prometheus client code to support adding headers to request.
+
+	if rt.basicAuthUser != "" {
+		req.SetBasicAuth(rt.basicAuthUser, rt.basicAuthPass)
+	}
+
+	for key, value := range rt.headers {
+		if strings.EqualFold(key, "host") {
+			req.Host = value
+		} else {
+			req.Header.Add(key, value)
+		}
+	}
+	return http.DefaultTransport.RoundTrip(req)
+}
+
+type arrayFlags []string
+
+func (i *arrayFlags) String() string {
+	return "my string representation"
+}
+
+func (i *arrayFlags) Set(value string) error {
+	*i = append(*i, value)
+	return nil
+}
+
+func main() {
+	var configFiles arrayFlags
+	flag.Var(&configFiles, "config-file", "The path to the configuration file. If repeated, the specified files will be concatenated before YAML parsing.")
+	outputFormat := flag.String("output-format", "text", "The comparison output format. Valid values: [text, html, json]")
+	outputHTMLTemplate := flag.String("output-html-template", "", "The HTML template to use when using HTML as the output format.")
+	outputPassing := flag.Bool("output-passing", false, "Whether to also include passing test cases in the output.")
+	queryParallelism := flag.Int("query-parallelism", 20, "Maximum number of comparison queries to run in parallel.")
+	endDelta := flag.Duration("end", 12*time.Minute, "The delta between the end time and current time, negated")
+	rangeDuration := flag.Duration("range", 10*time.Minute, "The duration of the query range.")
+	resolutionDuration := flag.Duration("resolution", 10*time.Second, "The resolution of the query.")
+	flag.Parse()
+
+	var outp output.Outputter
+	switch *outputFormat {
+	case "text":
+		outp = output.Text
+	case "html":
+		var err error
+		outp, err = output.HTML(*outputHTMLTemplate)
+		if err != nil {
+			log.Fatalf("Error reading output HTML template: %v", err)
+		}
+	case "json":
+		outp = output.JSON
+	case "tsv":
+		outp = output.TSV
+	default:
+		log.Fatalf("Invalid output format %q", *outputFormat)
+	}
+
+	cfg, err := config.LoadFromFiles(configFiles)
+	if err != nil {
+		log.Fatalf("Error loading configuration file: %v", err)
+	}
+	refAPI, err := newPromAPI(cfg.ReferenceTargetConfig)
+	if err != nil {
+		log.Fatalf("Error creating reference API: %v", err)
+	}
+	testAPI, err := newPromAPI(cfg.TestTargetConfig)
+	if err != nil {
+		log.Fatalf("Error creating test API: %v", err)
+	}
+
+	comp := comparer.New(refAPI, testAPI, cfg.QueryTweaks)
+
+	end := getTime(cfg.QueryTimeParameters.EndTime, time.Now().Add(-*endDelta))
+	start := end.Add(
+		-getNonZeroDuration(cfg.QueryTimeParameters.RangeInSeconds, *rangeDuration),
+	)
+	resolution := getNonZeroDuration(
+		cfg.QueryTimeParameters.ResolutionInSeconds, *resolutionDuration,
+	)
+	expandedTestCases := testcases.ExpandTestCases(cfg.TestCases, cfg.QueryTweaks, start, end, resolution)
+
+	var wg sync.WaitGroup
+	results := make([]*comparer.Result, len(expandedTestCases))
+	progressBar := pb.StartNew(len(results))
+	wg.Add(len(results))
+
+	workCh := make(chan struct{}, *queryParallelism)
+
+	allSuccess := atomic.NewBool(true)
+	for i, tc := range expandedTestCases {
+		workCh <- struct{}{}
+
+		go func(i int, tc *comparer.TestCase) {
+			res, err := comp.Compare(tc)
+			if err != nil {
+				log.Fatalf("Error running comparison: %v", err)
+			}
+			results[i] = res
+			if !res.Success() {
+				allSuccess.Store(false)
+			}
+			progressBar.Increment()
+			<-workCh
+			wg.Done()
+		}(i, tc)
+	}
+
+	wg.Wait()
+	progressBar.Finish()
+
+	outp(results, *outputPassing, cfg.QueryTweaks)
+
+	if !allSuccess.Load() {
+		os.Exit(1)
+	}
+}
+
+func getTime(timeStr string, defaultTime time.Time) time.Time {
+	result, err := parseTime(timeStr)
+	if err != nil {
+		return defaultTime
+	}
+	return result
+}
+
+func getNonZeroDuration(
+	seconds float64, defaultDuration time.Duration) time.Duration {
+	if seconds == 0.0 {
+		return defaultDuration
+	}
+	return time.Duration(seconds * float64(time.Second))
+}
+
+func parseTime(s string) (time.Time, error) {
+	if t, err := strconv.ParseFloat(s, 64); err == nil {
+		s, ns := math.Modf(t)
+		return time.Unix(int64(s), int64(ns*float64(time.Second))).UTC(), nil
+	}
+	if t, err := time.Parse(time.RFC3339Nano, s); err == nil {
+		return t, nil
+	}
+	return time.Time{}, errors.Errorf("cannot parse %q to a valid timestamp", s)
+}

dev/local/ch-compliance/.gitignore

@@ -1 +1,3 @@
-result.*
+result.*.txt
+result.*.json
+result.*.html

dev/local/ch-compliance/README.md

@@ -4,16 +4,16 @@ https://github.com/prometheus/compliance/tree/main/promql#promql-compliance-test
 
 To build and install:
 ```
-cd ./compliance/promql && go install ./cmd/promql-compliance-tester && cd -
+go install github.com/go-faster/oteldb/cmd/promql-compliance-tester
 ```
 
 To run with targets in docker-compose:
 ```console
-promql-compliance-tester -config-file promql-test-queries.yml -config-file test.local.yml
+promql-compliance-tester -config-file promql-test-queries.yml -config-file test-oteldb.yml
 ```
 
 **NOTE:**
-Results will be false-positive until enough data (5-10min?) is gathered.
+Results will be false-positive until enough data (~20min) is gathered.
 
 This check was disabled as being broken on latest prometheus reference:
 ```yaml

dev/local/ch-compliance/cmd/compliance-wait/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"flag"
 	"fmt"
 	"net/http"
 	"time"
@@ -9,6 +10,12 @@ import (
 )
 
 func main() {
+	var arg struct {
+		Wait time.Duration
+	}
+	flag.DurationVar(&arg.Wait, "wait", time.Second*5, "wait time")
+	flag.Parse()
+
 	fmt.Println(">> waiting for prometheus API")
 	bo := backoff.NewExponentialBackOff()
 	_ = backoff.RetryNotify(func() error {
@@ -27,7 +34,7 @@ func main() {
 		fmt.Println(err)
 	})
 	fmt.Println(">> prometheus api ready")
-	for i := 0; i < 3; i++ {
+	for i := 0; i < int(arg.Wait.Seconds()); i++ {
 		fmt.Println(">> waiting for some scrapes")
 		time.Sleep(time.Second * 1)
 	}

dev/local/ch-compliance/run.sh

@@ -2,12 +2,15 @@
 
 set -e -x
 
-cd ./compliance/promql && go install ./cmd/promql-compliance-tester && cd -
-
 docker compose up -d --remove-orphans --build --force-recreate
-go run ./cmd/compliance-wait
+
+go run ./cmd/compliance-wait -wait 10s
 
 echo ">> Testing oteldb implementation"
-promql-compliance-tester -config-file promql-test-queries.yml -config-file test.oteldb.yml | tee result.oteldb.txt || true
+RANGE="1m"
+END="1m"
+go run github.com/go-faster/oteldb/cmd/promql-compliance-tester \
+  -end "${END}" -range "${RANGE}" \
+  -config-file promql-test-queries.yml -config-file test-oteldb.yml | tee result.oteldb.txt || true
 
 docker compose down -v

dev/local/ch-compliance/test.oteldb.yml → dev/local/ch-compliance/test-oteldb.yml

@@ -1,4 +1,4 @@
-# promql-compliance-tester -config-file promql-test-queries.yml -config-file test.oteldb.yml
+# promql-compliance-tester -config-file promql-test-queries.yml -config-file test-oteldb.yml
 
 reference_target_config:
   query_url: http://localhost:9091