From d61428670c838d604b40c4251f25d1db66a15c03 Mon Sep 17 00:00:00 2001 From: Anurag Rajawat Date: Sat, 23 Nov 2024 21:47:22 +0530 Subject: [PATCH] feat: Add API Speculator module for OAS Signed-off-by: Anurag Rajawat --- speculator/Dockerfile | 4 + speculator/Makefile | 73 + speculator/cmd/root.go | 56 + speculator/cmd/version.go | 29 + speculator/config/default.yaml | 30 + speculator/go.mod | 106 + speculator/go.sum | 354 +++ speculator/main.go | 9 + speculator/notes | 5 + speculator/pkg/apispec/approved_spec.go | 40 + speculator/pkg/apispec/conflict.go | 78 + speculator/pkg/apispec/conflict_test.go | 171 ++ speculator/pkg/apispec/constants.go | 15 + speculator/pkg/apispec/default.go | 43 + speculator/pkg/apispec/diff.go | 288 ++ speculator/pkg/apispec/diff_test.go | 1140 ++++++++ speculator/pkg/apispec/form.go | 78 + speculator/pkg/apispec/form_test.go | 215 ++ speculator/pkg/apispec/format.go | 85 + speculator/pkg/apispec/format_test.go | 116 + speculator/pkg/apispec/headers.go | 92 + speculator/pkg/apispec/headers_test.go | 284 ++ speculator/pkg/apispec/learning_spec.go | 24 + speculator/pkg/apispec/merge.go | 516 ++++ speculator/pkg/apispec/merge_test.go | 2472 +++++++++++++++++ speculator/pkg/apispec/operation.go | 463 +++ speculator/pkg/apispec/operation_test.go | 605 ++++ speculator/pkg/apispec/path_item.go | 76 + speculator/pkg/apispec/path_params.go | 160 ++ speculator/pkg/apispec/path_params_test.go | 294 ++ speculator/pkg/apispec/provided_spec.go | 354 +++ speculator/pkg/apispec/provided_spec_test.go | 1819 ++++++++++++ speculator/pkg/apispec/query.go | 38 + speculator/pkg/apispec/query_test.go | 96 + speculator/pkg/apispec/review.go | 158 ++ speculator/pkg/apispec/review_test.go | 904 ++++++ speculator/pkg/apispec/schema.go | 146 + speculator/pkg/apispec/schema_test.go | 84 + speculator/pkg/apispec/schemas.go | 157 ++ speculator/pkg/apispec/schemas_test.go | 579 ++++ speculator/pkg/apispec/security.go | 150 + speculator/pkg/apispec/security_test.go | 127 + speculator/pkg/apispec/spec.go | 354 +++ speculator/pkg/apispec/spec_test.go | 281 ++ speculator/pkg/apispec/spec_version.go | 57 + speculator/pkg/apispec/spec_version_test.go | 67 + speculator/pkg/apispec/testing.go | 205 ++ speculator/pkg/apispec/utils.go | 57 + speculator/pkg/apispec/utils_test.go | 61 + speculator/pkg/config/config.go | 121 + speculator/pkg/core/core.go | 45 + speculator/pkg/database/database.go | 67 + speculator/pkg/pathtrie/path_trie.go | 219 ++ speculator/pkg/pathtrie/path_trie_test.go | 998 +++++++ speculator/pkg/speculator/map_repository.go | 85 + speculator/pkg/speculator/speculator.go | 286 ++ .../pkg/speculator/speculator_accesssor.go | 42 + speculator/pkg/speculator/speculator_test.go | 125 + speculator/pkg/util/errors/errors.go | 5 + speculator/pkg/util/logger.go | 24 + speculator/pkg/util/mime.go | 11 + speculator/pkg/util/mime_test.go | 52 + speculator/pkg/util/path_param.go | 15 + 63 files changed, 15710 insertions(+) create mode 100644 speculator/Dockerfile create mode 100644 speculator/Makefile create mode 100644 speculator/cmd/root.go create mode 100644 speculator/cmd/version.go create mode 100644 speculator/config/default.yaml create mode 100644 speculator/go.mod create mode 100644 speculator/go.sum create mode 100644 speculator/main.go create mode 100644 speculator/notes create mode 100644 speculator/pkg/apispec/approved_spec.go create mode 100644 speculator/pkg/apispec/conflict.go create mode 100644 speculator/pkg/apispec/conflict_test.go create mode 100644 speculator/pkg/apispec/constants.go create mode 100644 speculator/pkg/apispec/default.go create mode 100644 speculator/pkg/apispec/diff.go create mode 100644 speculator/pkg/apispec/diff_test.go create mode 100644 speculator/pkg/apispec/form.go create mode 100644 speculator/pkg/apispec/form_test.go create mode 100644 speculator/pkg/apispec/format.go create mode 100644 speculator/pkg/apispec/format_test.go create mode 100644 speculator/pkg/apispec/headers.go create mode 100644 speculator/pkg/apispec/headers_test.go create mode 100644 speculator/pkg/apispec/learning_spec.go create mode 100644 speculator/pkg/apispec/merge.go create mode 100644 speculator/pkg/apispec/merge_test.go create mode 100644 speculator/pkg/apispec/operation.go create mode 100644 speculator/pkg/apispec/operation_test.go create mode 100644 speculator/pkg/apispec/path_item.go create mode 100644 speculator/pkg/apispec/path_params.go create mode 100644 speculator/pkg/apispec/path_params_test.go create mode 100644 speculator/pkg/apispec/provided_spec.go create mode 100644 speculator/pkg/apispec/provided_spec_test.go create mode 100644 speculator/pkg/apispec/query.go create mode 100644 speculator/pkg/apispec/query_test.go create mode 100644 speculator/pkg/apispec/review.go create mode 100644 speculator/pkg/apispec/review_test.go create mode 100644 speculator/pkg/apispec/schema.go create mode 100644 speculator/pkg/apispec/schema_test.go create mode 100644 speculator/pkg/apispec/schemas.go create mode 100644 speculator/pkg/apispec/schemas_test.go create mode 100644 speculator/pkg/apispec/security.go create mode 100644 speculator/pkg/apispec/security_test.go create mode 100644 speculator/pkg/apispec/spec.go create mode 100644 speculator/pkg/apispec/spec_test.go create mode 100644 speculator/pkg/apispec/spec_version.go create mode 100644 speculator/pkg/apispec/spec_version_test.go create mode 100644 speculator/pkg/apispec/testing.go create mode 100644 speculator/pkg/apispec/utils.go create mode 100644 speculator/pkg/apispec/utils_test.go create mode 100644 speculator/pkg/config/config.go create mode 100644 speculator/pkg/core/core.go create mode 100644 speculator/pkg/database/database.go create mode 100644 speculator/pkg/pathtrie/path_trie.go create mode 100644 speculator/pkg/pathtrie/path_trie_test.go create mode 100644 speculator/pkg/speculator/map_repository.go create mode 100644 speculator/pkg/speculator/speculator.go create mode 100644 speculator/pkg/speculator/speculator_accesssor.go create mode 100644 speculator/pkg/speculator/speculator_test.go create mode 100644 speculator/pkg/util/errors/errors.go create mode 100644 speculator/pkg/util/logger.go create mode 100644 speculator/pkg/util/mime.go create mode 100644 speculator/pkg/util/mime_test.go create mode 100644 speculator/pkg/util/path_param.go diff --git a/speculator/Dockerfile b/speculator/Dockerfile new file mode 100644 index 0000000..983409c --- /dev/null +++ b/speculator/Dockerfile @@ -0,0 +1,4 @@ +FROM ubuntu:latest +LABEL authors="anurag" + +ENTRYPOINT ["top", "-b"] \ No newline at end of file diff --git a/speculator/Makefile b/speculator/Makefile new file mode 100644 index 0000000..8c1be38 --- /dev/null +++ b/speculator/Makefile @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2024 Authors of SentryFlow + +BINARY_NAME ?= speculator +REGISTRY ?= docker.io/5gsec +VERSION ?= $(shell git rev-parse HEAD) +BUILD_TS ?= $(shell date) +DOCKER_IMAGE ?= $(REGISTRY)/$(BINARY_NAME) +DOCKER_TAG ?= latest +CONTAINER_TOOL ?= docker + +.PHONY: help +help: ## Display this help + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +.DEFAULT_GOAL := help + +##@ Development +.PHONY: run +run: fmt vet build ## Run speculator on your host + @./bin/"${BINARY_NAME}" --development true + +.PHONY: fmt +fmt: ## Run go fmt against code + @go fmt ./... + +.PHONY: vet +vet: ## Run go vet against code + @go vet ./... + +GOLANGCI_LINT = $(shell pwd)/bin/golangci-lint +GOLANGCI_LINT_VERSION ?= v1.60.3 +golangci-lint: + @[ -f $(GOLANGCI_LINT) ] || { \ + set -e ;\ + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell dirname $(GOLANGCI_LINT)) $(GOLANGCI_LINT_VERSION) ;\ + } + +.PHONY: lint +lint: golangci-lint ## Run golangci-lint linter + @$(GOLANGCI_LINT) run + +.PHONY: license +license: ## Check and fix license header on all go files + @../scripts/add-license-header + +.PHONY: test +test: ## Run unit tests + @go test -v ./... +##@ Build + +.PHONY: build +build: fmt vet ## Build speculator binary + @CGO_ENABLED=0 go build -ldflags="-s" -o bin/"${BINARY_NAME}" . + +.PHONY: image +image: ## Build speculator's container image + $(CONTAINER_TOOL) build -t ${DOCKER_IMAGE}:${DOCKER_TAG} -f Dockerfile ../ + +.PHONY: push +push: ## Push speculator's container image + $(CONTAINER_TOOL) push ${DOCKER_IMAGE}:${DOCKER_TAG} + +PLATFORMS ?= linux/arm64,linux/amd64 +.PHONY: imagex +imagex: ## Build and push speculator's container image for cross-platform support + # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile + sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross + - $(CONTAINER_TOOL) buildx create --name project-v3-builder + $(CONTAINER_TOOL) buildx use project-v3-builder + - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${DOCKER_IMAGE}:${DOCKER_TAG} -f Dockerfile.cross ../ || { $(CONTAINER_TOOL) buildx rm project-v3-builder; rm Dockerfile.cross; exit 1; } + - $(CONTAINER_TOOL) buildx rm project-v3-builder + rm Dockerfile.cross diff --git a/speculator/cmd/root.go b/speculator/cmd/root.go new file mode 100644 index 0000000..cbf16e7 --- /dev/null +++ b/speculator/cmd/root.go @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2024 Authors of SentryFlow + +package cmd + +import ( + "context" + + "github.com/spf13/cobra" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + ctrl "sigs.k8s.io/controller-runtime" + + "github.com/5gsec/sentryflow/speculator/pkg/core" + "github.com/5gsec/sentryflow/speculator/pkg/util" +) + +var ( + configFilePath string + //kubeConfig string + debugMode bool + logger *zap.SugaredLogger +) + +func init() { + RootCmd.PersistentFlags().StringVar(&configFilePath, "config", "", "config file path") + //RootCmd.PersistentFlags().StringVar(&kubeConfig, "kubeconfig", "", "kubeconfig file path") + RootCmd.PersistentFlags().BoolVar(&debugMode, "debug", false, "run in debug mode") +} + +var RootCmd = &cobra.Command{ + Use: "speculator", + Run: func(cmd *cobra.Command, args []string) { + run() + }, +} + +func run() { + initLogger(debugMode) + logBuildInfo() + ctx := context.WithValue(ctrl.SetupSignalHandler(), util.LoggerContextKey{}, logger) + core.Run(ctx, configFilePath) +} + +func initLogger(debug bool) { + cfg := zap.NewProductionConfig() + cfg.EncoderConfig.EncodeLevel = zapcore.CapitalLevelEncoder + if debug { + cfg = zap.NewDevelopmentConfig() + cfg.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder + } + cfg.EncoderConfig.TimeKey = "timestamp" + cfg.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + coreLogger, _ := cfg.Build() + logger = coreLogger.Sugar() +} diff --git a/speculator/cmd/version.go b/speculator/cmd/version.go new file mode 100644 index 0000000..97359d6 --- /dev/null +++ b/speculator/cmd/version.go @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2024 Authors of SentryFlow + +package cmd + +import ( + "runtime" + "runtime/debug" +) + +func logBuildInfo() { + info, _ := debug.ReadBuildInfo() + vcsRev := "" + vcsTime := "" + for _, s := range info.Settings { + if s.Key == "vcs.revision" { + vcsRev = s.Value + } else if s.Key == "vcs.time" { + vcsTime = s.Value + } + } + logger.Infof("git revision: %s, build time: %s, build version: %s, go os/arch: %s/%s", + vcsRev, + vcsTime, + info.Main.Version, + runtime.GOOS, + runtime.GOARCH, + ) +} diff --git a/speculator/config/default.yaml b/speculator/config/default.yaml new file mode 100644 index 0000000..1908228 --- /dev/null +++ b/speculator/config/default.yaml @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2024 Authors of SentryFlow +# +#kmux: +# sink: +# stream: rabbitmq + +rabbitmq: + host: 127.0.0.1 + port: 5672 + username: "" + password: "" + exchange: + name: sentryflow + type: direct + durable: true + auto-delete: true + queueName: test + +database: + logLevel: debug + uri: mongodb://127.0.0.1:27017/?directConnection=true + user: speculator + password: speculator + name: speculator + +# tls: +# enabled: false +# cert-file: "" +# skip-verify: false diff --git a/speculator/go.mod b/speculator/go.mod new file mode 100644 index 0000000..364bd85 --- /dev/null +++ b/speculator/go.mod @@ -0,0 +1,106 @@ +module github.com/5gsec/sentryflow/speculator + +go 1.23.3 + +require ( + github.com/getkin/kin-openapi v0.128.0 + github.com/ghodss/yaml v1.0.0 + github.com/gofrs/uuid v4.4.0+incompatible + github.com/golang-jwt/jwt/v5 v5.2.1 + github.com/sirupsen/logrus v1.9.3 + github.com/spf13/cast v1.7.0 + github.com/spf13/cobra v1.8.1 + github.com/spf13/viper v1.19.0 + github.com/xeipuuv/gojsonschema v1.2.0 + github.com/yudai/gojsondiff v1.0.0 + go.mongodb.org/mongo-driver v1.17.1 + go.uber.org/zap v1.27.0 + k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 + sigs.k8s.io/controller-runtime v0.19.2 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/imdario/mergo v0.3.6 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/invopop/yaml v0.3.1 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.2 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/montanaflynn/stats v0.7.1 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/onsi/ginkgo v1.16.5 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/perimeterx/marshmallow v1.1.5 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_golang v1.19.1 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sergi/go-diff v1.3.1 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/x448/float16 v0.8.4 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect + github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect + github.com/yudai/pp v2.0.1+incompatible // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.26.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/oauth2 v0.21.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.23.0 // indirect + golang.org/x/term v0.23.0 // indirect + golang.org/x/text v0.17.0 // indirect + golang.org/x/time v0.5.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/api v0.31.0 // indirect + k8s.io/apiextensions-apiserver v0.31.0 // indirect + k8s.io/apimachinery v0.31.0 // indirect + k8s.io/client-go v0.31.0 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/speculator/go.sum b/speculator/go.sum new file mode 100644 index 0000000..cd7b448 --- /dev/null +++ b/speculator/go.sum @@ -0,0 +1,354 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/getkin/kin-openapi v0.128.0 h1:jqq3D9vC9pPq1dGcOCv7yOp1DaEe7c/T1vzcLbITSp4= +github.com/getkin/kin-openapi v0.128.0/go.mod h1:OZrfXzUfGrNbsKj+xmFBx6E5c6yH3At/tAKSc2UszXM= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= +github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= +github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af h1:kmjWCqn2qkEml422C2Rrd27c3VGxi6a/6HNq8QmHRKM= +github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/invopop/yaml v0.3.1 h1:f0+ZpmhfBSS4MhG+4HYseMdJhoeeopbSKbq5Rpeelso= +github.com/invopop/yaml v0.3.1/go.mod h1:PMOp3nn4/12yEZUFfmOuNHJsZToEEOwoWsT+D81KkeA= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= +github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= +github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= +github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= +github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= +github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= +github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= +github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= +github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= +github.com/yudai/pp v2.0.1+incompatible h1:Q4//iY4pNF6yPLZIigmvcl7k/bPgrcTPIFIcmawg5bI= +github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.mongodb.org/mongo-driver v1.17.1 h1:Wic5cJIwJgSpBhe3lx3+/RybR5PiYRMpVFgO7cOHyIM= +go.mongodb.org/mongo-driver v1.17.1/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= +golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.31.0 h1:b9LiSjR2ym/SzTOlfMHm1tr7/21aD7fSkqgD/CVJBCo= +k8s.io/api v0.31.0/go.mod h1:0YiFF+JfFxMM6+1hQei8FY8M7s1Mth+z/q7eF1aJkTE= +k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24kyLSk= +k8s.io/apiextensions-apiserver v0.31.0/go.mod h1:b9aMDEYaEe5sdK+1T0KU78ApR/5ZVp4i56VacZYEHxk= +k8s.io/apimachinery v0.31.0 h1:m9jOiSr3FoSSL5WO9bjm1n6B9KROYYgNZOb4tyZ1lBc= +k8s.io/apimachinery v0.31.0/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8= +k8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/controller-runtime v0.19.2 h1:3sPrF58XQEPzbE8T81TN6selQIMGbtYwuaJ6eDssDF8= +sigs.k8s.io/controller-runtime v0.19.2/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/speculator/main.go b/speculator/main.go new file mode 100644 index 0000000..3b7024d --- /dev/null +++ b/speculator/main.go @@ -0,0 +1,9 @@ +package main + +import ( + "github.com/5gsec/sentryflow/speculator/cmd" +) + +func main() { + _ = cmd.RootCmd.Execute() +} diff --git a/speculator/notes b/speculator/notes new file mode 100644 index 0000000..ee62579 --- /dev/null +++ b/speculator/notes @@ -0,0 +1,5 @@ +Group by to save in DB: + - base path + method + +Parsing: +- Path \ No newline at end of file diff --git a/speculator/pkg/apispec/approved_spec.go b/speculator/pkg/apispec/approved_spec.go new file mode 100644 index 0000000..813fd06 --- /dev/null +++ b/speculator/pkg/apispec/approved_spec.go @@ -0,0 +1,40 @@ +package apispec + +import ( + "encoding/json" + "fmt" + + "github.com/getkin/kin-openapi/openapi3" +) + +type ApprovedSpec struct { + PathItems map[string]*openapi3.PathItem + SecuritySchemes openapi3.SecuritySchemes + SpecVersion OASVersion +} + +func (a *ApprovedSpec) GetPathItem(path string) *openapi3.PathItem { + if pi, exists := a.PathItems[path]; exists { + return pi + } + return nil +} + +func (a *ApprovedSpec) GetSpecVersion() OASVersion { + return a.SpecVersion +} + +func (a *ApprovedSpec) Clone() (*ApprovedSpec, error) { + clonedApprovedSpec := new(ApprovedSpec) + + approvedSpecB, err := json.Marshal(a) + if err != nil { + return nil, fmt.Errorf("failed to marshal approved spec: %w", err) + } + + if err := json.Unmarshal(approvedSpecB, &clonedApprovedSpec); err != nil { + return nil, fmt.Errorf("failed to unmarshal approved spec: %w", err) + } + + return clonedApprovedSpec, nil +} diff --git a/speculator/pkg/apispec/conflict.go b/speculator/pkg/apispec/conflict.go new file mode 100644 index 0000000..998ae2f --- /dev/null +++ b/speculator/pkg/apispec/conflict.go @@ -0,0 +1,78 @@ +package apispec + +import ( + "fmt" + + "github.com/getkin/kin-openapi/openapi3" + "k8s.io/utils/field" +) + +type conflict struct { + path *field.Path + obj1 any + obj2 any + msg string +} + +func createConflictMsg(path *field.Path, t1, t2 any) string { + return fmt.Sprintf("%s: type mismatch: %+v != %+v", path, t1, t2) +} + +func createHeaderInConflictMsg(path *field.Path, in, in2 any) string { + return fmt.Sprintf("%s: header in mismatch: %+v != %+v", path, in, in2) +} + +func (c conflict) String() string { + return c.msg +} + +const ( + NoConflict = iota + PreferType1 + PreferType2 + ConflictUnresolved +) + +// conflictSolver will get 2 types and returns: +// +// NoConflict - type1 and type2 are equal +// PreferType1 - type1 should be used +// PreferType2 - type2 should be used +// ConflictUnresolved - types conflict can't be resolved +func conflictSolver(type1, type2 *openapi3.Types) int { + if type1.Is(type2.Slice()[0]) { + return NoConflict + } + + if shouldPreferType(type1, type2) { + return PreferType1 + } + + if shouldPreferType(type2, type1) { + return PreferType2 + } + + return ConflictUnresolved +} + +// shouldPreferType return true if type1 should be preferred over type2. +func shouldPreferType(type1, type2 *openapi3.Types) bool { + if type1.Includes(openapi3.TypeBoolean) || + type1.Includes(openapi3.TypeObject) || + type1.Includes(openapi3.TypeArray) { + // Should not prefer boolean, object and array type over any other type. + return false + } + + if type1.Includes(openapi3.TypeNumber) { + // Preferring number to integer type. + return type2.Includes(openapi3.TypeInteger) + } + + if type1.Includes(openapi3.TypeString) { + // Preferring string to any type. + return true + } + + return false +} diff --git a/speculator/pkg/apispec/conflict_test.go b/speculator/pkg/apispec/conflict_test.go new file mode 100644 index 0000000..6909389 --- /dev/null +++ b/speculator/pkg/apispec/conflict_test.go @@ -0,0 +1,171 @@ +package apispec + +import ( + "testing" + + "github.com/getkin/kin-openapi/openapi3" +) + +func Test_shouldPreferType(t *testing.T) { + type args struct { + t1 *openapi3.Types + t2 *openapi3.Types + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "should not prefer - bool", + args: args{ + t1: &openapi3.Types{ + openapi3.TypeBoolean, + }, + }, + want: false, + }, + { + name: "should not prefer - obj", + args: args{ + t1: &openapi3.Types{ + openapi3.TypeObject, + }, + }, + want: false, + }, + { + name: "should not prefer - array", + args: args{ + t1: &openapi3.Types{ + openapi3.TypeArray, + }, + }, + want: false, + }, + { + name: "should not prefer - number over object", + args: args{ + t1: &openapi3.Types{ + openapi3.TypeNumber, + }, + t2: &openapi3.Types{ + openapi3.TypeObject, + }, + }, + want: false, + }, + { + name: "prefer - number over int", + args: args{ + t1: &openapi3.Types{ + openapi3.TypeNumber, + }, + t2: &openapi3.Types{ + openapi3.TypeInteger, + }, + }, + want: true, + }, + { + name: "prefer - string over anything", + args: args{ + t1: &openapi3.Types{ + openapi3.TypeString, + }, + }, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := shouldPreferType(tt.args.t1, tt.args.t2); got != tt.want { + t.Errorf("shouldPreferType() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_conflictSolver(t *testing.T) { + type args struct { + t1 *openapi3.Types + t2 *openapi3.Types + } + tests := []struct { + name string + args args + want int + }{ + { + name: "no conflict", + args: args{ + t1: &openapi3.Types{openapi3.TypeNumber}, + t2: &openapi3.Types{openapi3.TypeNumber}, + }, + want: NoConflict, + }, + { + name: "prefer string over anything", + args: args{ + t1: &openapi3.Types{openapi3.TypeString}, + t2: &openapi3.Types{openapi3.TypeNumber}, + }, + want: PreferType1, + }, + { + name: "prefer string over anything", + args: args{ + t1: &openapi3.Types{openapi3.TypeInteger}, + t2: &openapi3.Types{openapi3.TypeString}, + }, + want: PreferType2, + }, + { + name: "prefer number over int", + args: args{ + t1: &openapi3.Types{openapi3.TypeNumber}, + t2: &openapi3.Types{openapi3.TypeInteger}, + }, + want: PreferType1, + }, + { + name: "prefer number over int", + args: args{ + t1: &openapi3.Types{openapi3.TypeInteger}, + t2: &openapi3.Types{openapi3.TypeNumber}, + }, + want: PreferType2, + }, + { + name: "conflict - bool", + args: args{ + t1: &openapi3.Types{openapi3.TypeInteger}, + t2: &openapi3.Types{openapi3.TypeBoolean}, + }, + want: ConflictUnresolved, + }, + { + name: "conflict - obj", + args: args{ + t1: &openapi3.Types{openapi3.TypeObject}, + t2: &openapi3.Types{openapi3.TypeBoolean}, + }, + want: ConflictUnresolved, + }, + { + name: "conflict - array", + args: args{ + t1: &openapi3.Types{openapi3.TypeObject}, + t2: &openapi3.Types{openapi3.TypeArray}, + }, + want: ConflictUnresolved, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := conflictSolver(tt.args.t1, tt.args.t2); got != tt.want { + t.Errorf("conflictSolver() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/speculator/pkg/apispec/constants.go b/speculator/pkg/apispec/constants.go new file mode 100644 index 0000000..9cc0e9e --- /dev/null +++ b/speculator/pkg/apispec/constants.go @@ -0,0 +1,15 @@ +package apispec + +const ( + contentTypeHeaderName = "content-type" + acceptTypeHeaderName = "accept" + authorizationTypeHeaderName = "authorization" + cookieTypeHeaderName = "cookie" +) + +const ( + mediaTypeApplicationJSON = "application/json" + mediaTypeApplicationHalJSON = "application/hal+json" + mediaTypeApplicationForm = "application/x-www-form-urlencoded" + mediaTypeMultipartFormData = "multipart/form-data" +) diff --git a/speculator/pkg/apispec/default.go b/speculator/pkg/apispec/default.go new file mode 100644 index 0000000..9311592 --- /dev/null +++ b/speculator/pkg/apispec/default.go @@ -0,0 +1,43 @@ +package apispec + +import ( + "github.com/getkin/kin-openapi/openapi3" + + "github.com/5gsec/sentryflow/speculator/pkg/pathtrie" +) + +func CreateDefaultSpec(host string, port string, config OperationGeneratorConfig) *Spec { + return &Spec{ + SpecInfo: SpecInfo{ + Host: host, + Port: port, + LearningSpec: &LearningSpec{ + PathItems: map[string]*openapi3.PathItem{}, + SecuritySchemes: openapi3.SecuritySchemes{}, + }, + ApprovedSpec: &ApprovedSpec{ + PathItems: map[string]*openapi3.PathItem{}, + SecuritySchemes: openapi3.SecuritySchemes{}, + }, + ApprovedPathTrie: pathtrie.New(), + ProvidedPathTrie: pathtrie.New(), + }, + OpGenerator: NewOperationGenerator(config), + } +} + +func createDefaultSwaggerInfo() *openapi3.Info { + return &openapi3.Info{ + Description: "This is a generated Open API Spec", + Title: "Swagger", + TermsOfService: "https://swagger.io/terms/", + Contact: &openapi3.Contact{ + Email: "apiteam@swagger.io", + }, + License: &openapi3.License{ + Name: "Apache 2.0", + URL: "https://www.apache.org/licenses/LICENSE-2.0.html", + }, + Version: "1.0.0", + } +} diff --git a/speculator/pkg/apispec/diff.go b/speculator/pkg/apispec/diff.go new file mode 100644 index 0000000..fbe1785 --- /dev/null +++ b/speculator/pkg/apispec/diff.go @@ -0,0 +1,288 @@ +package apispec + +import ( + "bytes" + "encoding/json" + "fmt" + "sort" + "strings" + + "github.com/getkin/kin-openapi/openapi3" + "github.com/gofrs/uuid" + log "github.com/sirupsen/logrus" +) + +type DiffType string + +const ( + DiffTypeNoDiff DiffType = "NO_DIFF" + DiffTypeZombieDiff DiffType = "ZOMBIE_DIFF" + DiffTypeShadowDiff DiffType = "SHADOW_DIFF" + DiffTypeGeneralDiff DiffType = "GENERAL_DIFF" +) + +type APIDiff struct { + Type DiffType + Path string + OriginalPathItem *openapi3.PathItem + ModifiedPathItem *openapi3.PathItem + InteractionID uuid.UUID + SpecID uuid.UUID +} + +type operationDiff struct { + OriginalOperation *openapi3.Operation + ModifiedOperation *openapi3.Operation +} + +type DiffParams struct { + operation *openapi3.Operation + method string + path string + requestID string + response *Response +} + +func (s *Spec) createDiffParamsFromTelemetry(telemetry *Telemetry) (*DiffParams, error) { + securitySchemes := openapi3.SecuritySchemes{} + + path, _ := GetPathAndQuery(telemetry.Request.Path) + telemetryOp, err := s.telemetryToOperation(telemetry, securitySchemes) + if err != nil { + return nil, fmt.Errorf("failed to convert telemetry to operation: %w", err) + } + return &DiffParams{ + operation: telemetryOp, + method: telemetry.Request.Method, + path: path, + requestID: telemetry.RequestID, + response: telemetry.Response, + }, nil +} + +func (s *Spec) DiffTelemetry(telemetry *Telemetry, specSource SpecSource) (*APIDiff, error) { + s.lock.Lock() + defer s.lock.Unlock() + + var apiDiff *APIDiff + var err error + diffParams, err := s.createDiffParamsFromTelemetry(telemetry) + if err != nil { + return nil, fmt.Errorf("failed to create diff params from telemetry. %w", err) + } + + switch specSource { + case SpecSourceProvided: + if !s.HasProvidedSpec() { + log.Infof("No provided spec to diff") + return nil, nil + } + apiDiff, err = s.diffProvidedSpec(diffParams) + if err != nil { + return nil, fmt.Errorf("failed to diff provided spec. %w", err) + } + case SpecSourceReconstructed: + if !s.HasApprovedSpec() { + log.Infof("No approved spec to diff") + return nil, nil + } + apiDiff, err = s.diffApprovedSpec(diffParams) + if err != nil { + return nil, fmt.Errorf("failed to diff approved spec. %w", err) + } + default: + return nil, fmt.Errorf("spec source: %v is not valid", specSource) + } + + return apiDiff, nil +} + +func (s *Spec) diffApprovedSpec(diffParams *DiffParams) (*APIDiff, error) { + var pathItem *openapi3.PathItem + pathFromTrie, _, found := s.ApprovedPathTrie.GetPathAndValue(diffParams.path) + if found { + diffParams.path = pathFromTrie // The diff will show the parametrized path if matched and not the telemetry path + pathItem = s.ApprovedSpec.GetPathItem(pathFromTrie) + } + return s.diffPathItem(pathItem, diffParams) +} + +func (s *Spec) diffProvidedSpec(diffParams *DiffParams) (*APIDiff, error) { + var pathItem *openapi3.PathItem + + basePath := s.ProvidedSpec.GetBasePath() + + pathNoBase := trimBasePathIfNeeded(basePath, diffParams.path) + + pathFromTrie, _, found := s.ProvidedPathTrie.GetPathAndValue(pathNoBase) + if found { + // The diff will show the parametrized path if matched and not the telemetry path + diffParams.path = addBasePathIfNeeded(basePath, pathFromTrie) + pathItem = s.ProvidedSpec.GetPathItem(pathFromTrie) + } + + return s.diffPathItem(pathItem, diffParams) +} + +// For path /api/foo/bar and base path of /api, the path that will be saved in paths map will be /foo/bar +// All paths must start with a slash. We can't trim a leading slash. +func trimBasePathIfNeeded(basePath, path string) string { + if hasBasePath(basePath) { + return strings.TrimPrefix(path, basePath) + } + + return path +} + +func addBasePathIfNeeded(basePath, path string) string { + if hasBasePath(basePath) { + return basePath + path + } + + return path +} + +func hasBasePath(basePath string) bool { + return basePath != "" && basePath != "/" +} + +func (s *Spec) diffPathItem(pathItem *openapi3.PathItem, diffParams *DiffParams) (*APIDiff, error) { + var apiDiff *APIDiff + method := diffParams.method + telemetryOp := diffParams.operation + path := diffParams.path + requestID := diffParams.requestID + reqUUID := uuid.NewV5(uuid.Nil, requestID) + + if pathItem == nil { + apiDiff = s.createAPIDiffEvent(DiffTypeShadowDiff, nil, createPathItemFromOperation(method, telemetryOp), + reqUUID, path) + return apiDiff, nil + } + + specOp := GetOperationFromPathItem(pathItem, method) + if specOp == nil { + // new operation + apiDiff := s.createAPIDiffEvent(DiffTypeShadowDiff, pathItem, CopyPathItemWithNewOperation(pathItem, method, telemetryOp), + reqUUID, path) + return apiDiff, nil + } + + diff, err := calculateOperationDiff(specOp, telemetryOp, diffParams.response) + if err != nil { + return nil, fmt.Errorf("failed to calculate operation diff: %w", err) + } + if diff != nil { + diffType := DiffTypeGeneralDiff + if specOp.Deprecated { + diffType = DiffTypeZombieDiff + } + apiDiff := s.createAPIDiffEvent(diffType, createPathItemFromOperation(method, diff.OriginalOperation), + createPathItemFromOperation(method, diff.ModifiedOperation), reqUUID, path) + return apiDiff, nil + } + + // no diff + return s.createAPIDiffEvent(DiffTypeNoDiff, nil, nil, reqUUID, path), nil +} + +func (s *Spec) createAPIDiffEvent(diffType DiffType, original, modified *openapi3.PathItem, interactionID uuid.UUID, path string) *APIDiff { + return &APIDiff{ + Type: diffType, + Path: path, + OriginalPathItem: original, + ModifiedPathItem: modified, + InteractionID: interactionID, + SpecID: s.ID, + } +} + +func createPathItemFromOperation(method string, operation *openapi3.Operation) *openapi3.PathItem { + pathItem := openapi3.PathItem{} + AddOperationToPathItem(&pathItem, method, operation) + return &pathItem +} + +func calculateOperationDiff(specOp, telemetryOp *openapi3.Operation, telemetryResponse *Response) (*operationDiff, error) { + clonedTelemetryOp, err := CloneOperation(telemetryOp) + if err != nil { + return nil, fmt.Errorf("failed to clone telemetry operation: %w", err) + } + + clonedSpecOp, err := CloneOperation(specOp) + if err != nil { + return nil, fmt.Errorf("failed to clone spec operation: %w", err) + } + + clonedTelemetryOp = sortParameters(clonedTelemetryOp) + clonedSpecOp = sortParameters(clonedSpecOp) + + // Keep only telemetry status code + clonedSpecOp = keepResponseStatusCode(clonedSpecOp, telemetryResponse.StatusCode) + + hasDiff, err := compareObjects(clonedSpecOp, clonedTelemetryOp) + if err != nil { + return nil, fmt.Errorf("failed to compare operations: %w", err) + } + + if hasDiff { + return &operationDiff{ + OriginalOperation: clonedSpecOp, + ModifiedOperation: clonedTelemetryOp, + }, nil + } + + // no diff + return nil, nil +} + +func compareObjects(obj1, obj2 any) (hasDiff bool, err error) { + obj1B, err := json.Marshal(obj1) + if err != nil { + return false, fmt.Errorf("failed to marshal obj1: %w", err) + } + + obj2B, err := json.Marshal(obj2) + if err != nil { + return false, fmt.Errorf("failed to marshal obj2: %w", err) + } + + return !bytes.Equal(obj1B, obj2B), nil +} + +// keepResponseStatusCode will remove all status codes from StatusCodeResponses map except the `statusCodeToKeep`. +func keepResponseStatusCode(op *openapi3.Operation, statusCodeToKeep string) *openapi3.Operation { + // keep only the provided status code + if op.Responses != nil { + filteredResponses := &openapi3.Responses{} + if responseRef := op.Responses.Value(statusCodeToKeep); responseRef != nil { + filteredResponses.Set(statusCodeToKeep, responseRef) + } + // keep default if exists + if responseRef := op.Responses.Value("default"); responseRef != nil { + filteredResponses.Set("default", responseRef) + } + + if filteredResponses.Len() == 0 { + op.Responses = nil + } else { + op.Responses = filteredResponses + } + } + + return op +} + +func sortParameters(operation *openapi3.Operation) *openapi3.Operation { + if operation == nil { + return operation + } + sort.Slice(operation.Parameters, func(i, j int) bool { + right := operation.Parameters[i].Value + left := operation.Parameters[j].Value + // Sibling parameters must have unique name + in values + return right.Name+right.In < left.Name+left.In + }) + + return operation +} diff --git a/speculator/pkg/apispec/diff_test.go b/speculator/pkg/apispec/diff_test.go new file mode 100644 index 0000000..b775600 --- /dev/null +++ b/speculator/pkg/apispec/diff_test.go @@ -0,0 +1,1140 @@ +package apispec + +import ( + "net/http" + "reflect" + "testing" + + "github.com/getkin/kin-openapi/openapi3" + "github.com/gofrs/uuid" + + "github.com/5gsec/sentryflow/speculator/pkg/pathtrie" +) + +var Data = &HTTPInteractionData{ + ReqBody: req1, + RespBody: res1, + ReqHeaders: map[string]string{ + contentTypeHeaderName: mediaTypeApplicationJSON, + }, + RespHeaders: map[string]string{ + contentTypeHeaderName: mediaTypeApplicationJSON, + }, + statusCode: 200, +} + +var DataWithAuth = &HTTPInteractionData{ + ReqBody: req1, + RespBody: res1, + ReqHeaders: map[string]string{ + contentTypeHeaderName: mediaTypeApplicationJSON, + authorizationTypeHeaderName: BearerAuthPrefix, + }, + RespHeaders: map[string]string{ + contentTypeHeaderName: mediaTypeApplicationJSON, + }, + statusCode: 200, +} + +var Data2 = &HTTPInteractionData{ + ReqBody: req2, + RespBody: res2, + ReqHeaders: map[string]string{ + contentTypeHeaderName: mediaTypeApplicationJSON, + }, + RespHeaders: map[string]string{ + contentTypeHeaderName: mediaTypeApplicationJSON, + }, + statusCode: 200, +} + +var DiffOAuthScopes = []string{"superadmin", "write:all_your_base"} + +func createTelemetry(reqID, method, path, host, statusCode string, reqBody, respBody string) *Telemetry { + return &Telemetry{ + RequestID: reqID, + Scheme: "", + Request: &Request{ + Method: method, + Path: path, + Host: host, + Common: &Common{ + Version: "", + Headers: []*Header{ + { + Key: contentTypeHeaderName, + Value: mediaTypeApplicationJSON, + }, + }, + Body: []byte(reqBody), + TruncatedBody: false, + }, + }, + Response: &Response{ + StatusCode: statusCode, + Common: &Common{ + Version: "", + Headers: []*Header{ + { + Key: contentTypeHeaderName, + Value: mediaTypeApplicationJSON, + }, + }, + Body: []byte(respBody), + TruncatedBody: false, + }, + }, + } +} + +func createTelemetryWithSecurity(reqID, method, path, host, statusCode string, reqBody, respBody string) *Telemetry { + bearerToken, _ := generateDefaultOAuthToken(DiffOAuthScopes) + + telemetry := createTelemetry(reqID, method, path, host, statusCode, reqBody, respBody) + telemetry.Request.Common.Headers = append(telemetry.Request.Common.Headers, &Header{ + Key: authorizationTypeHeaderName, + Value: BearerAuthPrefix + bearerToken, + }) + return telemetry +} + +func TestSpec_DiffTelemetry_Reconstructed(t *testing.T) { + reqID := "req-id" + reqUUID := uuid.NewV5(uuid.Nil, reqID) + specUUID := uuid.NewV5(uuid.Nil, "openapi3-id") + bearerToken, _ := generateDefaultOAuthToken(DiffOAuthScopes) + DataWithAuth.ReqHeaders[authorizationTypeHeaderName] = BearerAuthPrefix + bearerToken + type fields struct { + ID uuid.UUID + ApprovedSpec *ApprovedSpec + LearningSpec *LearningSpec + ApprovedPathTrie pathtrie.PathTrie + } + type args struct { + telemetry *Telemetry + } + tests := []struct { + name string + fields fields + args args + want *APIDiff + wantErr bool + }{ + { + name: "No diff", + fields: fields{ + ID: specUUID, + ApprovedSpec: &ApprovedSpec{ + PathItems: map[string]*openapi3.PathItem{ + "/api": &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + }, + }, + ApprovedPathTrie: createPathTrie(map[string]string{ + "/api": "1", + }), + }, + args: args{ + telemetry: createTelemetry(reqID, http.MethodGet, "/api", "host", "200", Data.ReqBody, Data.RespBody), + }, + want: &APIDiff{ + Type: DiffTypeNoDiff, + Path: "/api", + OriginalPathItem: nil, + ModifiedPathItem: nil, + InteractionID: reqUUID, + SpecID: specUUID, + }, + wantErr: false, + }, + { + name: "Security diff", + fields: fields{ + ID: specUUID, + ApprovedSpec: &ApprovedSpec{ + PathItems: map[string]*openapi3.PathItem{ + "/api": &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + }, + }, + ApprovedPathTrie: createPathTrie(map[string]string{ + "/api": "1", + }), + }, + args: args{ + telemetry: createTelemetryWithSecurity(reqID, http.MethodGet, "/api", "host", "200", Data.ReqBody, Data.RespBody), + }, + want: &APIDiff{ + Type: DiffTypeGeneralDiff, + Path: "/api", + OriginalPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + ModifiedPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, DataWithAuth).Op).PathItem, + InteractionID: reqUUID, + SpecID: specUUID, + }, + wantErr: false, + }, + { + name: "New PathItem", + fields: fields{ + ID: specUUID, + ApprovedSpec: &ApprovedSpec{ + PathItems: map[string]*openapi3.PathItem{ + "/api": &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + }, + }, + ApprovedPathTrie: createPathTrie(map[string]string{ + "/api": "1", + }), + }, + args: args{ + telemetry: createTelemetry(reqID, http.MethodGet, "/api/new", "host", "200", Data.ReqBody, Data.RespBody), + }, + want: &APIDiff{ + Type: DiffTypeShadowDiff, + Path: "/api/new", + OriginalPathItem: nil, + ModifiedPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + InteractionID: reqUUID, + SpecID: specUUID, + }, + wantErr: false, + }, + { + name: "New Operation", + fields: fields{ + ID: specUUID, + ApprovedSpec: &ApprovedSpec{ + PathItems: map[string]*openapi3.PathItem{ + "/api": &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + }, + }, + ApprovedPathTrie: createPathTrie(map[string]string{ + "/api": "1", + }), + }, + args: args{ + telemetry: createTelemetry(reqID, http.MethodPost, "/api", "host", "200", req2, res2), + }, + want: &APIDiff{ + Type: DiffTypeShadowDiff, + Path: "/api", + OriginalPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + ModifiedPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).WithOperation(http.MethodPost, NewOperation(t, Data2).Op).PathItem, + InteractionID: reqUUID, + SpecID: specUUID, + }, + wantErr: false, + }, + { + name: "Changed Operation", + fields: fields{ + ID: specUUID, + ApprovedSpec: &ApprovedSpec{ + PathItems: map[string]*openapi3.PathItem{ + "/api": &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + }, + }, + ApprovedPathTrie: createPathTrie(map[string]string{ + "/api": "1", + }), + }, + args: args{ + telemetry: createTelemetry(reqID, http.MethodGet, "/api", "host", "200", req2, res2), + }, + want: &APIDiff{ + Type: DiffTypeGeneralDiff, + Path: "/api", + OriginalPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + ModifiedPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data2).Op).PathItem, + InteractionID: reqUUID, + SpecID: specUUID, + }, + wantErr: false, + }, + { + name: "Parameterized path", + fields: fields{ + ID: specUUID, + ApprovedSpec: &ApprovedSpec{ + PathItems: map[string]*openapi3.PathItem{ + "/api/{my-param}": &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + }, + }, + ApprovedPathTrie: createPathTrie(map[string]string{ + "/api/{my-param}": "1", + }), + }, + args: args{ + telemetry: createTelemetry(reqID, http.MethodGet, "/api/2", "host", "200", req2, res2), + }, + want: &APIDiff{ + Type: DiffTypeGeneralDiff, + Path: "/api/{my-param}", + OriginalPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + ModifiedPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data2).Op).PathItem, + InteractionID: reqUUID, + SpecID: specUUID, + }, + wantErr: false, + }, + { + name: "Parameterized path but also exact path", + fields: fields{ + ID: specUUID, + ApprovedSpec: &ApprovedSpec{ + PathItems: map[string]*openapi3.PathItem{ + "/api/1": &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + }, + }, + ApprovedPathTrie: createPathTrie(map[string]string{ + "/api/{my-param}": "1", + "/api/1": "2", + }), + }, + args: args{ + telemetry: createTelemetry(reqID, http.MethodGet, "/api/1", "host", "200", req2, res2), + }, + want: &APIDiff{ + Type: DiffTypeGeneralDiff, + Path: "/api/1", + OriginalPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + ModifiedPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data2).Op).PathItem, + InteractionID: reqUUID, + SpecID: specUUID, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &Spec{ + SpecInfo: SpecInfo{ + ID: tt.fields.ID, + ApprovedSpec: tt.fields.ApprovedSpec, + LearningSpec: tt.fields.LearningSpec, + ApprovedPathTrie: tt.fields.ApprovedPathTrie, + }, + OpGenerator: CreateTestNewOperationGenerator(), + } + + got, err := s.DiffTelemetry(tt.args.telemetry, SpecSourceReconstructed) + if (err != nil) != tt.wantErr { + t.Errorf("DiffTelemetry() error = %v, wantErr %v", err, tt.wantErr) + return + } + + assertEqual(t, got, tt.want) + }) + } +} + +func TestSpec_DiffTelemetry_Provided(t *testing.T) { + reqID := "req-id" + reqUUID := uuid.NewV5(uuid.Nil, reqID) + specUUID := uuid.NewV5(uuid.Nil, "openapi3-id") + bearerToken, _ := generateDefaultOAuthToken(DiffOAuthScopes) + DataWithAuth.ReqHeaders[authorizationTypeHeaderName] = BearerAuthPrefix + bearerToken + type fields struct { + ID uuid.UUID + ProvidedSpec *ProvidedSpec + ProvidedPathTrie pathtrie.PathTrie + } + type args struct { + telemetry *Telemetry + } + tests := []struct { + name string + fields fields + args args + want *APIDiff + wantErr bool + }{ + { + name: "No diff", + fields: fields{ + ID: specUUID, + ProvidedSpec: &ProvidedSpec{ + Doc: &openapi3.T{ + Paths: createPath("/api", &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem), + }, + }, + ProvidedPathTrie: createPathTrie(map[string]string{ + "/api": "1", + }), + }, + args: args{ + telemetry: createTelemetry(reqID, http.MethodGet, "/api", "host", "200", Data.ReqBody, Data.RespBody), + }, + want: &APIDiff{ + Type: DiffTypeNoDiff, + Path: "/api", + OriginalPathItem: nil, + ModifiedPathItem: nil, + InteractionID: reqUUID, + SpecID: specUUID, + }, + wantErr: false, + }, + { + name: "Security diff", + fields: fields{ + ID: specUUID, + ProvidedSpec: &ProvidedSpec{ + Doc: &openapi3.T{ + Paths: createPath("/api", &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem), + }, + }, + ProvidedPathTrie: createPathTrie(map[string]string{ + "/api": "1", + }), + }, + args: args{ + telemetry: createTelemetryWithSecurity(reqID, http.MethodGet, "/api", "host", "200", Data.ReqBody, Data.RespBody), + }, + want: &APIDiff{ + Type: DiffTypeGeneralDiff, + Path: "/api", + OriginalPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + ModifiedPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, DataWithAuth).Op).PathItem, + InteractionID: reqUUID, + SpecID: specUUID, + }, + wantErr: false, + }, + { + name: "New PathItem", + fields: fields{ + ID: specUUID, + ProvidedSpec: &ProvidedSpec{ + Doc: &openapi3.T{ + Paths: createPath("/api", &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem), + }, + }, + ProvidedPathTrie: createPathTrie(map[string]string{ + "/api": "1", + }), + }, + args: args{ + telemetry: createTelemetry(reqID, http.MethodGet, "/api/new", "host", "200", Data.ReqBody, Data.RespBody), + }, + want: &APIDiff{ + Type: DiffTypeShadowDiff, + Path: "/api/new", + OriginalPathItem: nil, + ModifiedPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + InteractionID: reqUUID, + SpecID: specUUID, + }, + wantErr: false, + }, + { + name: "New Operation", + fields: fields{ + ID: specUUID, + ProvidedSpec: &ProvidedSpec{ + Doc: &openapi3.T{ + Paths: createPath("/api", &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem), + }, + }, + ProvidedPathTrie: createPathTrie(map[string]string{ + "/api": "1", + }), + }, + args: args{ + telemetry: createTelemetry(reqID, http.MethodPost, "/api", "host", "200", req2, res2), + }, + want: &APIDiff{ + Type: DiffTypeShadowDiff, + Path: "/api", + OriginalPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + ModifiedPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).WithOperation(http.MethodPost, NewOperation(t, Data2).Op).PathItem, + InteractionID: reqUUID, + SpecID: specUUID, + }, + wantErr: false, + }, + { + name: "Changed Operation", + fields: fields{ + ID: specUUID, + ProvidedSpec: &ProvidedSpec{ + Doc: &openapi3.T{ + Paths: createPath("/api", &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem), + }, + }, + ProvidedPathTrie: createPathTrie(map[string]string{ + "/api": "1", + }), + }, + args: args{ + telemetry: createTelemetry(reqID, http.MethodGet, "/api", "host", "200", req2, res2), + }, + want: &APIDiff{ + Type: DiffTypeGeneralDiff, + Path: "/api", + OriginalPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + ModifiedPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data2).Op).PathItem, + InteractionID: reqUUID, + SpecID: specUUID, + }, + wantErr: false, + }, + { + name: "test remove base path + parametrized path", + fields: fields{ + ID: specUUID, + ProvidedSpec: &ProvidedSpec{ + Doc: &openapi3.T{ + Servers: openapi3.Servers{ + { + URL: "https://example.com/api", + }, + }, + Paths: createPath("/foo/{param}", &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem), + }, + }, + ProvidedPathTrie: createPathTrie(map[string]string{ + "/foo/{param}": "1", + }), + }, + args: args{ + telemetry: createTelemetry(reqID, http.MethodGet, "/api/foo/bar", "host", "200", req2, res2), + }, + want: &APIDiff{ + Type: DiffTypeGeneralDiff, + Path: "/api/foo/{param}", + OriginalPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + ModifiedPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data2).Op).PathItem, + InteractionID: reqUUID, + SpecID: specUUID, + }, + wantErr: false, + }, + { + name: "test base path = / (default)", + fields: fields{ + ID: specUUID, + ProvidedSpec: &ProvidedSpec{ + Doc: &openapi3.T{ + Servers: openapi3.Servers{ + { + URL: "https://example.com/", + }, + }, + Paths: createPath("/foo/bar", &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem), + }, + }, + ProvidedPathTrie: createPathTrie(map[string]string{ + "/foo/bar": "1", + }), + }, + args: args{ + telemetry: createTelemetry(reqID, http.MethodGet, "/foo/bar", "host", "200", req2, res2), + }, + want: &APIDiff{ + Type: DiffTypeGeneralDiff, + Path: "/foo/bar", + OriginalPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + ModifiedPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data2).Op).PathItem, + InteractionID: reqUUID, + SpecID: specUUID, + }, + wantErr: false, + }, + { + name: "Parameterized path", + fields: fields{ + ID: specUUID, + ProvidedSpec: &ProvidedSpec{ + Doc: &openapi3.T{ + Servers: openapi3.Servers{ + { + URL: "https://example.com/", + }, + }, + Paths: createPath("/api/{my-param}", &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem), + }, + }, + ProvidedPathTrie: createPathTrie(map[string]string{ + "/api/{my-param}": "1", + }), + }, + args: args{ + telemetry: createTelemetry(reqID, http.MethodGet, "/api/2", "host", "200", req2, res2), + }, + want: &APIDiff{ + Type: DiffTypeGeneralDiff, + Path: "/api/{my-param}", + OriginalPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + ModifiedPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data2).Op).PathItem, + InteractionID: reqUUID, + SpecID: specUUID, + }, + wantErr: false, + }, + { + name: "Parameterized path but also exact path", + fields: fields{ + ID: specUUID, + ProvidedSpec: &ProvidedSpec{ + Doc: &openapi3.T{ + Servers: openapi3.Servers{ + { + URL: "https://example.com/", + }, + }, + Paths: createPath("/api/1", &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem), + }, + }, + ProvidedPathTrie: createPathTrie(map[string]string{ + "/api/{my-param}": "1", + "/api/1": "2", + }), + }, + args: args{ + telemetry: createTelemetry(reqID, http.MethodGet, "/api/1", "host", "200", req2, res2), + }, + want: &APIDiff{ + Type: DiffTypeGeneralDiff, + Path: "/api/1", + OriginalPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + ModifiedPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data2).Op).PathItem, + InteractionID: reqUUID, + SpecID: specUUID, + }, + wantErr: false, + }, + { + name: "Deprecated API expected Zombie API diff", + fields: fields{ + ID: specUUID, + ProvidedSpec: &ProvidedSpec{ + Doc: &openapi3.T{ + Paths: createPath("/api", &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Deprecated().Op).PathItem), + }, + }, + ProvidedPathTrie: createPathTrie(map[string]string{ + "/api": "1", + }), + }, + args: args{ + telemetry: createTelemetry(reqID, http.MethodGet, "/api", "host", "200", Data.ReqBody, Data.RespBody), + }, + want: &APIDiff{ + Type: DiffTypeZombieDiff, + Path: "/api", + + OriginalPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Deprecated().Op).PathItem, + ModifiedPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + InteractionID: reqUUID, + SpecID: specUUID, + }, + wantErr: false, + }, + { + name: "Deprecated and simple diff expected Zombie API diff", + fields: fields{ + ID: specUUID, + ProvidedSpec: &ProvidedSpec{ + Doc: &openapi3.T{ + Paths: createPath("/api", &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Deprecated().Op).PathItem), + }, + }, + ProvidedPathTrie: createPathTrie(map[string]string{ + "/api": "1", + }), + }, + args: args{ + telemetry: createTelemetry(reqID, http.MethodGet, "/api", "host", "200", req2, res2), + }, + want: &APIDiff{ + Type: DiffTypeZombieDiff, + Path: "/api", + OriginalPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Deprecated().Op).PathItem, + ModifiedPathItem: &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data2).Op).PathItem, + InteractionID: reqUUID, + SpecID: specUUID, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &Spec{ + SpecInfo: SpecInfo{ + ID: tt.fields.ID, + ProvidedSpec: tt.fields.ProvidedSpec, + ProvidedPathTrie: tt.fields.ProvidedPathTrie, + }, + OpGenerator: CreateTestNewOperationGenerator(), + } + got, err := s.DiffTelemetry(tt.args.telemetry, SpecSourceProvided) + if (err != nil) != tt.wantErr { + t.Errorf("DiffTelemetry() error = %v, wantErr %v", err, tt.wantErr) + return + } + assertEqual(t, got, tt.want) + }) + } +} + +func createPath(path string, pathItems *openapi3.PathItem) *openapi3.Paths { + paths := &openapi3.Paths{} + paths.Set(path, pathItems) + return paths +} + +func createPathTrie(pathToValue map[string]string) pathtrie.PathTrie { + pt := pathtrie.New() + for path, value := range pathToValue { + pt.Insert(path, value) + } + return pt +} + +func Test_keepResponseStatusCode(t *testing.T) { + type args struct { + op *openapi3.Operation + statusCodeToKeep string + } + tests := []struct { + name string + args args + want *openapi3.Operation + }{ + { + name: "keep 1 remove 1", + args: args{ + op: createTestOperation(). + WithResponse(200, openapi3.NewResponse().WithDescription("keep")). + WithResponse(300, openapi3.NewResponse().WithDescription("delete")).Op, + statusCodeToKeep: "200", + }, + want: createTestOperation().WithResponse(200, openapi3.NewResponse().WithDescription("keep")).Op, + }, + { + name: "status code to keep not found - remove all", + args: args{ + op: createTestOperation(). + WithResponse(202, openapi3.NewResponse().WithDescription("delete")). + WithResponse(300, openapi3.NewResponse().WithDescription("delete")).Op, + statusCodeToKeep: "200", + }, + want: openapi3.NewOperation(), + }, + { + name: "status code to keep not found - remove all keep default response", + args: args{ + op: createTestOperation(). + WithResponse(202, openapi3.NewResponse().WithDescription("delete")). + WithResponse(300, openapi3.NewResponse().WithDescription("delete")). + WithResponse(0, openapi3.NewResponse().WithDescription("keep-default")).Op, + statusCodeToKeep: "200", + }, + want: createTestOperation(). + WithResponse(0, openapi3.NewResponse().WithDescription("keep-default")).Op, + }, + { + name: "only status code to keep is found", + args: args{ + op: createTestOperation(). + WithResponse(200, openapi3.NewResponse().WithDescription("keep")).Op, + statusCodeToKeep: "200", + }, + want: createTestOperation(). + WithResponse(200, openapi3.NewResponse().WithDescription("keep")).Op, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := keepResponseStatusCode(tt.args.op, tt.args.statusCodeToKeep) + assertEqual(t, got, tt.want) + }) + } +} + +func Test_calculateOperationDiff(t *testing.T) { + type args struct { + specOp *openapi3.Operation + telemetryOp *openapi3.Operation + telemetryResponse *Response + } + tests := []struct { + name string + args args + want *operationDiff + wantErr bool + }{ + { + name: "no diff", + args: args{ + specOp: createTestOperation(). + WithResponse(200, openapi3.NewResponse().WithDescription("test")). + WithParameter(openapi3.NewHeaderParameter("header")).Op, + telemetryOp: createTestOperation(). + WithResponse(200, openapi3.NewResponse().WithDescription("test")). + WithParameter(openapi3.NewHeaderParameter("header")).Op, + telemetryResponse: &Response{ + StatusCode: "200", + }, + }, + want: nil, + wantErr: false, + }, + { + name: "no diff - parameters are not sorted", + args: args{ + specOp: createTestOperation(). + WithResponse(200, openapi3.NewResponse().WithDescription("test")). + WithParameter(openapi3.NewHeaderParameter("header2")). + WithParameter(openapi3.NewHeaderParameter("header1")).Op, + telemetryOp: createTestOperation(). + WithResponse(200, openapi3.NewResponse().WithDescription("test")). + WithParameter(openapi3.NewHeaderParameter("header1")). + WithParameter(openapi3.NewHeaderParameter("header2")).Op, + telemetryResponse: &Response{ + StatusCode: "200", + }, + }, + want: nil, + wantErr: false, + }, + { + name: "no diff - existing response should be removed", + args: args{ + specOp: createTestOperation(). + WithResponse(200, openapi3.NewResponse().WithDescription("test")). + WithResponse(300, openapi3.NewResponse().WithDescription("remove")). + WithParameter(openapi3.NewHeaderParameter("header")).Op, + telemetryOp: createTestOperation(). + WithResponse(200, openapi3.NewResponse().WithDescription("test")). + WithParameter(openapi3.NewHeaderParameter("header")).Op, + telemetryResponse: &Response{ + StatusCode: "200", + }, + }, + want: nil, + wantErr: false, + }, + { + name: "no diff", + args: args{ + specOp: createTestOperation(). + WithResponse(200, openapi3.NewResponse().WithDescription("test")). + WithResponse(403, openapi3.NewResponse().WithDescription("keep")). + WithParameter(openapi3.NewHeaderParameter("header")).Op, + telemetryOp: createTestOperation(). + WithResponse(403, openapi3.NewResponse().WithDescription("keep")). + WithParameter(openapi3.NewHeaderParameter("header")).Op, + telemetryResponse: &Response{ + StatusCode: "403", + }, + }, + want: nil, + wantErr: false, + }, + { + name: "has diff", + args: args{ + specOp: createTestOperation(). + WithResponse(200, openapi3.NewResponse().WithDescription("test")). + WithParameter(openapi3.NewHeaderParameter("header")).Op, + telemetryOp: createTestOperation(). + WithResponse(200, openapi3.NewResponse().WithDescription("test")). + WithParameter(openapi3.NewHeaderParameter("new-header")).Op, + telemetryResponse: &Response{ + StatusCode: "200", + }, + }, + want: &operationDiff{ + OriginalOperation: createTestOperation(). + WithResponse(200, openapi3.NewResponse().WithDescription("test")). + WithParameter(openapi3.NewHeaderParameter("header")).Op, + ModifiedOperation: createTestOperation(). + WithResponse(200, openapi3.NewResponse().WithDescription("test")). + WithParameter(openapi3.NewHeaderParameter("new-header")).Op, + }, + wantErr: false, + }, + { + name: "has diff in param and not in response", + args: args{ + specOp: createTestOperation(). + WithResponse(200, openapi3.NewResponse().WithDescription("200")). + WithResponse(403, openapi3.NewResponse().WithDescription("403")). + WithParameter(openapi3.NewHeaderParameter("header")).Op, + telemetryOp: createTestOperation(). + WithResponse(200, openapi3.NewResponse().WithDescription("200")). + WithParameter(openapi3.NewHeaderParameter("new-header")).Op, + telemetryResponse: &Response{ + StatusCode: "200", + }, + }, + want: &operationDiff{ + OriginalOperation: createTestOperation(). + WithResponse(200, openapi3.NewResponse().WithDescription("200")). + WithParameter(openapi3.NewHeaderParameter("header")).Op, + ModifiedOperation: createTestOperation(). + WithResponse(200, openapi3.NewResponse().WithDescription("200")). + WithParameter(openapi3.NewHeaderParameter("new-header")).Op, + }, + wantErr: false, + }, + { + name: "has diff in response", + args: args{ + specOp: createTestOperation(). + WithResponse(200, openapi3.NewResponse().WithDescription("200")). + WithResponse(403, openapi3.NewResponse().WithDescription("403")). + WithParameter(openapi3.NewHeaderParameter("header")).Op, + telemetryOp: createTestOperation(). + WithResponse(200, openapi3.NewResponse().WithDescription("new-200")). + WithParameter(openapi3.NewHeaderParameter("new-header")).Op, + telemetryResponse: &Response{ + StatusCode: "200", + }, + }, + want: &operationDiff{ + OriginalOperation: createTestOperation(). + WithResponse(200, openapi3.NewResponse().WithDescription("200")). + WithParameter(openapi3.NewHeaderParameter("header")).Op, + ModifiedOperation: createTestOperation(). + WithResponse(200, openapi3.NewResponse().WithDescription("new-200")). + WithParameter(openapi3.NewHeaderParameter("new-header")).Op, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := calculateOperationDiff(tt.args.specOp, tt.args.telemetryOp, tt.args.telemetryResponse) + if (err != nil) != tt.wantErr { + t.Errorf("calculateOperationDiff() error = %v, wantErr %v", err, tt.wantErr) + return + } + assertEqual(t, got, tt.want) + }) + } +} + +func Test_compareObjects(t *testing.T) { + type args struct { + obj1 any + obj2 any + } + tests := []struct { + name string + args args + wantHasDiff bool + wantErr bool + }{ + { + name: "no diff", + args: args{ + obj1: createTestOperation().WithParameter(openapi3.NewHeaderParameter("test")).Op, + obj2: createTestOperation().WithParameter(openapi3.NewHeaderParameter("test")).Op, + }, + wantHasDiff: false, + wantErr: false, + }, + { + name: "has diff (compare only Responses)", + args: args{ + obj1: createTestOperation(). + WithResponse(200, openapi3.NewResponse().WithDescription("test")).Op.Responses, + obj2: createTestOperation(). + WithResponse(200, openapi3.NewResponse().WithDescription("diff")).Op.Responses, + }, + wantHasDiff: true, + wantErr: false, + }, + { + name: "has diff (different objects - Operation vs Responses)", + args: args{ + obj1: createTestOperation(). + WithResponse(200, openapi3.NewResponse().WithDescription("test")).Op, + obj2: createTestOperation(). + WithResponse(200, openapi3.NewResponse().WithDescription("diff")).Op.Responses, + }, + wantHasDiff: true, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotHasDiff, err := compareObjects(tt.args.obj1, tt.args.obj2) + if (err != nil) != tt.wantErr { + t.Errorf("compareObjects() error = %v, wantErr %v", err, tt.wantErr) + return + } + if gotHasDiff != tt.wantHasDiff { + t.Errorf("compareObjects() gotHasDiff = %v, want %v", gotHasDiff, tt.wantHasDiff) + } + }) + } +} + +func Test_sortParameters(t *testing.T) { + type args struct { + operation *openapi3.Operation + } + tests := []struct { + name string + args args + want *openapi3.Operation + }{ + { + name: "already sorted", + args: args{ + operation: createTestOperation(). + WithParameter(openapi3.NewHeaderParameter("1")). + WithParameter(openapi3.NewHeaderParameter("2")).Op, + }, + want: createTestOperation(). + WithParameter(openapi3.NewHeaderParameter("1")). + WithParameter(openapi3.NewHeaderParameter("2")).Op, + }, + { + name: "sort is needed - sort by 'name'", + args: args{ + operation: createTestOperation(). + WithParameter(openapi3.NewHeaderParameter("3")). + WithParameter(openapi3.NewHeaderParameter("1")). + WithParameter(openapi3.NewHeaderParameter("2")).Op, + }, + want: createTestOperation(). + WithParameter(openapi3.NewHeaderParameter("1")). + WithParameter(openapi3.NewHeaderParameter("2")). + WithParameter(openapi3.NewHeaderParameter("3")).Op, + }, + { + name: "param name is the same - sort by 'in'", + args: args{ + operation: createTestOperation(). + WithParameter(openapi3.NewHeaderParameter("1")). + WithParameter(openapi3.NewCookieParameter("1")).Op, + }, + want: createTestOperation(). + WithParameter(openapi3.NewCookieParameter("1")). + WithParameter(openapi3.NewHeaderParameter("1")).Op, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := sortParameters(tt.args.operation); !reflect.DeepEqual(got, tt.want) { + t.Errorf("sortParameters() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_hasBasePath(t *testing.T) { + type args struct { + basePath string + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "empty base path", + args: args{ + basePath: "", + }, + want: false, + }, + { + name: "slash base path", + args: args{ + basePath: "/", + }, + want: false, + }, + { + name: "base path exist", + args: args{ + basePath: "/api", + }, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := hasBasePath(tt.args.basePath); got != tt.want { + t.Errorf("hasBasePath() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_addBasePathIfNeeded(t *testing.T) { + type args struct { + basePath string + path string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "no need to add base path", + args: args{ + basePath: "", + path: "/no-need", + }, + want: "/no-need", + }, + { + name: "need to add base path", + args: args{ + basePath: "/api", + path: "/need", + }, + want: "/api/need", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := addBasePathIfNeeded(tt.args.basePath, tt.args.path); got != tt.want { + t.Errorf("addBasePathIfNeeded() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_trimBasePathIfNeeded(t *testing.T) { + type args struct { + basePath string + path string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "no need to trim base path", + args: args{ + basePath: "", + path: "/no-need", + }, + want: "/no-need", + }, + { + name: "need to trim base path", + args: args{ + basePath: "/api", + path: "/api/need", + }, + want: "/need", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := trimBasePathIfNeeded(tt.args.basePath, tt.args.path); got != tt.want { + t.Errorf("trimBasePathIfNeeded() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/speculator/pkg/apispec/form.go b/speculator/pkg/apispec/form.go new file mode 100644 index 0000000..87dbeae --- /dev/null +++ b/speculator/pkg/apispec/form.go @@ -0,0 +1,78 @@ +package apispec + +import ( + "fmt" + "mime/multipart" + "net/url" + "strings" + + "github.com/getkin/kin-openapi/openapi3" +) + +const ( + // taken from net/http/request.go. + defaultMaxMemory = 32 << 20 // 32 MB +) + +func handleApplicationFormURLEncodedBody(operation *openapi3.Operation, securitySchemes openapi3.SecuritySchemes, body string) (*openapi3.Operation, openapi3.SecuritySchemes, error) { + parseQuery, err := url.ParseQuery(body) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse query. body=%v: %v", body, err) + } + + objSchema := openapi3.NewObjectSchema() + + for key, values := range parseQuery { + if key == AccessTokenParamKey { + // https://datatracker.ietf.org/doc/html/rfc6750#section-2.2 + operation, securitySchemes = handleAuthQueryParam(operation, securitySchemes, values) + } else { + objSchema.WithProperty(key, getSchemaFromQueryValues(values)) + } + } + + if len(objSchema.Properties) != 0 { + operationSetRequestBody(operation, openapi3.NewRequestBody().WithContent(openapi3.NewContentWithSchema(objSchema, []string{mediaTypeApplicationForm}))) + // TODO: handle encoding + // https://swagger.io/docs/specification/describing-request-body/ + // operation.RequestBody.Value.GetMediaType(mediaTypeApplicationForm).Encoding + } + + return operation, securitySchemes, nil +} + +func getMultipartFormDataSchema(body string, mediaTypeParams map[string]string) (*openapi3.Schema, error) { + boundary, ok := mediaTypeParams["boundary"] + if !ok { + return nil, fmt.Errorf("no multipart boundary param in Content-Type") + } + + form, err := multipart.NewReader(strings.NewReader(body), boundary).ReadForm(defaultMaxMemory) + if err != nil { + return nil, fmt.Errorf("failed to read form: %w", err) + } + + schema := openapi3.NewObjectSchema() + + // https://swagger.io/docs/specification/describing-request-body/file-upload/ + for key, fileHeaders := range form.File { + fileSchema := openapi3.NewStringSchema().WithFormat("binary") + switch len(fileHeaders) { + case 0: + // do nothing + case 1: + // single file + schema.WithProperty(key, fileSchema) + default: + // array of files + schema.WithProperty(key, openapi3.NewArraySchema().WithItems(fileSchema)) + } + } + + // add values formData + for key, values := range form.Value { + schema.WithProperty(key, getSchemaFromValues(values, false, "")) + } + + return schema, nil +} diff --git a/speculator/pkg/apispec/form_test.go b/speculator/pkg/apispec/form_test.go new file mode 100644 index 0000000..7bffdf9 --- /dev/null +++ b/speculator/pkg/apispec/form_test.go @@ -0,0 +1,215 @@ +package apispec + +import ( + "testing" + + "github.com/getkin/kin-openapi/openapi3" +) + +func newBoolSchemaWithAllowEmptyValue() *openapi3.Schema { + schema := openapi3.NewBoolSchema() + schema.AllowEmptyValue = true + return schema +} + +func Test_handleApplicationFormURLEncodedBody(t *testing.T) { + type args struct { + operation *openapi3.Operation + securitySchemes openapi3.SecuritySchemes + body string + } + tests := []struct { + name string + args args + want *openapi3.Operation + want1 openapi3.SecuritySchemes + wantErr bool + }{ + { + name: "sanity", + args: args{ + operation: openapi3.NewOperation(), + body: "name=Amy&fav_number=321.1", + }, + want: createTestOperation().WithRequestBody(openapi3.NewRequestBody().WithSchema( + openapi3.NewObjectSchema().WithProperties(map[string]*openapi3.Schema{ + "name": openapi3.NewStringSchema(), + "fav_number": openapi3.NewFloat64Schema(), + }), []string{mediaTypeApplicationForm})).Op, + }, + { + name: "parameters without a value", + args: args{ + operation: openapi3.NewOperation(), + body: "foo&bar&baz", + }, + want: createTestOperation().WithRequestBody(openapi3.NewRequestBody().WithSchema( + openapi3.NewObjectSchema().WithProperties(map[string]*openapi3.Schema{ + "foo": newBoolSchemaWithAllowEmptyValue(), + "bar": newBoolSchemaWithAllowEmptyValue(), + "baz": newBoolSchemaWithAllowEmptyValue(), + }), []string{mediaTypeApplicationForm})).Op, + }, + { + name: "multiple parameter instances", + args: args{ + operation: openapi3.NewOperation(), + body: "param=value1¶m=value2¶m=value3", + }, + want: createTestOperation().WithRequestBody(openapi3.NewRequestBody().WithSchema( + openapi3.NewObjectSchema().WithProperties(map[string]*openapi3.Schema{ + "param": openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema()), + }), []string{mediaTypeApplicationForm})).Op, + }, + { + name: "bad query", + args: args{ + operation: openapi3.NewOperation(), + body: "name%2", + }, + want: nil, + wantErr: true, + }, + { + name: "OAuth2 security", + args: args{ + operation: openapi3.NewOperation(), + body: AccessTokenParamKey + "=token", + securitySchemes: openapi3.SecuritySchemes{}, + }, + want: createTestOperation(). + WithSecurityRequirement(map[string][]string{OAuth2SecuritySchemeKey: {}}).Op, + want1: openapi3.SecuritySchemes{ + OAuth2SecuritySchemeKey: {Value: NewOAuth2SecurityScheme([]string{})}, + }, + }, + { + name: "OAuth2 security + some params", + args: args{ + operation: openapi3.NewOperation(), + body: AccessTokenParamKey + "=token&name=Amy", + securitySchemes: openapi3.SecuritySchemes{}, + }, + want: createTestOperation(). + WithSecurityRequirement(map[string][]string{OAuth2SecuritySchemeKey: {}}). + WithRequestBody(openapi3.NewRequestBody().WithSchema( + openapi3.NewObjectSchema().WithProperties(map[string]*openapi3.Schema{ + "name": openapi3.NewStringSchema(), + }), []string{mediaTypeApplicationForm})).Op, + want1: openapi3.SecuritySchemes{ + OAuth2SecuritySchemeKey: {Value: NewOAuth2SecurityScheme([]string{})}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + op, securitySchemes, err := handleApplicationFormURLEncodedBody(tt.args.operation, tt.args.securitySchemes, tt.args.body) + if (err != nil) != tt.wantErr { + t.Errorf("handleApplicationFormURLEncodedBody() error = %v, wantErr %v", err, tt.wantErr) + return + } + op = sortParameters(op) + tt.want = sortParameters(tt.want) + + assertEqual(t, op, tt.want) + assertEqual(t, securitySchemes, tt.want1) + }) + } +} + +var formDataBodyMultipleFileUpload = "--cdce6441022a3dcf\r\n" + + "Content-Disposition: form-data; name=\"fileName\"; filename=\"file1.txt\"\r\n\r\n" + + "Content-Type: text/plain\r\n\r\n" + + "File contents go here.\r\n" + + "--cdce6441022a3dcf\r\n" + + "Content-Disposition: form-data; name=\"fileName\"; filename=\"file2.png\"\r\n\r\n" + + "Content-Type: image/png\r\n\r\n" + + "File contents go here.\r\n" + + "--cdce6441022a3dcf\r\n" + + "Content-Disposition: form-data; name=\"fileName\"; filename=\"file3.jpg\"\r\n\r\n" + + "Content-Type: image/jpeg\r\n\r\n" + + "File contents go here.\r\n" + + "--cdce6441022a3dcf--\r\n" + +var formDataBody = "--cdce6441022a3dcf\r\n" + + "Content-Disposition: form-data; name=\"upfile\"; filename=\"example.txt\"\r\n" + + "Content-Type: text/plain\r\n\r\n" + + "File contents go here.\r\n" + + "--cdce6441022a3dcf\r\n" + + "Content-Disposition: form-data; name=\"array-to-ignore-expected-string\"\r\n\r\n" + + "1,2\r\n" + + "--cdce6441022a3dcf\r\n" + + "Content-Disposition: form-data; name=\"string\"\r\n\r\n" + + "str\r\n" + + "--cdce6441022a3dcf\r\n" + + "Content-Disposition: form-data; name=\"integer\"\r\n\r\n" + + "12\r\n" + + "--cdce6441022a3dcf\r\n" + + "Content-Disposition: form-data; name=\"id\"\r\n" + + "Content-Type: text/plain\r\n\r\n" + + "123e4567-e89b-12d3-a456-426655440000\r\n" + + "--cdce6441022a3dcf\r\n" + + "Content-Disposition: form-data; name=\"boolean\"\r\n\r\n" + + "false\r\n" + + "--cdce6441022a3dcf--\r\n" + +func Test_addMultipartFormDataParams(t *testing.T) { + type args struct { + body string + params map[string]string + } + tests := []struct { + name string + args args + want *openapi3.Schema + wantErr bool + }{ + { + name: "sanity", + args: args{ + body: formDataBody, + params: map[string]string{"boundary": "cdce6441022a3dcf"}, + }, + want: openapi3.NewObjectSchema().WithProperties(map[string]*openapi3.Schema{ + "upfile": openapi3.NewStringSchema().WithFormat("binary"), + "integer": openapi3.NewInt64Schema(), + "boolean": openapi3.NewBoolSchema(), + "string": openapi3.NewStringSchema(), + "array-to-ignore-expected-string": openapi3.NewStringSchema(), + "id": openapi3.NewUUIDSchema(), + }), + wantErr: false, + }, + { + name: "Multiple File Upload", + args: args{ + body: formDataBodyMultipleFileUpload, + params: map[string]string{"boundary": "cdce6441022a3dcf"}, + }, + want: openapi3.NewObjectSchema().WithProperties(map[string]*openapi3.Schema{ + "fileName": openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema().WithFormat("binary")), + }), + wantErr: false, + }, + { + name: "missing boundary param", + args: args{ + body: formDataBody, + params: map[string]string{}, + }, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getMultipartFormDataSchema(tt.args.body, tt.args.params) + if (err != nil) != tt.wantErr { + t.Errorf("getMultipartFormDataSchema() error = %v, wantErr %v", err, tt.wantErr) + return + } + + assertEqual(t, got, tt.want) + }) + } +} diff --git a/speculator/pkg/apispec/format.go b/speculator/pkg/apispec/format.go new file mode 100644 index 0000000..754efba --- /dev/null +++ b/speculator/pkg/apispec/format.go @@ -0,0 +1,85 @@ +package apispec + +import ( + "time" + + "github.com/xeipuuv/gojsonschema" +) + +var formats = []string{ + "date", + "time", + "date-time", + "email", + "ipv4", + "ipv6", + "uuid", + "json-pointer", + // "relative-json-pointer", // matched with "1.147.1" + // "hostname", + // "regex", + // "uri", // can be also iri + // "uri-reference", // can be also iri-reference + // "uri-template", +} + +func getStringFormat(value interface{}) string { + str, ok := value.(string) + if !ok || str == "" { + return "" + } + + for _, format := range formats { + if gojsonschema.FormatCheckers.IsFormat(format, value) { + return format + } + } + + return "" +} + +// isDateFormat checks if input is a correctly formatted date with spaces (excluding RFC3339 = "2006-01-02T15:04:05Z07:00") +// This is useful to identify date string instead of an array. +func isDateFormat(input interface{}) bool { + asString, ok := input.(string) + if !ok { + return false + } + if _, err := time.Parse(time.ANSIC, asString); err == nil { + return true + } + if _, err := time.Parse(time.UnixDate, asString); err == nil { + return true + } + if _, err := time.Parse(time.RubyDate, asString); err == nil { + return true + } + if _, err := time.Parse(time.RFC822, asString); err == nil { + return true + } + if _, err := time.Parse(time.RFC822Z, asString); err == nil { + return true + } + if _, err := time.Parse(time.RFC850, asString); err == nil { + return true + } + if _, err := time.Parse(time.RFC1123, asString); err == nil { + return true + } + if _, err := time.Parse(time.RFC1123Z, asString); err == nil { + return true + } + if _, err := time.Parse(time.Stamp, asString); err == nil { + return true + } + if _, err := time.Parse(time.StampMilli, asString); err == nil { + return true + } + if _, err := time.Parse(time.StampMicro, asString); err == nil { + return true + } + if _, err := time.Parse(time.StampNano, asString); err == nil { + return true + } + return false +} diff --git a/speculator/pkg/apispec/format_test.go b/speculator/pkg/apispec/format_test.go new file mode 100644 index 0000000..0d00e29 --- /dev/null +++ b/speculator/pkg/apispec/format_test.go @@ -0,0 +1,116 @@ +package apispec + +import ( + "testing" +) + +// format taken from time/format.go. +func Test_isDateFormat(t *testing.T) { + type args struct { + input interface{} + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "RFC3339 should not match", + args: args{ + input: "2021-08-23T06:52:48Z03:00", + }, + want: false, + }, + { + name: "StampNano", + args: args{ + input: "Aug 23 06:52:48.000000000", + }, + want: true, + }, + { + name: "StampMicro", + args: args{ + input: "Aug 23 06:52:48.000000", + }, + want: true, + }, + { + name: "StampMilli", + args: args{ + input: "Aug 23 06:52:48.000", + }, + want: true, + }, + { + name: "Stamp", + args: args{ + input: "Aug 23 06:52:48", + }, + want: true, + }, + { + name: "RFC1123Z", + args: args{ + input: "Mon, 23 Aug 2021 06:52:48 -0300", + }, + want: true, + }, + { + name: "RFC1123", + args: args{ + input: "Mon, 23 Aug 2021 06:52:48 GMT", + }, + want: true, + }, + { + name: "RFC850", + args: args{ + input: "Monday, 23-Aug-21 06:52:48 GMT", + }, + want: true, + }, + { + name: "RFC822Z", + args: args{ + input: "23 Aug 21 06:52 -0300", + }, + want: true, + }, + { + name: "RFC822", + args: args{ + input: "23 Aug 21 06:52 GMT", + }, + want: true, + }, + { + name: "RubyDate", + args: args{ + input: "Mon Aug 23 06:52:48 -0300 2021", + }, + want: true, + }, + { + name: "UnixDate", + args: args{ + input: "Mon Aug 23 06:52:48 GMT 2021", + }, + want: true, + }, + { + name: "ANSIC", + args: args{ + input: "Mon Aug 23 06:52:48 2021", + }, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isDateFormat(tt.args.input); got != tt.want { + t.Errorf("isDateFormat() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/speculator/pkg/apispec/headers.go b/speculator/pkg/apispec/headers.go new file mode 100644 index 0000000..9d320cc --- /dev/null +++ b/speculator/pkg/apispec/headers.go @@ -0,0 +1,92 @@ +package apispec + +import ( + "strings" + + "github.com/getkin/kin-openapi/openapi3" + log "github.com/sirupsen/logrus" +) + +var defaultIgnoredHeaders = []string{ + contentTypeHeaderName, + acceptTypeHeaderName, + authorizationTypeHeaderName, +} + +func createHeadersToIgnore(headers []string) map[string]struct{} { + ret := make(map[string]struct{}) + + for _, header := range append(defaultIgnoredHeaders, headers...) { + ret[strings.ToLower(header)] = struct{}{} + } + + return ret +} + +func shouldIgnoreHeader(headerToIgnore map[string]struct{}, headerKey string) bool { + _, ok := headerToIgnore[strings.ToLower(headerKey)] + return ok +} + +func (o *OperationGenerator) addResponseHeader(response *openapi3.Response, headerKey, headerValue string) *openapi3.Response { + if shouldIgnoreHeader(o.ResponseHeadersToIgnore, headerKey) { + return response + } + + if response.Headers == nil { + response.Headers = make(openapi3.Headers) + } + + response.Headers[headerKey] = &openapi3.HeaderRef{ + Value: &openapi3.Header{ + Parameter: openapi3.Parameter{ + Schema: openapi3.NewSchemaRef("", + getSchemaFromValue(headerValue, true, openapi3.ParameterInHeader)), + }, + }, + } + + return response +} + +// https://swagger.io/docs/specification/describing-parameters/#header-parameters +func (o *OperationGenerator) addHeaderParam(operation *openapi3.Operation, headerKey, headerValue string) *openapi3.Operation { + if shouldIgnoreHeader(o.RequestHeadersToIgnore, headerKey) { + return operation + } + + headerParam := openapi3.NewHeaderParameter(headerKey). + WithSchema(getSchemaFromValue(headerValue, true, openapi3.ParameterInHeader)) + operation.AddParameter(headerParam) + + return operation +} + +// https://swagger.io/docs/specification/describing-parameters/#cookie-parameters +func (o *OperationGenerator) addCookieParam(operation *openapi3.Operation, headerValue string) *openapi3.Operation { + // Multiple cookie parameters are sent in the same header, separated by a semicolon and space. + for _, cookie := range strings.Split(headerValue, "; ") { + cookieKeyAndValue := strings.Split(cookie, "=") + if len(cookieKeyAndValue) != 2 { // nolint:gomnd + log.Warnf("unsupported cookie param. %v", cookie) + continue + } + key, value := cookieKeyAndValue[0], cookieKeyAndValue[1] + // Cookie parameters can be primitive values, arrays and objects. + // Arrays and objects are serialized using the form style. + headerParam := openapi3.NewCookieParameter(key).WithSchema(getSchemaFromValue(value, true, openapi3.ParameterInCookie)) + operation.AddParameter(headerParam) + } + + return operation +} + +func ConvertHeadersToMap(headers []*Header) map[string]string { + headersMap := make(map[string]string) + + for _, header := range headers { + headersMap[strings.ToLower(header.Key)] = header.Value + } + + return headersMap +} diff --git a/speculator/pkg/apispec/headers_test.go b/speculator/pkg/apispec/headers_test.go new file mode 100644 index 0000000..0736e50 --- /dev/null +++ b/speculator/pkg/apispec/headers_test.go @@ -0,0 +1,284 @@ +package apispec + +import ( + "reflect" + "testing" + + "github.com/getkin/kin-openapi/openapi3" +) + +func Test_shouldIgnoreHeader(t *testing.T) { + ignoredHeaders := map[string]struct{}{ + contentTypeHeaderName: {}, + acceptTypeHeaderName: {}, + authorizationTypeHeaderName: {}, + } + type args struct { + headerKey string + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "should ignore", + args: args{ + headerKey: "Accept", + }, + want: true, + }, + { + name: "should ignore", + args: args{ + headerKey: "Content-Type", + }, + want: true, + }, + { + name: "should ignore", + args: args{ + headerKey: "Authorization", + }, + want: true, + }, + { + name: "should not ignore", + args: args{ + headerKey: "X-Test", + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := shouldIgnoreHeader(ignoredHeaders, tt.args.headerKey); got != tt.want { + t.Errorf("shouldIgnoreHeader() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_addResponseHeader(t *testing.T) { + op := NewOperationGenerator(OperationGeneratorConfig{}) + type args struct { + response *openapi3.Response + headerKey string + headerValue string + } + tests := []struct { + name string + args args + want *openapi3.Response + }{ + { + name: "primitive", + args: args{ + response: openapi3.NewResponse(), + headerKey: "X-Test-Uuid", + headerValue: "77e1c83b-7bb0-437b-bc50-a7a58e5660ac", + }, + want: createTestResponse(). + WithHeader("X-Test-Uuid", openapi3.NewUUIDSchema()).Response, + }, + { + name: "array", + args: args{ + response: openapi3.NewResponse(), + headerKey: "X-Test-Array", + headerValue: "1,2,3,4", + }, + want: createTestResponse(). + WithHeader("X-Test-Array", openapi3.NewArraySchema().WithItems(openapi3.NewInt64Schema())).Response, + }, + { + name: "date", + args: args{ + response: openapi3.NewResponse(), + headerKey: "date", + headerValue: "Mon, 23 Aug 2021 06:52:48 GMT", + }, + want: createTestResponse(). + WithHeader("date", openapi3.NewStringSchema()).Response, + }, + { + name: "ignore header", + args: args{ + response: openapi3.NewResponse(), + headerKey: "Accept", + headerValue: "", + }, + want: openapi3.NewResponse(), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := op.addResponseHeader(tt.args.response, tt.args.headerKey, tt.args.headerValue); !reflect.DeepEqual(got, tt.want) { + t.Errorf("addResponseHeader() = %v, want %v", marshal(got), marshal(tt.want)) + } + }) + } +} + +func Test_addHeaderParam(t *testing.T) { + op := NewOperationGenerator(OperationGeneratorConfig{}) + type args struct { + operation *openapi3.Operation + headerKey string + headerValue string + } + tests := []struct { + name string + args args + want *openapi3.Operation + }{ + { + name: "primitive", + args: args{ + operation: openapi3.NewOperation(), + headerKey: "X-Test-Uuid", + headerValue: "77e1c83b-7bb0-437b-bc50-a7a58e5660ac", + }, + want: createTestOperation().WithParameter(openapi3.NewHeaderParameter("X-Test-Uuid"). + WithSchema(openapi3.NewUUIDSchema())).Op, + }, + { + name: "array", + args: args{ + operation: openapi3.NewOperation(), + headerKey: "X-Test-Array", + headerValue: "1,2,3,4", + }, + want: createTestOperation().WithParameter(openapi3.NewHeaderParameter("X-Test-Array"). + WithSchema(openapi3.NewArraySchema().WithItems(openapi3.NewInt64Schema()))).Op, + }, + { + name: "ignore header", + args: args{ + operation: openapi3.NewOperation(), + headerKey: "Accept", + headerValue: "", + }, + want: openapi3.NewOperation(), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := op.addHeaderParam(tt.args.operation, tt.args.headerKey, tt.args.headerValue); !reflect.DeepEqual(got, tt.want) { + t.Errorf("addHeaderParam() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_createHeadersToIgnore(t *testing.T) { + type args struct { + headers []string + } + tests := []struct { + name string + args args + want map[string]struct{} + }{ + { + name: "only default headers", + args: args{ + headers: nil, + }, + want: map[string]struct{}{ + acceptTypeHeaderName: {}, + contentTypeHeaderName: {}, + authorizationTypeHeaderName: {}, + }, + }, + { + name: "with custom headers", + args: args{ + headers: []string{ + "X-H1", + "X-H2", + }, + }, + want: map[string]struct{}{ + acceptTypeHeaderName: {}, + contentTypeHeaderName: {}, + authorizationTypeHeaderName: {}, + "x-h1": {}, + "x-h2": {}, + }, + }, + { + name: "custom headers are sub list of the default headers", + args: args{ + headers: []string{ + acceptTypeHeaderName, + contentTypeHeaderName, + }, + }, + want: map[string]struct{}{ + acceptTypeHeaderName: {}, + contentTypeHeaderName: {}, + authorizationTypeHeaderName: {}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := createHeadersToIgnore(tt.args.headers); !reflect.DeepEqual(got, tt.want) { + t.Errorf("createHeadersToIgnore() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestOperationGenerator_addCookieParam(t *testing.T) { + op := NewOperationGenerator(OperationGeneratorConfig{}) + type args struct { + operation *openapi3.Operation + headerValue string + } + tests := []struct { + name string + args args + want *openapi3.Operation + }{ + { + name: "sanity", + args: args{ + operation: openapi3.NewOperation(), + headerValue: "debug=0; csrftoken=BUSe35dohU3O1MZvDCUOJ", + }, + want: createTestOperation(). + WithParameter(openapi3.NewCookieParameter("debug").WithSchema(openapi3.NewInt64Schema())). + WithParameter(openapi3.NewCookieParameter("csrftoken").WithSchema(openapi3.NewStringSchema())). + Op, + }, + { + name: "array", + args: args{ + operation: openapi3.NewOperation(), + headerValue: "array=1,2,3", + }, + want: createTestOperation(). + WithParameter(openapi3.NewCookieParameter("array").WithSchema(openapi3.NewArraySchema().WithItems(openapi3.NewInt64Schema()))). + Op, + }, + { + name: "unsupported cookie param", + args: args{ + operation: openapi3.NewOperation(), + headerValue: "unsupported=unsupported=unsupported; csrftoken=BUSe35dohU3O1MZvDCUOJ", + }, + want: createTestOperation(). + WithParameter(openapi3.NewCookieParameter("csrftoken").WithSchema(openapi3.NewStringSchema())). + Op, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := op.addCookieParam(tt.args.operation, tt.args.headerValue); !reflect.DeepEqual(got, tt.want) { + t.Errorf("addCookieParam() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/speculator/pkg/apispec/learning_spec.go b/speculator/pkg/apispec/learning_spec.go new file mode 100644 index 0000000..fb9bd84 --- /dev/null +++ b/speculator/pkg/apispec/learning_spec.go @@ -0,0 +1,24 @@ +package apispec + +import ( + "github.com/getkin/kin-openapi/openapi3" +) + +type LearningSpec struct { + // map parameterized path into path item + PathItems map[string]*openapi3.PathItem + SecuritySchemes openapi3.SecuritySchemes +} + +func (l *LearningSpec) AddPathItem(path string, pathItem *openapi3.PathItem) { + l.PathItems[path] = pathItem +} + +func (l *LearningSpec) GetPathItem(path string) *openapi3.PathItem { + pi, ok := l.PathItems[path] + if !ok { + return nil + } + + return pi +} diff --git a/speculator/pkg/apispec/merge.go b/speculator/pkg/apispec/merge.go new file mode 100644 index 0000000..1019f28 --- /dev/null +++ b/speculator/pkg/apispec/merge.go @@ -0,0 +1,516 @@ +package apispec + +import ( + "github.com/getkin/kin-openapi/openapi3" + log "github.com/sirupsen/logrus" + "k8s.io/utils/field" + + "github.com/5gsec/sentryflow/speculator/pkg/util" +) + +var supportedParametersInTypes = []string{openapi3.ParameterInHeader, openapi3.ParameterInQuery, openapi3.ParameterInPath, openapi3.ParameterInCookie} + +func mergeOperation(operation, operation2 *openapi3.Operation) (*openapi3.Operation, []conflict) { + if op, shouldReturn := shouldReturnIfNil(operation, operation2); shouldReturn { + return op.(*openapi3.Operation), nil + } + + var requestBodyConflicts, paramConflicts, resConflicts []conflict + + ret := openapi3.NewOperation() + + ret.RequestBody, requestBodyConflicts = mergeRequestBody(operation.RequestBody, operation2.RequestBody, + field.NewPath("requestBody")) + ret.Parameters, paramConflicts = mergeParameters(operation.Parameters, operation2.Parameters, + field.NewPath("parameters")) + ret.Responses, resConflicts = mergeResponses(*operation.Responses, *operation2.Responses, + field.NewPath("responses")) + + ret.Security = mergeOperationSecurity(operation.Security, operation2.Security) + + conflicts := append(paramConflicts, resConflicts...) + conflicts = append(conflicts, requestBodyConflicts...) + + if len(conflicts) > 0 { + log.Warnf("Found conflicts while merging operation: %v and operation: %v. conflicts: %v", operation, operation2, conflicts) + } + + return ret, conflicts +} + +func mergeOperationSecurity(security, security2 *openapi3.SecurityRequirements) *openapi3.SecurityRequirements { + if s, shouldReturn := shouldReturnIfNil(security, security2); shouldReturn { + return s.(*openapi3.SecurityRequirements) + } + + var mergedSecurity openapi3.SecurityRequirements + + ignoreSecurityKeyMap := map[string]bool{} + + for _, securityMap := range *security { + mergedSecurity, ignoreSecurityKeyMap = appendSecurityIfNeeded(securityMap, mergedSecurity, ignoreSecurityKeyMap) + } + for _, securityMap := range *security2 { + mergedSecurity, ignoreSecurityKeyMap = appendSecurityIfNeeded(securityMap, mergedSecurity, ignoreSecurityKeyMap) + } + + return &mergedSecurity +} + +func appendSecurityIfNeeded(securityMap openapi3.SecurityRequirement, mergedSecurity openapi3.SecurityRequirements, ignoreSecurityKeyMap map[string]bool) (openapi3.SecurityRequirements, map[string]bool) { + for key, values := range securityMap { + // ignore if already appended the exact security key + if ignoreSecurityKeyMap[key] { + continue + } + // https://swagger.io/docs/specification/authentication/ + // We will treat multiple authentication types as an OR + // (Security schemes combined via OR are alternatives – any one can be used in the given context) + mergedSecurity = append(mergedSecurity, map[string][]string{key: values}) + ignoreSecurityKeyMap[key] = true + } + + return mergedSecurity, ignoreSecurityKeyMap +} + +func mergeRequestBody(body, body2 *openapi3.RequestBodyRef, path *field.Path) (*openapi3.RequestBodyRef, []conflict) { + if p, shouldReturn := shouldReturnIfEmptyRequestBody(body, body2); shouldReturn { + return p, nil + } + + content, conflicts := mergeContent(body.Value.Content, body2.Value.Content, path.Child("content")) + + return &openapi3.RequestBodyRef{ + Value: openapi3.NewRequestBody().WithContent(content), + }, conflicts +} + +func shouldReturnIfEmptyRequestBody(body, body2 *openapi3.RequestBodyRef) (*openapi3.RequestBodyRef, bool) { + if isEmptyRequestBody(body) { + return body2, true + } + + if isEmptyRequestBody(body2) { + return body, true + } + + return nil, false +} + +func isEmptyRequestBody(body *openapi3.RequestBodyRef) bool { + return body == nil || body.Value == nil || len(body.Value.Content) == 0 +} + +func mergeParameters(parameters, parameters2 openapi3.Parameters, path *field.Path) (openapi3.Parameters, []conflict) { + if p, shouldReturn := shouldReturnIfEmptyParameters(parameters, parameters2); shouldReturn { + return p, nil + } + + var retParameters openapi3.Parameters + var retConflicts []conflict + + parametersByIn := getParametersByIn(parameters) + parameters2ByIn := getParametersByIn(parameters2) + for _, inType := range supportedParametersInTypes { + mergedParameters, conflicts := mergeParametersByInType(parametersByIn[inType], parameters2ByIn[inType], path) + retParameters = append(retParameters, mergedParameters...) + retConflicts = append(retConflicts, conflicts...) + } + + return retParameters, retConflicts +} + +func getParametersByIn(parameters openapi3.Parameters) map[string]openapi3.Parameters { + ret := make(map[string]openapi3.Parameters) + + for i, parameter := range parameters { + if parameter.Value == nil { + continue + } + + switch parameter.Value.In { + case openapi3.ParameterInCookie, openapi3.ParameterInHeader, openapi3.ParameterInQuery, openapi3.ParameterInPath: + ret[parameter.Value.In] = append(ret[parameter.Value.In], parameters[i]) + default: + log.Warnf("in parameter not supported. %v", parameter.Value.In) + } + } + + return ret +} + +func mergeParametersByInType(parameters, parameters2 openapi3.Parameters, path *field.Path) (openapi3.Parameters, []conflict) { + if p, shouldReturn := shouldReturnIfEmptyParameters(parameters, parameters2); shouldReturn { + return p, nil + } + + var retParameters openapi3.Parameters + var retConflicts []conflict + + parametersMapByName := makeParametersMapByName(parameters) + parameters2MapByName := makeParametersMapByName(parameters2) + + // go over first parameters list + // 1. merge mutual parameters + // 2. add non-mutual parameters + for name, param := range parametersMapByName { + if param2, ok := parameters2MapByName[name]; ok { + mergedParameter, conflicts := mergeParameter(param.Value, param2.Value, path.Child(name)) + retConflicts = append(retConflicts, conflicts...) + retParameters = append(retParameters, &openapi3.ParameterRef{Value: mergedParameter}) + } else { + retParameters = append(retParameters, param) + } + } + + // add non-mutual parameters from the second list + for name, param := range parameters2MapByName { + if _, ok := parametersMapByName[name]; !ok { + retParameters = append(retParameters, param) + } + } + + return retParameters, retConflicts +} + +func makeParametersMapByName(parameters openapi3.Parameters) map[string]*openapi3.ParameterRef { + ret := make(map[string]*openapi3.ParameterRef) + + for i := range parameters { + ret[parameters[i].Value.Name] = parameters[i] + } + + return ret +} + +func mergeParameter(parameter, parameter2 *openapi3.Parameter, path *field.Path) (*openapi3.Parameter, []conflict) { + if p, shouldReturn := shouldReturnIfEmptyParameter(parameter, parameter2); shouldReturn { + return p, nil + } + + type1, type2 := parameter.Schema.Value.Type, parameter2.Schema.Value.Type + switch conflictSolver(type1, type2) { + case NoConflict, PreferType1: + // do nothing, parameter is used. + case PreferType2: + // use parameter2. + type1 = type2 + parameter = parameter2 + case ConflictUnresolved: + return parameter, []conflict{ + { + path: path, + obj1: parameter, + obj2: parameter2, + msg: createConflictMsg(path, type1, type2), + }, + } + } + + if type1.Includes(openapi3.TypeBoolean) || type1.Includes(openapi3.TypeInteger) || type1.Includes(openapi3.TypeNumber) || type1.Includes(openapi3.TypeString) { + schema, conflicts := mergeSchema(parameter.Schema.Value, parameter2.Schema.Value, path) + return parameter.WithSchema(schema), conflicts + } + if type1.Includes(openapi3.TypeArray) { + items, conflicts := mergeSchemaItems(parameter.Schema.Value.Items, parameter2.Schema.Value.Items, path) + return parameter.WithSchema(openapi3.NewArraySchema().WithItems(items.Value)), conflicts + } + if type1.Includes(openapi3.TypeObject) || type1.Includes("") { + // when type is missing it is probably an object - we should try and merge the parameter schema + schema, conflicts := mergeSchema(parameter.Schema.Value, parameter2.Schema.Value, path.Child("schema")) + return parameter.WithSchema(schema), conflicts + } + log.Warnf("unsupported schema type in parameter: %v", type1) + + return parameter, nil +} + +func mergeSchemaItems(items, items2 *openapi3.SchemaRef, path *field.Path) (*openapi3.SchemaRef, []conflict) { + if s, shouldReturn := shouldReturnIfNil(items, items2); shouldReturn { + return s.(*openapi3.SchemaRef), nil + } + schema, conflicts := mergeSchema(items.Value, items2.Value, path.Child("items")) + return &openapi3.SchemaRef{Value: schema}, conflicts +} + +func mergeSchema(schema, schema2 *openapi3.Schema, path *field.Path) (*openapi3.Schema, []conflict) { + if s, shouldReturn := shouldReturnIfNil(schema, schema2); shouldReturn { + return s.(*openapi3.Schema), nil + } + + if s, shouldReturn := shouldReturnIfEmptySchemaType(schema, schema2); shouldReturn { + return s, nil + } + + switch conflictSolver(schema.Type, schema2.Type) { + case NoConflict, PreferType1: + // do nothing, schema is used. + case PreferType2: + // use schema2. + schema = schema2 + case ConflictUnresolved: + return schema, []conflict{ + { + path: path, + obj1: schema, + obj2: schema2, + msg: createConflictMsg(path, schema.Type, schema2.Type), + }, + } + } + + if schema.Type.Includes(openapi3.TypeBoolean) || schema.Type.Includes(openapi3.TypeInteger) || schema.Type.Includes(openapi3.TypeNumber) { + return schema, nil + } + if schema.Type.Includes(openapi3.TypeString) { + // Ignore format only if both schemas are string type and formats are different. + if schema2.Type.Includes(openapi3.TypeString) && schema.Format != schema2.Format { + schema.Format = "" + } + return schema, nil + } + + return schema, nil +} + +func mergeProperties(properties, properties2 openapi3.Schemas, path *field.Path) (openapi3.Schemas, []conflict) { + retProperties := make(openapi3.Schemas) + var retConflicts []conflict + + // go over first properties list + // 1. merge mutual properties + // 2. add non-mutual properties + for key := range properties { + schema := properties[key] + if schema2, ok := properties2[key]; ok { + mergedSchema, conflicts := mergeSchema(schema.Value, schema2.Value, path.Child(key)) + retConflicts = append(retConflicts, conflicts...) + retProperties[key] = &openapi3.SchemaRef{Value: mergedSchema} + } else { + retProperties[key] = schema + } + } + + // add non-mutual properties from the second list + for key, schema := range properties2 { + if _, ok := properties[key]; !ok { + retProperties[key] = schema + } + } + + return retProperties, retConflicts +} + +func mergeResponses(responses, responses2 openapi3.Responses, path *field.Path) (*openapi3.Responses, []conflict) { + if r, shouldReturn := shouldReturnIfEmptyResponses(&responses, &responses2); shouldReturn { + return r, nil + } + + var retConflicts []conflict + + retResponses := openapi3.NewResponses() + + // go over first responses list + // 1. merge mutual response code responses + // 2. add non-mutual response code responses + for code, response := range responses.Map() { + if response2 := responses2.Value(code); response2 != nil { + mergedResponse, conflicts := mergeResponse(response.Value, response2.Value, path.Child(code)) + retConflicts = append(retConflicts, conflicts...) + retResponses.Set(code, &openapi3.ResponseRef{Value: mergedResponse}) + } else { + retResponses.Set(code, responses.Value(code)) + } + } + + // add non-mutual parameters from the second list + for code := range responses2.Map() { + if val := responses.Value(code); val != nil { + retResponses.Set(code, responses2.Value(code)) + } + } + + return retResponses, retConflicts +} + +func mergeResponse(response, response2 *openapi3.Response, path *field.Path) (*openapi3.Response, []conflict) { + var retConflicts []conflict + retResponse := openapi3.NewResponse() + if response.Description != nil { + retResponse = retResponse.WithDescription(*response.Description) + } else if response2.Description != nil { + retResponse = retResponse.WithDescription(*response2.Description) + } + + content, conflicts := mergeContent(response.Content, response2.Content, path.Child("content")) + if len(content) > 0 { + retResponse = retResponse.WithContent(content) + } + retConflicts = append(retConflicts, conflicts...) + + headers, conflicts := mergeResponseHeader(response.Headers, response2.Headers, path.Child("headers")) + if len(headers) > 0 { + retResponse.Headers = headers + } + retConflicts = append(retConflicts, conflicts...) + + return retResponse, retConflicts +} + +func mergeContent(content openapi3.Content, content2 openapi3.Content, path *field.Path) (openapi3.Content, []conflict) { + var retConflicts []conflict + retContent := openapi3.NewContent() + + // go over first content list + // 1. merge mutual content media type + // 2. add non-mutual content media type + for name, mediaType := range content { + if mediaType2, ok := content2[name]; ok { + mergedSchema, conflicts := mergeSchema(mediaType.Schema.Value, mediaType2.Schema.Value, path.Child(name)) + // TODO: handle mediaType.Encoding + retConflicts = append(retConflicts, conflicts...) + retContent[name] = openapi3.NewMediaType().WithSchema(mergedSchema) + } else { + retContent[name] = content[name] + } + } + + // add non-mutual content media type from the second list + for name := range content2 { + if _, ok := content[name]; !ok { + retContent[name] = content2[name] + } + } + + return retContent, retConflicts +} + +func mergeResponseHeader(headers, headers2 openapi3.Headers, path *field.Path) (openapi3.Headers, []conflict) { + var retConflicts []conflict + retHeaders := make(openapi3.Headers) + + // go over first headers list + // 1. merge mutual headers + // 2. add non-mutual headers + for name, header := range headers { + if header2, ok := headers2[name]; ok { + mergedHeader, conflicts := mergeHeader(header.Value, header2.Value, path.Child(name)) + retConflicts = append(retConflicts, conflicts...) + retHeaders[name] = &openapi3.HeaderRef{Value: mergedHeader} + } else { + retHeaders[name] = headers[name] + } + } + + // add non-mutual headers from the second list + for name := range headers2 { + if _, ok := headers[name]; !ok { + retHeaders[name] = headers2[name] + } + } + + return retHeaders, retConflicts +} + +func mergeHeader(header, header2 *openapi3.Header, path *field.Path) (*openapi3.Header, []conflict) { + if h, shouldReturn := shouldReturnIfEmptyHeader(header, header2); shouldReturn { + return h, nil + } + + if header.In != header2.In { + return header, []conflict{ + { + path: path, + obj1: header, + obj2: header2, + msg: createHeaderInConflictMsg(path, header.In, header2.In), + }, + } + } + + schema, conflicts := mergeSchema(header.Schema.Value, header2.Schema.Value, path) + header.Parameter = *header.WithSchema(schema) + + return header, conflicts +} + +func shouldReturnIfEmptyParameter(param, param2 *openapi3.Parameter) (*openapi3.Parameter, bool) { + if isEmptyParameter(param) { + return param2, true + } + + if isEmptyParameter(param2) { + return param, true + } + + return nil, false +} + +func isEmptyParameter(param *openapi3.Parameter) bool { + return param == nil || isEmptySchemaRef(param.Schema) +} + +func shouldReturnIfEmptyHeader(header, header2 *openapi3.Header) (*openapi3.Header, bool) { + if isEmptyHeader(header) { + return header2, true + } + + if isEmptyHeader(header2) { + return header, true + } + + return nil, false +} + +func isEmptyHeader(header *openapi3.Header) bool { + return header == nil || isEmptySchemaRef(header.Schema) +} + +func isEmptySchemaRef(schemaRef *openapi3.SchemaRef) bool { + return schemaRef == nil || schemaRef.Value == nil +} + +func shouldReturnIfEmptyResponses(r, r2 *openapi3.Responses) (*openapi3.Responses, bool) { + if r.Len() == 0 { + return r2, true + } + if r2.Len() == 0 { + return r, true + } + // both are not empty + return nil, false +} + +func shouldReturnIfEmptyParameters(parameters, parameters2 openapi3.Parameters) (openapi3.Parameters, bool) { + if len(parameters) == 0 { + return parameters2, true + } + if len(parameters2) == 0 { + return parameters, true + } + // both are not empty + return nil, false +} + +func shouldReturnIfEmptySchemaType(s, s2 *openapi3.Schema) (*openapi3.Schema, bool) { + if len(s.Type.Slice()) == 0 { + return s2, true + } + if len(s2.Type.Slice()) == 0 { + return s, true + } + // both are not empty + return nil, false +} + +// used only with pointers. +func shouldReturnIfNil(a, b interface{}) (interface{}, bool) { + if util.IsNil(a) { + return b, true + } + if util.IsNil(b) { + return a, true + } + // both are not nil + return nil, false +} diff --git a/speculator/pkg/apispec/merge_test.go b/speculator/pkg/apispec/merge_test.go new file mode 100644 index 0000000..24c5cc8 --- /dev/null +++ b/speculator/pkg/apispec/merge_test.go @@ -0,0 +1,2472 @@ +package apispec + +import ( + "reflect" + "sort" + "testing" + + "github.com/getkin/kin-openapi/openapi3" + "k8s.io/utils/field" +) + +func Test_merge(t *testing.T) { + securitySchemes := openapi3.SecuritySchemes{} + op := CreateTestNewOperationGenerator() + op1, err := op.GenerateSpecOperation(&HTTPInteractionData{ + ReqBody: req1, + RespBody: res1, + ReqHeaders: map[string]string{"X-Test-Req-1": "1", contentTypeHeaderName: mediaTypeApplicationJSON}, + RespHeaders: map[string]string{"X-Test-Res-1": "1", contentTypeHeaderName: mediaTypeApplicationJSON}, + statusCode: 200, + }, securitySchemes) + if err != nil { + t.Fatal(err) + } + + op2, err := op.GenerateSpecOperation(&HTTPInteractionData{ + ReqBody: req2, + RespBody: res2, + ReqHeaders: map[string]string{"X-Test-Req-2": "2", contentTypeHeaderName: mediaTypeApplicationJSON}, + RespHeaders: map[string]string{"X-Test-Res-2": "2", contentTypeHeaderName: mediaTypeApplicationJSON}, + statusCode: 200, + }, securitySchemes) + if err != nil { + t.Fatal(err) + } + + combinedOp, err := op.GenerateSpecOperation(&HTTPInteractionData{ + ReqBody: combinedReq, + RespBody: combinedRes, + ReqHeaders: map[string]string{"X-Test-Req-1": "1", "X-Test-Req-2": "2", contentTypeHeaderName: mediaTypeApplicationJSON}, + RespHeaders: map[string]string{"X-Test-Res-1": "1", "X-Test-Res-2": "2", contentTypeHeaderName: mediaTypeApplicationJSON}, + statusCode: 200, + }, securitySchemes) + if err != nil { + t.Fatal(err) + } + + type args struct { + operation1 *openapi3.Operation + operation2 *openapi3.Operation + } + tests := []struct { + name string + args args + want *openapi3.Operation + wantConflicts bool + }{ + { + name: "sanity", + args: args{ + operation1: op1, + operation2: op2, + }, + want: combinedOp, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, conflicts := mergeOperation(tt.args.operation1, tt.args.operation2) + if (len(conflicts) > 0) != tt.wantConflicts { + t.Errorf("merge() conflicts = %v, wantConflicts %v", conflicts, tt.wantConflicts) + return + } + got = sortParameters(got) + tt.want = sortParameters(tt.want) + assertEqual(t, got, tt.want) + //assert.DeepEqual(t, got, tt.want, cmpopts.IgnoreUnexported(openapi3.Schema{})) + }) + } +} + +func Test_shouldReturnIfEmpty(t *testing.T) { + type args struct { + a interface{} + b interface{} + } + tests := []struct { + name string + args args + want interface{} + want1 bool + }{ + { + name: "second nil", + args: args{ + a: openapi3.NewOperation(), + b: nil, + }, + want: openapi3.NewOperation(), + want1: true, + }, + { + name: "first nil", + args: args{ + a: nil, + b: openapi3.NewOperation(), + }, + want: openapi3.NewOperation(), + want1: true, + }, + { + name: "both nil", + args: args{ + a: nil, + b: nil, + }, + want: nil, + want1: true, + }, + { + name: "not nil", + args: args{ + a: openapi3.NewOperation(), + b: openapi3.NewOperation(), + }, + want: nil, + want1: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := shouldReturnIfNil(tt.args.a, tt.args.b) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("shouldReturnIfNil() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("shouldReturnIfNil() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func Test_shouldReturnIfNil(t *testing.T) { + var nilSchema *openapi3.Schema + schema := openapi3.Schema{Type: &openapi3.Types{"test"}} + type args struct { + a interface{} + b interface{} + } + tests := []struct { + name string + args args + want interface{} + want1 bool + }{ + { + name: "a is nil b is not", + args: args{ + a: nilSchema, + b: schema, + }, + want: schema, + want1: true, + }, + { + name: "b is nil a is not", + args: args{ + a: schema, + b: nilSchema, + }, + want: schema, + want1: true, + }, + { + name: "both nil", + args: args{ + a: nilSchema, + b: nilSchema, + }, + want: nilSchema, + want1: true, + }, + { + name: "both not nil", + args: args{ + a: schema, + b: schema, + }, + want: nil, + want1: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := shouldReturnIfNil(tt.args.a, tt.args.b) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("shouldReturnIfNil() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("shouldReturnIfNil() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func Test_shouldReturnIfEmptySchemaType(t *testing.T) { + emptySchemaType := &openapi3.Schema{} + schema := &openapi3.Schema{Type: &openapi3.Types{"test"}} + type args struct { + s *openapi3.Schema + s2 *openapi3.Schema + } + tests := []struct { + name string + args args + want *openapi3.Schema + want1 bool + }{ + { + name: "first is empty second is not", + args: args{ + s: emptySchemaType, + s2: schema, + }, + want: schema, + want1: true, + }, + { + name: "second is empty first is not", + args: args{ + s: schema, + s2: emptySchemaType, + }, + want: schema, + want1: true, + }, + { + name: "both empty", + args: args{ + s: emptySchemaType, + s2: emptySchemaType, + }, + want: emptySchemaType, + want1: true, + }, + { + name: "both not empty", + args: args{ + s: schema, + s2: schema, + }, + want: nil, + want1: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := shouldReturnIfEmptySchemaType(tt.args.s, tt.args.s2) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("shouldReturnIfEmptySchemaType() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("shouldReturnIfEmptySchemaType() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func Test_shouldReturnIfEmptyParameters(t *testing.T) { + var emptyParameters openapi3.Parameters + parameters := openapi3.Parameters{ + &openapi3.ParameterRef{Value: openapi3.NewHeaderParameter("test")}, + } + type args struct { + parameters openapi3.Parameters + parameters2 openapi3.Parameters + } + tests := []struct { + name string + args args + want openapi3.Parameters + want1 bool + }{ + { + name: "first is empty second is not", + args: args{ + parameters: emptyParameters, + parameters2: parameters, + }, + want: parameters, + want1: true, + }, + { + name: "second is empty first is not", + args: args{ + parameters: parameters, + parameters2: emptyParameters, + }, + want: parameters, + want1: true, + }, + { + name: "both empty", + args: args{ + parameters: emptyParameters, + parameters2: emptyParameters, + }, + want: emptyParameters, + want1: true, + }, + { + name: "both not empty", + args: args{ + parameters: parameters, + parameters2: parameters, + }, + want: nil, + want1: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := shouldReturnIfEmptyParameters(tt.args.parameters, tt.args.parameters2) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("shouldReturnIfEmptyParameters() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("shouldReturnIfEmptyParameters() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func Test_mergeHeader(t *testing.T) { + type args struct { + header *openapi3.Header + header2 *openapi3.Header + child *field.Path + } + tests := []struct { + name string + args args + want *openapi3.Header + want1 []conflict + }{ + { + name: "nothing to merge", + args: args{ + header: &openapi3.Header{ + Parameter: *openapi3.NewHeaderParameter("test").WithSchema(openapi3.NewStringSchema()), + }, + header2: &openapi3.Header{ + Parameter: *openapi3.NewHeaderParameter("test").WithSchema(openapi3.NewStringSchema()), + }, + child: nil, + }, + want: &openapi3.Header{ + Parameter: *openapi3.NewHeaderParameter("test").WithSchema(openapi3.NewStringSchema()), + }, + want1: nil, + }, + { + name: "merge string type removal", + args: args{ + header: &openapi3.Header{ + Parameter: *openapi3.NewHeaderParameter("test").WithSchema(openapi3.NewStringSchema()), + }, + header2: &openapi3.Header{ + Parameter: *openapi3.NewHeaderParameter("test").WithSchema(openapi3.NewUUIDSchema()), + }, + child: nil, + }, + want: &openapi3.Header{ + Parameter: *openapi3.NewHeaderParameter("test").WithSchema(openapi3.NewStringSchema()), + }, + want1: nil, + }, + { + name: "header in conflicts", + args: args{ + header: &openapi3.Header{ + Parameter: *openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewStringSchema()), + }, + header2: &openapi3.Header{ + Parameter: *openapi3.NewCookieParameter("cookie").WithSchema(openapi3.NewArraySchema()), + }, + child: field.NewPath("test"), + }, + want: &openapi3.Header{ + Parameter: *openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewStringSchema()), + }, + want1: []conflict{ + { + path: field.NewPath("test"), + obj1: &openapi3.Header{ + Parameter: *openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewStringSchema()), + }, + obj2: &openapi3.Header{ + Parameter: *openapi3.NewCookieParameter("cookie").WithSchema(openapi3.NewArraySchema()), + }, + msg: createHeaderInConflictMsg(field.NewPath("test"), openapi3.ParameterInHeader, openapi3.ParameterInCookie), + }, + }, + }, + { + name: "type conflicts prefer string", + args: args{ + header: &openapi3.Header{ + Parameter: *openapi3.NewHeaderParameter("test").WithSchema(openapi3.NewStringSchema()), + }, + header2: &openapi3.Header{ + Parameter: *openapi3.NewHeaderParameter("test").WithSchema(openapi3.NewArraySchema()), + }, + child: field.NewPath("test"), + }, + want: &openapi3.Header{ + Parameter: *openapi3.NewHeaderParameter("test").WithSchema(openapi3.NewStringSchema()), + }, + want1: nil, + }, + { + name: "type conflicts", + args: args{ + header: &openapi3.Header{ + Parameter: *openapi3.NewHeaderParameter("test").WithSchema(openapi3.NewInt64Schema()), + }, + header2: &openapi3.Header{ + Parameter: *openapi3.NewHeaderParameter("test").WithSchema(openapi3.NewArraySchema()), + }, + child: field.NewPath("test"), + }, + want: &openapi3.Header{ + Parameter: *openapi3.NewHeaderParameter("test").WithSchema(openapi3.NewInt64Schema()), + }, + want1: []conflict{ + { + path: field.NewPath("test"), + obj1: openapi3.NewInt64Schema(), + obj2: openapi3.NewArraySchema(), + msg: createConflictMsg(field.NewPath("test"), openapi3.TypeInteger, openapi3.TypeArray), + }, + }, + }, + { + name: "empty header", + args: args{ + header: &openapi3.Header{ + Parameter: *openapi3.NewHeaderParameter("empty"), + }, + header2: &openapi3.Header{ + Parameter: *openapi3.NewHeaderParameter("test").WithSchema(openapi3.NewArraySchema()), + }, + child: field.NewPath("test"), + }, + want: &openapi3.Header{ + Parameter: *openapi3.NewHeaderParameter("test").WithSchema(openapi3.NewArraySchema()), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := mergeHeader(tt.args.header, tt.args.header2, tt.args.child) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("mergeHeader() got = %v, want %v", got, tt.want) + } + if !reflect.DeepEqual(got1, tt.want1) { + t.Errorf("mergeHeader() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func Test_mergeResponseHeader(t *testing.T) { + type args struct { + headers openapi3.Headers + headers2 openapi3.Headers + path *field.Path + } + tests := []struct { + name string + args args + want openapi3.Headers + want1 []conflict + }{ + { + name: "first headers list empty", + args: args{ + headers: openapi3.Headers{}, + headers2: openapi3.Headers{ + "test": createHeaderRef(openapi3.NewStringSchema()), + }, + path: nil, + }, + want: openapi3.Headers{ + "test": createHeaderRef(openapi3.NewStringSchema()), + }, + want1: nil, + }, + { + name: "second headers list empty", + args: args{ + headers: openapi3.Headers{ + "test": createHeaderRef(openapi3.NewStringSchema()), + }, + headers2: openapi3.Headers{}, + path: nil, + }, + want: openapi3.Headers{ + "test": createHeaderRef(openapi3.NewStringSchema()), + }, + want1: nil, + }, + { + name: "no common headers", + args: args{ + headers: openapi3.Headers{ + "test": createHeaderRef(openapi3.NewStringSchema()), + }, + headers2: openapi3.Headers{ + "test2": createHeaderRef(openapi3.NewStringSchema()), + }, + path: nil, + }, + want: openapi3.Headers{ + "test": createHeaderRef(openapi3.NewStringSchema()), + "test2": createHeaderRef(openapi3.NewStringSchema()), + }, + want1: nil, + }, + { + name: "merge mutual headers", + args: args{ + headers: openapi3.Headers{ + "test": createHeaderRef(openapi3.NewStringSchema()), + }, + headers2: openapi3.Headers{ + "test": createHeaderRef(openapi3.NewUUIDSchema()), + }, + path: nil, + }, + want: openapi3.Headers{ + "test": createHeaderRef(openapi3.NewStringSchema()), + }, + want1: nil, + }, + { + name: "merge mutual headers and keep non mutual", + args: args{ + headers: openapi3.Headers{ + "mutual": createHeaderRef(openapi3.NewStringSchema()), + "nonmutual1": createHeaderRef(openapi3.NewInt64Schema()), + }, + headers2: openapi3.Headers{ + "mutual": createHeaderRef(openapi3.NewUUIDSchema()), + "nonmutual2": createHeaderRef(openapi3.NewBoolSchema()), + }, + path: nil, + }, + want: openapi3.Headers{ + "mutual": createHeaderRef(openapi3.NewStringSchema()), + "nonmutual1": createHeaderRef(openapi3.NewInt64Schema()), + "nonmutual2": createHeaderRef(openapi3.NewBoolSchema()), + }, + want1: nil, + }, + { + name: "merge mutual headers with conflicts", + args: args{ + headers: openapi3.Headers{ + "test": createHeaderRef(openapi3.NewInt64Schema()), + }, + headers2: openapi3.Headers{ + "test": createHeaderRef(openapi3.NewBoolSchema()), + }, + path: field.NewPath("headers"), + }, + want: openapi3.Headers{ + "test": createHeaderRef(openapi3.NewInt64Schema()), + }, + want1: []conflict{ + { + path: field.NewPath("headers").Child("test"), + obj1: openapi3.NewInt64Schema(), + obj2: openapi3.NewBoolSchema(), + msg: createConflictMsg(field.NewPath("headers").Child("test"), openapi3.TypeInteger, openapi3.TypeBoolean), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := mergeResponseHeader(tt.args.headers, tt.args.headers2, tt.args.path) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("mergeResponseHeader() got = %+v, want %+v", got, tt.want) + } + if !reflect.DeepEqual(got1, tt.want1) { + t.Errorf("mergeResponseHeader() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func createHeaderRef(schema *openapi3.Schema) *openapi3.HeaderRef { + return &openapi3.HeaderRef{ + Value: &openapi3.Header{ + Parameter: openapi3.Parameter{ + Schema: openapi3.NewSchemaRef("", schema), + }, + }, + } +} + +func Test_mergeResponse(t *testing.T) { + type args struct { + response *openapi3.Response + response2 *openapi3.Response + path *field.Path + } + tests := []struct { + name string + args args + want *openapi3.Response + want1 []conflict + }{ + { + name: "first response is empty", + args: args{ + response: openapi3.NewResponse(), + response2: createTestResponse().WithHeader("X-Header", openapi3.NewStringSchema()).Response, + path: nil, + }, + want: createTestResponse().WithHeader("X-Header", openapi3.NewStringSchema()).Response, + want1: nil, + }, + { + name: "second response is empty", + args: args{ + response: createTestResponse().WithHeader("X-Header", openapi3.NewStringSchema()).Response, + response2: openapi3.NewResponse(), + path: nil, + }, + want: createTestResponse().WithHeader("X-Header", openapi3.NewStringSchema()).Response, + want1: nil, + }, + { + name: "merge response schema", + args: args{ + response: createTestResponse(). + WithJSONSchema(openapi3.NewDateTimeSchema()). + WithHeader("X-Header", openapi3.NewStringSchema()).Response, + response2: createTestResponse(). + WithJSONSchema(openapi3.NewStringSchema()). + WithHeader("X-Header", openapi3.NewStringSchema()).Response, + path: nil, + }, + want: createTestResponse(). + WithJSONSchema(openapi3.NewStringSchema()). + WithHeader("X-Header", openapi3.NewStringSchema()).Response, + want1: nil, + }, + { + name: "merge response header", + args: args{ + response: createTestResponse(). + WithJSONSchema(openapi3.NewStringSchema()). + WithHeader("X-Header", openapi3.NewUUIDSchema()).Response, + response2: createTestResponse(). + WithJSONSchema(openapi3.NewStringSchema()). + WithHeader("X-Header", openapi3.NewStringSchema()).Response, + path: nil, + }, + want: createTestResponse(). + WithJSONSchema(openapi3.NewStringSchema()). + WithHeader("X-Header", openapi3.NewStringSchema()).Response, + want1: nil, + }, + { + name: "merge response header and schema", + args: args{ + response: createTestResponse(). + WithJSONSchema(openapi3.NewDateTimeSchema()). + WithHeader("X-Header", openapi3.NewUUIDSchema()).Response, + response2: createTestResponse(). + WithJSONSchema(openapi3.NewStringSchema()). + WithHeader("X-Header", openapi3.NewStringSchema()).Response, + path: nil, + }, + want: createTestResponse(). + WithJSONSchema(openapi3.NewStringSchema()). + WithHeader("X-Header", openapi3.NewStringSchema()).Response, + want1: nil, + }, + { + name: "merge response header and schema prefer number", + args: args{ + response: createTestResponse(). + WithJSONSchema(openapi3.NewFloat64Schema()). + WithHeader("X-Header", openapi3.NewUUIDSchema()).Response, + response2: createTestResponse(). + WithJSONSchema(openapi3.NewInt64Schema()). + WithHeader("X-Header", openapi3.NewBoolSchema()).Response, + path: field.NewPath("200"), + }, + want: createTestResponse(). + WithJSONSchema(openapi3.NewFloat64Schema()). + WithHeader("X-Header", openapi3.NewUUIDSchema()).Response, + want1: nil, + }, + { + name: "merge response header and schema with conflicts", + args: args{ + response: createTestResponse(). + WithJSONSchema(openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())). + WithHeader("X-Header", openapi3.NewFloat64Schema()).Response, + response2: createTestResponse(). + WithJSONSchema(openapi3.NewInt64Schema()). + WithHeader("X-Header", openapi3.NewBoolSchema()).Response, + path: field.NewPath("200"), + }, + want: createTestResponse(). + WithJSONSchema(openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())). + WithHeader("X-Header", openapi3.NewFloat64Schema()).Response, + want1: []conflict{ + { + path: field.NewPath("200").Child("content").Child("application/json"), + obj1: openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema()), + obj2: openapi3.NewInt64Schema(), + msg: createConflictMsg(field.NewPath("200").Child("content").Child("application/json"), + openapi3.TypeArray, openapi3.TypeInteger), + }, + { + path: field.NewPath("200").Child("headers").Child("X-Header"), + obj1: openapi3.NewFloat64Schema(), + obj2: openapi3.NewBoolSchema(), + msg: createConflictMsg(field.NewPath("200").Child("headers").Child("X-Header"), + openapi3.TypeNumber, openapi3.TypeBoolean), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := mergeResponse(tt.args.response, tt.args.response2, tt.args.path) + assertEqual(t, got, tt.want) + //assert.DeepEqual(t, got, tt.want, cmpopts.IgnoreUnexported(openapi3.Schema{})) + if !reflect.DeepEqual(got1, tt.want1) { + t.Errorf("mergeResponse() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func Test_mergeResponses(t *testing.T) { + type args struct { + responses openapi3.Responses + responses2 openapi3.Responses + path *field.Path + } + tests := []struct { + name string + args args + want openapi3.Responses + want1 []conflict + }{ + { + name: "first is nil", + args: args{ + responses: openapi3.Responses{}, + responses2: createTestResponses(). + WithResponse("200", createTestResponse(). + WithJSONSchema(openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())). + WithHeader("X-Header", openapi3.NewUUIDSchema()).Response).Responses, + path: nil, + }, + want: createTestResponses(). + WithResponse("200", createTestResponse(). + WithJSONSchema(openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())). + WithHeader("X-Header", openapi3.NewUUIDSchema()).Response).Responses, + want1: nil, + }, + { + name: "second is nil", + args: args{ + responses: createTestResponses(). + WithResponse("200", createTestResponse(). + WithJSONSchema(openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())). + WithHeader("X-Header", openapi3.NewUUIDSchema()).Response).Responses, + responses2: openapi3.Responses{}, + path: nil, + }, + want: createTestResponses(). + WithResponse("200", createTestResponse(). + WithJSONSchema(openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())). + WithHeader("X-Header", openapi3.NewUUIDSchema()).Response).Responses, + want1: nil, + }, + { + name: "both are nil", + args: args{ + responses: openapi3.Responses{}, + responses2: openapi3.Responses{}, + path: nil, + }, + want: openapi3.Responses{}, + want1: nil, + }, + { + name: "non mutual response code responses", + args: args{ + responses: createTestResponses(). + WithResponse("200", createTestResponse(). + WithJSONSchema(openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())). + WithHeader("X-Header", openapi3.NewUUIDSchema()).Response).Responses, + responses2: createTestResponses(). + WithResponse("201", createTestResponse(). + WithJSONSchema(openapi3.NewStringSchema()). + WithHeader("X-Header2", openapi3.NewUUIDSchema()).Response).Responses, + path: nil, + }, + want: createTestResponses(). + WithResponse("200", createTestResponse(). + WithJSONSchema(openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())). + WithHeader("X-Header", openapi3.NewUUIDSchema()).Response). + WithResponse("201", createTestResponse(). + WithJSONSchema(openapi3.NewStringSchema()). + WithHeader("X-Header2", openapi3.NewUUIDSchema()).Response).Responses, + want1: nil, + }, + { + name: "mutual response code responses", + args: args{ + responses: createTestResponses(). + WithResponse("200", createTestResponse(). + WithJSONSchema(openapi3.NewArraySchema().WithItems(openapi3.NewDateTimeSchema())). + WithHeader("X-Header", openapi3.NewUUIDSchema()).Response).Responses, + responses2: createTestResponses(). + WithResponse("200", createTestResponse(). + WithJSONSchema(openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())). + WithHeader("X-Header", openapi3.NewUUIDSchema()).Response).Responses, + path: nil, + }, + want: createTestResponses(). + WithResponse("200", createTestResponse(). + WithJSONSchema(openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())). + WithHeader("X-Header", openapi3.NewUUIDSchema()).Response).Responses, + want1: nil, + }, + { + name: "mutual and non mutual response code responses", + args: args{ + responses: createTestResponses(). + WithResponse("200", createTestResponse(). + WithJSONSchema(openapi3.NewArraySchema().WithItems(openapi3.NewDateTimeSchema())). + WithHeader("X-Header", openapi3.NewUUIDSchema()).Response). + WithResponse("201", createTestResponse(). + WithJSONSchema(openapi3.NewDateTimeSchema()). + WithHeader("X-Header1", openapi3.NewUUIDSchema()).Response).Responses, + responses2: createTestResponses(). + WithResponse("200", createTestResponse(). + WithJSONSchema(openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())). + WithHeader("X-Header", openapi3.NewUUIDSchema()).Response). + WithResponse("202", createTestResponse(). + WithJSONSchema(openapi3.NewBoolSchema()). + WithHeader("X-Header3", openapi3.NewUUIDSchema()).Response).Responses, + path: nil, + }, + want: createTestResponses(). + WithResponse("200", createTestResponse(). + WithJSONSchema(openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())). + WithHeader("X-Header", openapi3.NewUUIDSchema()).Response). + WithResponse("201", createTestResponse(). + WithJSONSchema(openapi3.NewDateTimeSchema()). + WithHeader("X-Header1", openapi3.NewUUIDSchema()).Response). + WithResponse("202", createTestResponse(). + WithJSONSchema(openapi3.NewBoolSchema()). + WithHeader("X-Header3", openapi3.NewUUIDSchema()).Response).Responses, + want1: nil, + }, + { + name: "mutual and non mutual response code responses solve conflicts", + args: args{ + responses: createTestResponses(). + WithResponse("200", createTestResponse(). + WithJSONSchema(openapi3.NewArraySchema().WithItems(openapi3.NewArraySchema().WithItems(openapi3.NewDateTimeSchema()))). + WithHeader("X-Header", openapi3.NewUUIDSchema()).Response). + WithResponse("201", createTestResponse(). + WithJSONSchema(openapi3.NewDateTimeSchema()). + WithHeader("X-Header1", openapi3.NewUUIDSchema()).Response).Responses, + responses2: createTestResponses(). + WithResponse("200", createTestResponse(). + WithJSONSchema(openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())). + WithHeader("X-Header", openapi3.NewUUIDSchema()).Response). + WithResponse("202", createTestResponse(). + WithJSONSchema(openapi3.NewBoolSchema()). + WithHeader("X-Header3", openapi3.NewUUIDSchema()).Response).Responses, + path: field.NewPath("responses"), + }, + want: createTestResponses(). + WithResponse("200", createTestResponse(). + WithJSONSchema(openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())). + WithHeader("X-Header", openapi3.NewUUIDSchema()).Response). + WithResponse("201", createTestResponse(). + WithJSONSchema(openapi3.NewDateTimeSchema()). + WithHeader("X-Header1", openapi3.NewUUIDSchema()).Response). + WithResponse("202", createTestResponse(). + WithJSONSchema(openapi3.NewBoolSchema()). + WithHeader("X-Header3", openapi3.NewUUIDSchema()).Response).Responses, + want1: nil, + }, + { + name: "mutual and non mutual response code responses with conflicts", + args: args{ + responses: createTestResponses(). + WithResponse("200", createTestResponse(). + WithJSONSchema(openapi3.NewArraySchema().WithItems(openapi3.NewArraySchema().WithItems(openapi3.NewDateTimeSchema()))). + WithHeader("X-Header", openapi3.NewUUIDSchema()).Response). + WithResponse("201", createTestResponse(). + WithJSONSchema(openapi3.NewDateTimeSchema()). + WithHeader("X-Header1", openapi3.NewUUIDSchema()).Response).Responses, + responses2: createTestResponses(). + WithResponse("200", createTestResponse(). + WithJSONSchema(openapi3.NewArraySchema().WithItems(openapi3.NewFloat64Schema())). + WithHeader("X-Header", openapi3.NewUUIDSchema()).Response). + WithResponse("202", createTestResponse(). + WithJSONSchema(openapi3.NewBoolSchema()). + WithHeader("X-Header3", openapi3.NewUUIDSchema()).Response).Responses, + path: field.NewPath("responses"), + }, + want: createTestResponses(). + WithResponse("200", createTestResponse(). + WithJSONSchema(openapi3.NewArraySchema().WithItems(openapi3.NewArraySchema().WithItems(openapi3.NewDateTimeSchema()))). + WithHeader("X-Header", openapi3.NewUUIDSchema()).Response). + WithResponse("201", createTestResponse(). + WithJSONSchema(openapi3.NewDateTimeSchema()). + WithHeader("X-Header1", openapi3.NewUUIDSchema()).Response). + WithResponse("202", createTestResponse(). + WithJSONSchema(openapi3.NewBoolSchema()). + WithHeader("X-Header3", openapi3.NewUUIDSchema()).Response).Responses, + want1: []conflict{ + { + path: field.NewPath("responses").Child("200").Child("content"). + Child("application/json").Child("items"), + obj1: openapi3.NewArraySchema().WithItems(openapi3.NewDateTimeSchema()), + obj2: openapi3.NewFloat64Schema(), + msg: createConflictMsg(field.NewPath("responses").Child("200").Child("content"). + Child("application/json").Child("items"), openapi3.TypeArray, openapi3.TypeNumber), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := mergeResponses(tt.args.responses, tt.args.responses2, tt.args.path) + //assert.DeepEqual(t, got, tt.want, cmpopts.IgnoreUnexported(openapi3.Schema{})) + assertEqual(t, got, tt.want) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("mergeResponses() got = %v, want %v", got, tt.want) + } + if !reflect.DeepEqual(got1, tt.want1) { + t.Errorf("mergeResponses() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func Test_mergeProperties(t *testing.T) { + type args struct { + properties openapi3.Schemas + properties2 openapi3.Schemas + path *field.Path + } + tests := []struct { + name string + args args + want openapi3.Schemas + want1 []conflict + }{ + { + name: "first is nil", + args: args{ + properties: nil, + properties2: openapi3.Schemas{ + "string-key": openapi3.NewSchemaRef("", openapi3.NewStringSchema()), + }, + path: nil, + }, + want: openapi3.Schemas{ + "string-key": openapi3.NewSchemaRef("", openapi3.NewStringSchema()), + }, + want1: nil, + }, + { + name: "second is nil", + args: args{ + properties: openapi3.Schemas{ + "string-key": openapi3.NewSchemaRef("", openapi3.NewStringSchema()), + }, + properties2: nil, + path: nil, + }, + want: openapi3.Schemas{ + "string-key": openapi3.NewSchemaRef("", openapi3.NewStringSchema()), + }, + want1: nil, + }, + { + name: "both are nil", + args: args{ + properties: nil, + properties2: nil, + path: nil, + }, + want: make(openapi3.Schemas), + want1: nil, + }, + { + name: "non mutual properties", + args: args{ + properties: openapi3.Schemas{ + "string-key": openapi3.NewSchemaRef("", openapi3.NewStringSchema()), + }, + properties2: openapi3.Schemas{ + "bool-key": openapi3.NewSchemaRef("", openapi3.NewBoolSchema()), + }, + path: nil, + }, + want: openapi3.Schemas{ + "string-key": openapi3.NewSchemaRef("", openapi3.NewStringSchema()), + "bool-key": openapi3.NewSchemaRef("", openapi3.NewBoolSchema()), + }, + want1: nil, + }, + { + name: "mutual properties", + args: args{ + properties: openapi3.Schemas{ + "string-key": openapi3.NewSchemaRef("", openapi3.NewStringSchema()), + }, + properties2: openapi3.Schemas{ + "string-key": openapi3.NewSchemaRef("", openapi3.NewStringSchema()), + }, + path: nil, + }, + want: openapi3.Schemas{ + "string-key": openapi3.NewSchemaRef("", openapi3.NewStringSchema()), + }, + want1: nil, + }, + { + name: "mutual and non mutual properties", + args: args{ + properties: openapi3.Schemas{ + "string-key": openapi3.NewSchemaRef("", openapi3.NewStringSchema()), + "bool-key": openapi3.NewSchemaRef("", openapi3.NewBoolSchema()), + }, + properties2: openapi3.Schemas{ + "string-key": openapi3.NewSchemaRef("", openapi3.NewStringSchema()), + "int-key": openapi3.NewSchemaRef("", openapi3.NewInt64Schema()), + }, + path: nil, + }, + want: openapi3.Schemas{ + "string-key": openapi3.NewSchemaRef("", openapi3.NewStringSchema()), + "int-key": openapi3.NewSchemaRef("", openapi3.NewInt64Schema()), + "bool-key": openapi3.NewSchemaRef("", openapi3.NewBoolSchema()), + }, + want1: nil, + }, + { + name: "mutual and non mutual response code responses with conflicts", + args: args{ + properties: openapi3.Schemas{ + "conflict": openapi3.NewSchemaRef("", openapi3.NewBoolSchema()), + "bool-key": openapi3.NewSchemaRef("", openapi3.NewBoolSchema()), + }, + properties2: openapi3.Schemas{ + "conflict": openapi3.NewSchemaRef("", openapi3.NewInt64Schema()), + "int-key": openapi3.NewSchemaRef("", openapi3.NewInt64Schema()), + }, + path: field.NewPath("properties"), + }, + want: openapi3.Schemas{ + "conflict": openapi3.NewSchemaRef("", openapi3.NewBoolSchema()), + "int-key": openapi3.NewSchemaRef("", openapi3.NewInt64Schema()), + "bool-key": openapi3.NewSchemaRef("", openapi3.NewBoolSchema()), + }, + want1: []conflict{ + { + path: field.NewPath("properties").Child("conflict"), + obj1: openapi3.NewBoolSchema(), + obj2: openapi3.NewInt64Schema(), + msg: createConflictMsg(field.NewPath("properties").Child("conflict"), + openapi3.TypeBoolean, openapi3.TypeInteger), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := mergeProperties(tt.args.properties, tt.args.properties2, tt.args.path) + //assert.DeepEqual(t, got, tt.want, cmpopts.IgnoreUnexported(openapi3.Schema{})) + assertEqual(t, got, tt.want) + if !reflect.DeepEqual(got1, tt.want1) { + t.Errorf("mergeProperties() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func Test_mergeSchemaItems(t *testing.T) { + type args struct { + items *openapi3.SchemaRef + items2 *openapi3.SchemaRef + path *field.Path + } + tests := []struct { + name string + args args + want *openapi3.SchemaRef + want1 []conflict + }{ + { + name: "no merge needed", + args: args{ + items: openapi3.NewSchemaRef("", openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())), + items2: openapi3.NewSchemaRef("", openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())), + path: field.NewPath("test"), + }, + want: openapi3.NewSchemaRef("", openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())), + want1: nil, + }, + { + name: "items with string format - format should be removed", + args: args{ + items: openapi3.NewSchemaRef("", openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())), + items2: openapi3.NewSchemaRef("", openapi3.NewArraySchema().WithItems(openapi3.NewUUIDSchema())), + path: field.NewPath("test"), + }, + want: openapi3.NewSchemaRef("", openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())), + want1: nil, + }, + { + name: "different type of items", + args: args{ + items: openapi3.NewSchemaRef("", openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())), + items2: openapi3.NewSchemaRef("", openapi3.NewInt64Schema()), + path: field.NewPath("test"), + }, + want: openapi3.NewSchemaRef("", openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())), + want1: []conflict{ + { + path: field.NewPath("test").Child("items"), + obj1: openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema()), + obj2: openapi3.NewInt64Schema(), + msg: createConflictMsg(field.NewPath("test").Child("items"), openapi3.TypeArray, openapi3.TypeInteger), + }, + }, + }, + { + name: "items2 nil items - expected to get items", + args: args{ + items: openapi3.NewSchemaRef("", openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())), + items2: nil, + path: field.NewPath("test"), + }, + want: openapi3.NewSchemaRef("", openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())), + want1: nil, + }, + { + name: "items2 nil schema - expected to get items", + args: args{ + items: openapi3.NewSchemaRef("", openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())), + items2: openapi3.NewSchemaRef("", openapi3.NewArraySchema().WithItems(nil)), + path: field.NewPath("test"), + }, + want: openapi3.NewSchemaRef("", openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())), + want1: nil, + }, + { + name: "items nil items - expected to get items2", + args: args{ + items: nil, + items2: openapi3.NewSchemaRef("", openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())), + path: field.NewPath("test"), + }, + want: openapi3.NewSchemaRef("", openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())), + want1: nil, + }, + { + name: "both schemas nil items - expected to get schema", + args: args{ + items: nil, + items2: nil, + path: field.NewPath("test"), + }, + want: nil, + want1: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := mergeSchemaItems(tt.args.items, tt.args.items2, tt.args.path) + //assert.DeepEqual(t, got, tt.want, cmpopts.IgnoreUnexported(openapi3.Schema{})) + assertEqual(t, got, tt.want) + if !reflect.DeepEqual(got1, tt.want1) { + t.Errorf("mergeSchemaItems() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func Test_mergeSchema(t *testing.T) { + emptySchemaType := openapi3.NewSchema() + type args struct { + schema *openapi3.Schema + schema2 *openapi3.Schema + path *field.Path + } + tests := []struct { + name string + args args + want *openapi3.Schema + want1 []conflict + }{ + { + name: "no merge needed", + args: args{ + schema: openapi3.NewInt64Schema(), + schema2: openapi3.NewInt64Schema(), + path: nil, + }, + want: openapi3.NewInt64Schema(), + want1: nil, + }, + { + name: "first is nil", + args: args{ + schema: nil, + schema2: openapi3.NewInt64Schema(), + path: nil, + }, + want: openapi3.NewInt64Schema(), + want1: nil, + }, + { + name: "second is nil", + args: args{ + schema: openapi3.NewInt64Schema(), + schema2: nil, + path: nil, + }, + want: openapi3.NewInt64Schema(), + want1: nil, + }, + { + name: "both are nil", + args: args{ + schema: nil, + schema2: nil, + path: nil, + }, + want: nil, + want1: nil, + }, + { + name: "first has empty schema type", + args: args{ + schema: emptySchemaType, + schema2: openapi3.NewInt64Schema(), + path: nil, + }, + want: openapi3.NewInt64Schema(), + want1: nil, + }, + { + name: "second has empty schema type", + args: args{ + schema: openapi3.NewInt64Schema(), + schema2: emptySchemaType, + path: nil, + }, + want: openapi3.NewInt64Schema(), + want1: nil, + }, + { + name: "both has empty schema type", + args: args{ + schema: emptySchemaType, + schema2: emptySchemaType, + path: nil, + }, + want: emptySchemaType, + want1: nil, + }, + { + name: "type conflict", + args: args{ + schema: openapi3.NewInt64Schema(), + schema2: openapi3.NewBoolSchema(), + path: field.NewPath("schema"), + }, + want: openapi3.NewInt64Schema(), + want1: []conflict{ + { + path: field.NewPath("schema"), + obj1: openapi3.NewInt64Schema(), + obj2: openapi3.NewBoolSchema(), + msg: createConflictMsg(field.NewPath("schema"), openapi3.TypeInteger, openapi3.TypeBoolean), + }, + }, + }, + { + name: "string type with different format - dismiss the format", + args: args{ + schema: openapi3.NewDateTimeSchema(), + schema2: openapi3.NewUUIDSchema(), + path: field.NewPath("schema"), + }, + want: openapi3.NewStringSchema(), + want1: nil, + }, + { + name: "array conflict prefer number", + args: args{ + schema: openapi3.NewArraySchema().WithItems(openapi3.NewInt64Schema()), + schema2: openapi3.NewArraySchema().WithItems(openapi3.NewFloat64Schema()), + path: field.NewPath("schema"), + }, + want: openapi3.NewArraySchema().WithItems(openapi3.NewFloat64Schema()), + want1: nil, + }, + { + name: "array conflict", + args: args{ + schema: openapi3.NewArraySchema().WithItems(openapi3.NewInt64Schema()), + schema2: openapi3.NewArraySchema().WithItems(openapi3.NewObjectSchema()), + path: field.NewPath("schema"), + }, + want: openapi3.NewArraySchema().WithItems(openapi3.NewInt64Schema()), + want1: []conflict{ + { + path: field.NewPath("schema").Child("items"), + obj1: openapi3.NewInt64Schema(), + obj2: openapi3.NewObjectSchema(), + msg: createConflictMsg(field.NewPath("schema").Child("items"), openapi3.TypeInteger, openapi3.TypeObject), + }, + }, + }, + { + name: "merge object with conflict", + args: args{ + schema: openapi3.NewObjectSchema(). + WithProperty("bool", openapi3.NewBoolSchema()). + WithProperty("conflict prefer string", openapi3.NewBoolSchema()). + WithProperty("conflict", openapi3.NewObjectSchema()), + schema2: openapi3.NewObjectSchema(). + WithProperty("float", openapi3.NewFloat64Schema()). + WithProperty("conflict prefer string", openapi3.NewStringSchema()). + WithProperty("conflict", openapi3.NewInt64Schema()), + path: field.NewPath("schema"), + }, + want: openapi3.NewObjectSchema(). + WithProperty("bool", openapi3.NewBoolSchema()). + WithProperty("conflict prefer string", openapi3.NewStringSchema()). + WithProperty("conflict", openapi3.NewObjectSchema()). + WithProperty("float", openapi3.NewFloat64Schema()), + want1: []conflict{ + { + path: field.NewPath("schema").Child("properties").Child("conflict"), + obj1: openapi3.NewObjectSchema(), + obj2: openapi3.NewInt64Schema(), + msg: createConflictMsg(field.NewPath("schema").Child("properties").Child("conflict"), + openapi3.TypeObject, openapi3.TypeInteger), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := mergeSchema(tt.args.schema, tt.args.schema2, tt.args.path) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("mergeSchema() got = %v, want %v", got, tt.want) + } + if !reflect.DeepEqual(got1, tt.want1) { + t.Errorf("mergeSchema() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func Test_mergeParameter(t *testing.T) { + type args struct { + parameter *openapi3.Parameter + parameter2 *openapi3.Parameter + path *field.Path + } + tests := []struct { + name string + args args + want *openapi3.Parameter + want1 []conflict + }{ + { + name: "param type solve conflict", + args: args{ + parameter: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewStringSchema()), + parameter2: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewBoolSchema()), + path: field.NewPath("param-name"), + }, + want: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewStringSchema()), + want1: nil, + }, + { + name: "param type conflict", + args: args{ + parameter: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewInt64Schema()), + parameter2: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewBoolSchema()), + path: field.NewPath("param-name"), + }, + want: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewInt64Schema()), + want1: []conflict{ + { + path: field.NewPath("param-name"), + obj1: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewInt64Schema()), + obj2: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewBoolSchema()), + msg: createConflictMsg(field.NewPath("param-name"), openapi3.TypeInteger, openapi3.TypeBoolean), + }, + }, + }, + { + name: "string merge", + args: args{ + parameter: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewStringSchema()), + parameter2: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewUUIDSchema()), + path: field.NewPath("param-name"), + }, + want: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewStringSchema()), + want1: nil, + }, + { + name: "array merge with conflict", + args: args{ + parameter: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())), + parameter2: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewArraySchema().WithItems(openapi3.NewBoolSchema())), + path: field.NewPath("param-name"), + }, + want: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema())), + want1: nil, + }, + { + name: "object merge", + args: args{ + parameter: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewObjectSchema().WithProperty("string", openapi3.NewStringSchema())), + parameter2: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewObjectSchema().WithProperty("bool", openapi3.NewBoolSchema())), + path: field.NewPath("param-name"), + }, + want: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewObjectSchema(). + WithProperty("bool", openapi3.NewBoolSchema()). + WithProperty("string", openapi3.NewStringSchema())), + want1: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := mergeParameter(tt.args.parameter, tt.args.parameter2, tt.args.path) + //assert.DeepEqual(t, got, tt.want, cmpopts.IgnoreUnexported(openapi3.Schema{})) + assertEqual(t, got, tt.want) + if !reflect.DeepEqual(got1, tt.want1) { + t.Errorf("mergeParameter() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func Test_makeParametersMapByName(t *testing.T) { + type args struct { + parameters openapi3.Parameters + } + tests := []struct { + name string + args args + want map[string]*openapi3.ParameterRef + }{ + { + name: "sanity", + args: args{ + parameters: openapi3.Parameters{ + &openapi3.ParameterRef{Value: openapi3.NewHeaderParameter("header")}, + &openapi3.ParameterRef{Value: openapi3.NewHeaderParameter("header2")}, + &openapi3.ParameterRef{Value: openapi3.NewPathParameter("path")}, + &openapi3.ParameterRef{Value: openapi3.NewPathParameter("path2")}, + }, + }, + want: map[string]*openapi3.ParameterRef{ + "header": {Value: openapi3.NewHeaderParameter("header")}, + "header2": {Value: openapi3.NewHeaderParameter("header2")}, + "path": {Value: openapi3.NewPathParameter("path")}, + "path2": {Value: openapi3.NewPathParameter("path2")}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := makeParametersMapByName(tt.args.parameters); !reflect.DeepEqual(got, tt.want) { + t.Errorf("makeParametersMapByName() = %v, want %v", marshal(got), marshal(tt.want)) + } + }) + } +} + +func Test_mergeParametersByInType(t *testing.T) { + type args struct { + parameters openapi3.Parameters + parameters2 openapi3.Parameters + path *field.Path + } + tests := []struct { + name string + args args + want openapi3.Parameters + want1 []conflict + }{ + { + name: "first is nil", + args: args{ + parameters: nil, + parameters2: openapi3.Parameters{{Value: openapi3.NewHeaderParameter("h")}}, + path: nil, + }, + want: openapi3.Parameters{{Value: openapi3.NewHeaderParameter("h")}}, + want1: nil, + }, + { + name: "second is nil", + args: args{ + parameters: openapi3.Parameters{{Value: openapi3.NewHeaderParameter("h")}}, + parameters2: nil, + path: nil, + }, + want: openapi3.Parameters{{Value: openapi3.NewHeaderParameter("h")}}, + want1: nil, + }, + { + name: "both are nil", + args: args{ + parameters: nil, + parameters2: nil, + path: nil, + }, + want: nil, + want1: nil, + }, + { + name: "non mutual parameters", + args: args{ + parameters: openapi3.Parameters{{Value: openapi3.NewHeaderParameter("X-Header-1")}}, + parameters2: openapi3.Parameters{{Value: openapi3.NewHeaderParameter("X-Header-2")}}, + path: nil, + }, + want: openapi3.Parameters{{Value: openapi3.NewHeaderParameter("X-Header-1")}, {Value: openapi3.NewHeaderParameter("X-Header-2")}}, + want1: nil, + }, + { + name: "mutual parameters", + args: args{ + parameters: openapi3.Parameters{{Value: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewUUIDSchema())}}, + parameters2: openapi3.Parameters{{Value: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewStringSchema())}}, + path: nil, + }, + want: openapi3.Parameters{{Value: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewStringSchema())}}, + want1: nil, + }, + { + name: "mutual and non mutual parameters", + args: args{ + parameters: openapi3.Parameters{ + {Value: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewUUIDSchema())}, + {Value: openapi3.NewHeaderParameter("X-Header-2").WithSchema(openapi3.NewBoolSchema())}, + }, + parameters2: openapi3.Parameters{ + {Value: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewStringSchema())}, + {Value: openapi3.NewHeaderParameter("X-Header-3").WithSchema(openapi3.NewInt64Schema())}, + }, + path: nil, + }, + want: openapi3.Parameters{ + {Value: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewStringSchema())}, + {Value: openapi3.NewHeaderParameter("X-Header-2").WithSchema(openapi3.NewBoolSchema())}, + {Value: openapi3.NewHeaderParameter("X-Header-3").WithSchema(openapi3.NewInt64Schema())}, + }, + want1: nil, + }, + { + name: "mutual and non mutual parameters solve conflicts", + args: args{ + parameters: openapi3.Parameters{ + {Value: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewUUIDSchema())}, + {Value: openapi3.NewHeaderParameter("X-Header-2").WithSchema(openapi3.NewBoolSchema())}, + }, + parameters2: openapi3.Parameters{ + {Value: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewBoolSchema())}, + {Value: openapi3.NewHeaderParameter("X-Header-3").WithSchema(openapi3.NewInt64Schema())}, + }, + path: field.NewPath("parameters"), + }, + want: openapi3.Parameters{ + {Value: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewUUIDSchema())}, + {Value: openapi3.NewHeaderParameter("X-Header-2").WithSchema(openapi3.NewBoolSchema())}, + {Value: openapi3.NewHeaderParameter("X-Header-3").WithSchema(openapi3.NewInt64Schema())}, + }, + want1: nil, + }, + { + name: "mutual and non mutual parameters with conflicts", + args: args{ + parameters: openapi3.Parameters{ + {Value: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewInt64Schema())}, + {Value: openapi3.NewHeaderParameter("X-Header-2").WithSchema(openapi3.NewBoolSchema())}, + }, + parameters2: openapi3.Parameters{ + {Value: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewBoolSchema())}, + {Value: openapi3.NewHeaderParameter("X-Header-3").WithSchema(openapi3.NewInt64Schema())}, + }, + path: field.NewPath("parameters"), + }, + want: openapi3.Parameters{ + {Value: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewInt64Schema())}, + {Value: openapi3.NewHeaderParameter("X-Header-2").WithSchema(openapi3.NewBoolSchema())}, + {Value: openapi3.NewHeaderParameter("X-Header-3").WithSchema(openapi3.NewInt64Schema())}, + }, + want1: []conflict{ + { + path: field.NewPath("parameters").Child("X-Header-1"), + obj1: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewInt64Schema()), + obj2: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewBoolSchema()), + msg: createConflictMsg(field.NewPath("parameters").Child("X-Header-1"), openapi3.TypeInteger, + openapi3.TypeBoolean), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := mergeParametersByInType(tt.args.parameters, tt.args.parameters2, tt.args.path) + sortParam(got) + sortParam(tt.want) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("mergeParametersByInType() got = %v, want %v", got, tt.want) + } + if !reflect.DeepEqual(got1, tt.want1) { + t.Errorf("mergeParametersByInType() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func Test_getParametersByIn(t *testing.T) { + type args struct { + parameters openapi3.Parameters + } + tests := []struct { + name string + args args + want map[string]openapi3.Parameters + }{ + { + name: "sanity", + args: args{ + parameters: openapi3.Parameters{ + {Value: openapi3.NewHeaderParameter("h1")}, + {Value: openapi3.NewHeaderParameter("h2")}, + {Value: openapi3.NewPathParameter("p1")}, + {Value: openapi3.NewPathParameter("p2")}, + {Value: openapi3.NewQueryParameter("q1")}, + {Value: openapi3.NewQueryParameter("q2")}, + {Value: openapi3.NewCookieParameter("c1")}, + {Value: openapi3.NewCookieParameter("c2")}, + {Value: &openapi3.Parameter{In: "not-supported"}}, + }, + }, + want: map[string]openapi3.Parameters{ + openapi3.ParameterInCookie: {{Value: openapi3.NewCookieParameter("c1")}, {Value: openapi3.NewCookieParameter("c2")}}, + openapi3.ParameterInHeader: {{Value: openapi3.NewHeaderParameter("h1")}, {Value: openapi3.NewHeaderParameter("h2")}}, + openapi3.ParameterInQuery: {{Value: openapi3.NewQueryParameter("q1")}, {Value: openapi3.NewQueryParameter("q2")}}, + openapi3.ParameterInPath: {{Value: openapi3.NewPathParameter("p1")}, {Value: openapi3.NewPathParameter("p2")}}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getParametersByIn(tt.args.parameters); !reflect.DeepEqual(got, tt.want) { + t.Errorf("getParametersByIn() = %v, want %v", marshal(got), marshal(tt.want)) + } + }) + } +} + +func Test_mergeParameters(t *testing.T) { + type args struct { + parameters openapi3.Parameters + parameters2 openapi3.Parameters + path *field.Path + } + tests := []struct { + name string + args args + want openapi3.Parameters + want1 []conflict + }{ + { + name: "first is nil", + args: args{ + parameters: nil, + parameters2: openapi3.Parameters{{Value: openapi3.NewHeaderParameter("h")}}, + path: nil, + }, + want: openapi3.Parameters{{Value: openapi3.NewHeaderParameter("h")}}, + want1: nil, + }, + { + name: "second is nil", + args: args{ + parameters: openapi3.Parameters{{Value: openapi3.NewHeaderParameter("h")}}, + parameters2: nil, + path: nil, + }, + want: openapi3.Parameters{{Value: openapi3.NewHeaderParameter("h")}}, + want1: nil, + }, + { + name: "both are nil", + args: args{ + parameters: nil, + parameters2: nil, + path: nil, + }, + want: nil, + want1: nil, + }, + { + name: "non mutual parameters", + args: args{ + parameters: openapi3.Parameters{ + {Value: openapi3.NewHeaderParameter("X-Header-1")}, + {Value: openapi3.NewQueryParameter("query-1")}, + }, + parameters2: openapi3.Parameters{ + {Value: openapi3.NewHeaderParameter("X-Header-2")}, + {Value: openapi3.NewQueryParameter("query-2")}, + {Value: openapi3.NewHeaderParameter("header")}, + }, + path: nil, + }, + want: openapi3.Parameters{ + {Value: openapi3.NewHeaderParameter("X-Header-1")}, + {Value: openapi3.NewQueryParameter("query-1")}, + {Value: openapi3.NewHeaderParameter("header")}, + {Value: openapi3.NewHeaderParameter("X-Header-2")}, + {Value: openapi3.NewQueryParameter("query-2")}, + }, + want1: nil, + }, + { + name: "mutual parameters", + args: args{ + parameters: openapi3.Parameters{ + {Value: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewUUIDSchema())}, + {Value: openapi3.NewQueryParameter("query-1").WithSchema(openapi3.NewUUIDSchema())}, + {Value: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewObjectSchema().WithProperty("str", openapi3.NewStringSchema()))}, + }, + parameters2: openapi3.Parameters{ + {Value: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewUUIDSchema())}, + {Value: openapi3.NewQueryParameter("query-1").WithSchema(openapi3.NewUUIDSchema())}, + {Value: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewObjectSchema().WithProperty("str", openapi3.NewDateTimeSchema()))}, + }, + path: nil, + }, + want: openapi3.Parameters{ + {Value: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewUUIDSchema())}, + {Value: openapi3.NewQueryParameter("query-1").WithSchema(openapi3.NewUUIDSchema())}, + {Value: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewObjectSchema().WithProperty("str", openapi3.NewStringSchema()))}, + }, + want1: nil, + }, + { + name: "mutual and non mutual parameters", + args: args{ + parameters: openapi3.Parameters{ + {Value: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewUUIDSchema())}, + {Value: openapi3.NewQueryParameter("query-1").WithSchema(openapi3.NewUUIDSchema())}, + {Value: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewObjectSchema().WithProperty("str", openapi3.NewStringSchema()))}, + {Value: openapi3.NewPathParameter("non-mutual-1").WithSchema(openapi3.NewStringSchema())}, + {Value: openapi3.NewCookieParameter("non-mutual-2").WithSchema(openapi3.NewStringSchema())}, + }, + parameters2: openapi3.Parameters{ + {Value: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewUUIDSchema())}, + {Value: openapi3.NewQueryParameter("query-1").WithSchema(openapi3.NewUUIDSchema())}, + {Value: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewObjectSchema().WithProperty("str", openapi3.NewDateTimeSchema()))}, + {Value: openapi3.NewPathParameter("non-mutual-3").WithSchema(openapi3.NewStringSchema())}, + {Value: openapi3.NewCookieParameter("non-mutual-4").WithSchema(openapi3.NewStringSchema())}, + }, + path: nil, + }, + want: openapi3.Parameters{ + {Value: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewUUIDSchema())}, + {Value: openapi3.NewQueryParameter("query-1").WithSchema(openapi3.NewUUIDSchema())}, + {Value: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewObjectSchema().WithProperty("str", openapi3.NewStringSchema()))}, + {Value: openapi3.NewPathParameter("non-mutual-1").WithSchema(openapi3.NewStringSchema())}, + {Value: openapi3.NewCookieParameter("non-mutual-2").WithSchema(openapi3.NewStringSchema())}, + {Value: openapi3.NewPathParameter("non-mutual-3").WithSchema(openapi3.NewStringSchema())}, + {Value: openapi3.NewCookieParameter("non-mutual-4").WithSchema(openapi3.NewStringSchema())}, + }, + want1: nil, + }, + { + name: "mutual and non mutual parameters solve conflicts", + args: args{ + parameters: openapi3.Parameters{ + {Value: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewBoolSchema())}, + {Value: openapi3.NewQueryParameter("query-1").WithSchema(openapi3.NewInt64Schema())}, + {Value: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewObjectSchema(). + WithProperty("bool", openapi3.NewBoolSchema()))}, + {Value: openapi3.NewPathParameter("non-mutual-1").WithSchema(openapi3.NewStringSchema())}, + {Value: openapi3.NewCookieParameter("non-mutual-2").WithSchema(openapi3.NewStringSchema())}, + }, + parameters2: openapi3.Parameters{ + {Value: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewUUIDSchema())}, + {Value: openapi3.NewQueryParameter("query-1").WithSchema(openapi3.NewUUIDSchema())}, + {Value: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewObjectSchema(). + WithProperty("str", openapi3.NewDateTimeSchema()))}, + {Value: openapi3.NewPathParameter("non-mutual-3").WithSchema(openapi3.NewStringSchema())}, + {Value: openapi3.NewCookieParameter("non-mutual-4").WithSchema(openapi3.NewStringSchema())}, + }, + path: field.NewPath("parameters"), + }, + want: openapi3.Parameters{ + {Value: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewUUIDSchema())}, + {Value: openapi3.NewQueryParameter("query-1").WithSchema(openapi3.NewUUIDSchema())}, + {Value: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewObjectSchema(). + WithProperty("str", openapi3.NewDateTimeSchema()). + WithProperty("bool", openapi3.NewBoolSchema()))}, + {Value: openapi3.NewPathParameter("non-mutual-1").WithSchema(openapi3.NewStringSchema())}, + {Value: openapi3.NewCookieParameter("non-mutual-2").WithSchema(openapi3.NewStringSchema())}, + {Value: openapi3.NewPathParameter("non-mutual-3").WithSchema(openapi3.NewStringSchema())}, + {Value: openapi3.NewCookieParameter("non-mutual-4").WithSchema(openapi3.NewStringSchema())}, + }, + want1: nil, + }, + { + name: "mutual and non mutual parameters with conflicts", + args: args{ + parameters: openapi3.Parameters{ + {Value: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewBoolSchema())}, + {Value: openapi3.NewQueryParameter("query-1").WithSchema(openapi3.NewInt64Schema())}, + {Value: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewObjectSchema(). + WithProperty("bool", openapi3.NewBoolSchema()))}, + {Value: openapi3.NewPathParameter("non-mutual-1").WithSchema(openapi3.NewStringSchema())}, + {Value: openapi3.NewCookieParameter("non-mutual-2").WithSchema(openapi3.NewStringSchema())}, + }, + parameters2: openapi3.Parameters{ + {Value: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewInt64Schema())}, + {Value: openapi3.NewQueryParameter("query-1").WithSchema(openapi3.NewBoolSchema())}, + {Value: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewObjectSchema(). + WithProperty("str", openapi3.NewDateTimeSchema()))}, + {Value: openapi3.NewPathParameter("non-mutual-3").WithSchema(openapi3.NewStringSchema())}, + {Value: openapi3.NewCookieParameter("non-mutual-4").WithSchema(openapi3.NewStringSchema())}, + }, + path: field.NewPath("parameters"), + }, + want: openapi3.Parameters{ + {Value: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewBoolSchema())}, + {Value: openapi3.NewQueryParameter("query-1").WithSchema(openapi3.NewInt64Schema())}, + {Value: openapi3.NewHeaderParameter("header").WithSchema(openapi3.NewObjectSchema(). + WithProperty("str", openapi3.NewDateTimeSchema()). + WithProperty("bool", openapi3.NewBoolSchema()))}, + {Value: openapi3.NewPathParameter("non-mutual-1").WithSchema(openapi3.NewStringSchema())}, + {Value: openapi3.NewCookieParameter("non-mutual-2").WithSchema(openapi3.NewStringSchema())}, + {Value: openapi3.NewPathParameter("non-mutual-3").WithSchema(openapi3.NewStringSchema())}, + {Value: openapi3.NewCookieParameter("non-mutual-4").WithSchema(openapi3.NewStringSchema())}, + }, + want1: []conflict{ + { + path: field.NewPath("parameters").Child("X-Header-1"), + obj1: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewBoolSchema()), + obj2: openapi3.NewHeaderParameter("X-Header-1").WithSchema(openapi3.NewInt64Schema()), + msg: createConflictMsg(field.NewPath("parameters").Child("X-Header-1"), openapi3.TypeBoolean, + openapi3.TypeInteger), + }, + { + path: field.NewPath("parameters").Child("query-1"), + obj1: openapi3.NewQueryParameter("query-1").WithSchema(openapi3.NewInt64Schema()), + obj2: openapi3.NewQueryParameter("query-1").WithSchema(openapi3.NewBoolSchema()), + msg: createConflictMsg(field.NewPath("parameters").Child("query-1"), openapi3.TypeInteger, + openapi3.TypeBoolean), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := mergeParameters(tt.args.parameters, tt.args.parameters2, tt.args.path) + sortParam(got) + sortParam(tt.want) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("mergeParameters() got = %v, want %v", marshal(got), marshal(tt.want)) + } + if !reflect.DeepEqual(got1, tt.want1) { + t.Errorf("mergeParameters() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func sortParam(got openapi3.Parameters) { + sort.Slice(got, func(i, j int) bool { + right := got[i] + left := got[j] + // Sibling parameters must have unique name + in values + return right.Value.Name+right.Value.In < left.Value.Name+left.Value.In + }) +} + +func Test_appendSecurityIfNeeded(t *testing.T) { + type args struct { + securityMap openapi3.SecurityRequirement + mergedSecurity openapi3.SecurityRequirements + ignoreSecurityKeyMap map[string]bool + } + tests := []struct { + name string + args args + wantMergedSecurity openapi3.SecurityRequirements + wantIgnoreSecurityKeyMap map[string]bool + }{ + { + name: "sanity", + args: args{ + securityMap: openapi3.SecurityRequirement{"key": {"val1", "val2"}}, + mergedSecurity: nil, + ignoreSecurityKeyMap: map[string]bool{}, + }, + wantMergedSecurity: openapi3.SecurityRequirements{{"key": {"val1", "val2"}}}, + wantIgnoreSecurityKeyMap: map[string]bool{"key": true}, + }, + { + name: "key should be ignored", + args: args{ + securityMap: openapi3.SecurityRequirement{"key": {"val1", "val2"}}, + mergedSecurity: openapi3.SecurityRequirements{{"old-key": {}}}, + ignoreSecurityKeyMap: map[string]bool{"key": true}, + }, + wantMergedSecurity: openapi3.SecurityRequirements{{"old-key": {}}}, + wantIgnoreSecurityKeyMap: map[string]bool{"key": true}, + }, + { + name: "new key should not be ignored, old key should be ignored", + args: args{ + securityMap: openapi3.SecurityRequirement{"old-key": {}, "new key": {"val1", "val2"}}, + mergedSecurity: openapi3.SecurityRequirements{{"old-key": {}}}, + ignoreSecurityKeyMap: map[string]bool{"old-key": true, "key": true}, + }, + wantMergedSecurity: openapi3.SecurityRequirements{{"old-key": {}}, {"new key": {"val1", "val2"}}}, + wantIgnoreSecurityKeyMap: map[string]bool{"old-key": true, "key": true, "new key": true}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := appendSecurityIfNeeded(tt.args.securityMap, tt.args.mergedSecurity, tt.args.ignoreSecurityKeyMap) + if !reflect.DeepEqual(got, tt.wantMergedSecurity) { + t.Errorf("appendSecurityIfNeeded() got = %v, want %v", got, tt.wantMergedSecurity) + } + if !reflect.DeepEqual(got1, tt.wantIgnoreSecurityKeyMap) { + t.Errorf("appendSecurityIfNeeded() got1 = %v, want %v", got1, tt.wantIgnoreSecurityKeyMap) + } + }) + } +} + +func Test_mergeOperationSecurity(t *testing.T) { + type args struct { + security *openapi3.SecurityRequirements + security2 *openapi3.SecurityRequirements + } + tests := []struct { + name string + args args + want *openapi3.SecurityRequirements + }{ + { + name: "no merge is needed", + args: args{ + security: &openapi3.SecurityRequirements{{"key1": {}}, {"key2": {"val1", "val2"}}}, + security2: &openapi3.SecurityRequirements{{"key1": {}}, {"key2": {"val1", "val2"}}}, + }, + want: &openapi3.SecurityRequirements{{"key1": {}}, {"key2": {"val1", "val2"}}}, + }, + { + name: "full merge", + args: args{ + security: &openapi3.SecurityRequirements{{"key1": {}}}, + security2: &openapi3.SecurityRequirements{{"key2": {"val1", "val2"}}}, + }, + want: &openapi3.SecurityRequirements{{"key1": {}}, {"key2": {"val1", "val2"}}}, + }, + { + name: "second list is a sub list of the first - result should be the first list", + args: args{ + security: &openapi3.SecurityRequirements{{"key1": {}}, {"key2": {"val1", "val2"}}, {"key3": {}}}, + security2: &openapi3.SecurityRequirements{{"key2": {"val1", "val2"}}}, + }, + want: &openapi3.SecurityRequirements{{"key1": {}}, {"key2": {"val1", "val2"}}, {"key3": {}}}, + }, + { + name: "first list is provided as an AND - output as OR", + args: args{ + security: &openapi3.SecurityRequirements{ + {"key1": {} /*AND*/, "key2": {"val1", "val2"}}, + }, + security2: &openapi3.SecurityRequirements{{"key2": {"val1", "val2"}}}, + }, + want: &openapi3.SecurityRequirements{ + {"key1": {}}, + /*OR*/ + {"key2": {"val1", "val2"}}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := mergeOperationSecurity(tt.args.security, tt.args.security2) + sort.Slice(*got, func(i, j int) bool { + _, ok := (*got)[i]["key1"] + return ok + }) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("mergeOperationSecurity() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_isEmptyRequestBody(t *testing.T) { + nonEmptyContent := openapi3.NewContent() + nonEmptyContent["test"] = openapi3.NewMediaType() + type args struct { + body *openapi3.RequestBodyRef + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "body == nil", + args: args{ + body: nil, + }, + want: true, + }, + { + name: "body.Value == nil", + args: args{ + body: &openapi3.RequestBodyRef{Value: nil}, + }, + want: true, + }, + { + name: "len(body.Value.Content) == 0", + args: args{ + body: &openapi3.RequestBodyRef{Value: openapi3.NewRequestBody().WithContent(nil)}, + }, + want: true, + }, + { + name: "len(body.Value.Content) == 0", + args: args{ + body: &openapi3.RequestBodyRef{Value: openapi3.NewRequestBody().WithContent(openapi3.Content{})}, + }, + want: true, + }, + { + name: "not empty", + args: args{ + body: &openapi3.RequestBodyRef{Value: openapi3.NewRequestBody().WithContent(nonEmptyContent)}, + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isEmptyRequestBody(tt.args.body); got != tt.want { + t.Errorf("isEmptyRequestBody() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_shouldReturnIfEmptyRequestBody(t *testing.T) { + nonEmptyContent := openapi3.NewContent() + nonEmptyContent["test"] = openapi3.NewMediaType() + reqBody := &openapi3.RequestBodyRef{Value: openapi3.NewRequestBody().WithContent(nonEmptyContent)} + type args struct { + body *openapi3.RequestBodyRef + body2 *openapi3.RequestBodyRef + } + tests := []struct { + name string + args args + want *openapi3.RequestBodyRef + want1 bool + }{ + { + name: "first body is nil", + args: args{ + body: nil, + body2: reqBody, + }, + want: reqBody, + want1: true, + }, + { + name: "second body is nil", + args: args{ + body: reqBody, + body2: nil, + }, + want: reqBody, + want1: true, + }, + { + name: "both bodies non nil", + args: args{ + body: reqBody, + body2: reqBody, + }, + want: nil, + want1: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := shouldReturnIfEmptyRequestBody(tt.args.body, tt.args.body2) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("shouldReturnIfEmptyRequestBody() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("shouldReturnIfEmptyRequestBody() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func Test_mergeRequestBody(t *testing.T) { + requestBody := openapi3.NewRequestBody() + requestBody.Content = openapi3.NewContent() + requestBody.Content["application/json"] = openapi3.NewMediaType().WithSchema(openapi3.NewStringSchema()) + requestBody.Content["application/xml"] = openapi3.NewMediaType().WithSchema(openapi3.NewStringSchema()) + + type args struct { + body *openapi3.RequestBodyRef + body2 *openapi3.RequestBodyRef + path *field.Path + } + tests := []struct { + name string + args args + want *openapi3.RequestBodyRef + want1 []conflict + }{ + { + name: "first is nil", + args: args{ + body: nil, + body2: &openapi3.RequestBodyRef{ + Value: openapi3.NewRequestBody().WithJSONSchema(openapi3.NewStringSchema()), + }, + path: nil, + }, + want: &openapi3.RequestBodyRef{ + Value: openapi3.NewRequestBody().WithJSONSchema(openapi3.NewStringSchema()), + }, + want1: nil, + }, + { + name: "second is nil", + args: args{ + body: &openapi3.RequestBodyRef{ + Value: openapi3.NewRequestBody().WithJSONSchema(openapi3.NewStringSchema()), + }, + body2: nil, + path: nil, + }, + want: &openapi3.RequestBodyRef{ + Value: openapi3.NewRequestBody().WithJSONSchema(openapi3.NewStringSchema()), + }, + want1: nil, + }, + { + name: "both are nil", + args: args{ + body: nil, + body2: nil, + path: nil, + }, + want: nil, + want1: nil, + }, + { + name: "non mutual contents", + args: args{ + body: &openapi3.RequestBodyRef{ + Value: openapi3.NewRequestBody().WithJSONSchema(openapi3.NewStringSchema()), + }, + body2: &openapi3.RequestBodyRef{ + Value: openapi3.NewRequestBody().WithSchema(openapi3.NewStringSchema(), []string{"application/xml"}), + }, + path: nil, + }, + want: &openapi3.RequestBodyRef{ + Value: openapi3.NewRequestBody(). + WithSchema(openapi3.NewStringSchema(), []string{"application/json", "application/xml"}), + }, + want1: nil, + }, + { + name: "mutual contents", + args: args{ + body: &openapi3.RequestBodyRef{ + Value: openapi3.NewRequestBody().WithJSONSchema(openapi3.NewStringSchema()), + }, + body2: &openapi3.RequestBodyRef{ + Value: openapi3.NewRequestBody().WithJSONSchema(openapi3.NewStringSchema()), + }, + path: nil, + }, + want: &openapi3.RequestBodyRef{ + Value: openapi3.NewRequestBody(). + WithJSONSchema(openapi3.NewStringSchema()), + }, + want1: nil, + }, + { + name: "mutual and non mutual contents", + args: args{ + body: &openapi3.RequestBodyRef{ + Value: openapi3.NewRequestBody().WithJSONSchema(openapi3.NewStringSchema()), + }, + body2: &openapi3.RequestBodyRef{ + Value: openapi3.NewRequestBody().WithSchema(openapi3.NewStringSchema(), []string{"application/xml", "application/json"}), + }, + path: nil, + }, + want: &openapi3.RequestBodyRef{ + Value: openapi3.NewRequestBody(). + WithSchema(openapi3.NewStringSchema(), []string{"application/xml", "application/json"}), + }, + want1: nil, + }, + { + name: "non mutual contents solve conflicts", + args: args{ + body: &openapi3.RequestBodyRef{ + Value: openapi3.NewRequestBody(). + WithSchema(openapi3.NewStringSchema(), []string{"application/xml"}), + }, + body2: &openapi3.RequestBodyRef{ + Value: openapi3.NewRequestBody(). + WithSchema(openapi3.NewInt64Schema(), []string{"application/xml"}), + }, + path: field.NewPath("requestBody"), + }, + want: &openapi3.RequestBodyRef{ + Value: openapi3.NewRequestBody(). + WithSchema(openapi3.NewStringSchema(), []string{"application/xml"}), + }, + want1: nil, + }, + { + name: "non mutual contents with conflicts", + args: args{ + body: &openapi3.RequestBodyRef{ + Value: openapi3.NewRequestBody(). + WithSchema(openapi3.NewBoolSchema(), []string{"application/xml"}), + }, + body2: &openapi3.RequestBodyRef{ + Value: openapi3.NewRequestBody(). + WithSchema(openapi3.NewInt64Schema(), []string{"application/xml"}), + }, + path: field.NewPath("requestBody"), + }, + want: &openapi3.RequestBodyRef{ + Value: openapi3.NewRequestBody(). + WithSchema(openapi3.NewBoolSchema(), []string{"application/xml"}), + }, + want1: []conflict{ + { + path: field.NewPath("requestBody").Child("content").Child("application/xml"), + obj1: openapi3.NewBoolSchema(), + obj2: openapi3.NewInt64Schema(), + msg: createConflictMsg(field.NewPath("requestBody").Child("content").Child("application/xml"), + openapi3.TypeBoolean, openapi3.TypeInteger), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := mergeRequestBody(tt.args.body, tt.args.body2, tt.args.path) + //assert.DeepEqual(t, got, tt.want, cmpopts.IgnoreUnexported(openapi3.Schema{})) + assertEqual(t, got, tt.want) + if !reflect.DeepEqual(got1, tt.want1) { + t.Errorf("mergeRequestBody() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func Test_mergeContent(t *testing.T) { + type args struct { + content openapi3.Content + content2 openapi3.Content + path *field.Path + } + tests := []struct { + name string + args args + want openapi3.Content + want1 []conflict + }{ + { + name: "first is nil", + args: args{ + content: nil, + content2: openapi3.Content{ + "json": openapi3.NewMediaType().WithSchema(openapi3.NewStringSchema()), + }, + path: nil, + }, + want: openapi3.Content{ + "json": openapi3.NewMediaType().WithSchema(openapi3.NewStringSchema()), + }, + want1: nil, + }, + { + name: "second is nil", + args: args{ + content: openapi3.Content{ + "json": openapi3.NewMediaType().WithSchema(openapi3.NewStringSchema()), + }, + content2: nil, + path: nil, + }, + want: openapi3.Content{ + "json": openapi3.NewMediaType().WithSchema(openapi3.NewStringSchema()), + }, + want1: nil, + }, + { + name: "both are nil", + args: args{ + content: nil, + content2: nil, + path: nil, + }, + want: openapi3.NewContent(), + want1: nil, + }, + { + name: "non mutual contents", + args: args{ + content: openapi3.Content{ + "json": openapi3.NewMediaType().WithSchema(openapi3.NewStringSchema()), + }, + content2: openapi3.Content{ + "xml": openapi3.NewMediaType().WithSchema(openapi3.NewStringSchema()), + }, + path: nil, + }, + want: openapi3.Content{ + "json": openapi3.NewMediaType().WithSchema(openapi3.NewStringSchema()), + "xml": openapi3.NewMediaType().WithSchema(openapi3.NewStringSchema()), + }, + want1: nil, + }, + { + name: "mutual contents", + args: args{ + content: openapi3.Content{ + "json": openapi3.NewMediaType().WithSchema(openapi3.NewStringSchema()), + }, + content2: openapi3.Content{ + "json": openapi3.NewMediaType().WithSchema(openapi3.NewStringSchema()), + }, + path: nil, + }, + want: openapi3.Content{ + "json": openapi3.NewMediaType().WithSchema(openapi3.NewStringSchema()), + }, + want1: nil, + }, + { + name: "mutual and non mutual contents", + args: args{ + content: openapi3.Content{ + "json": openapi3.NewMediaType().WithSchema(openapi3.NewStringSchema()), + "foo": openapi3.NewMediaType().WithSchema(openapi3.NewInt64Schema()), + }, + content2: openapi3.Content{ + "json": openapi3.NewMediaType().WithSchema(openapi3.NewStringSchema()), + "xml": openapi3.NewMediaType().WithSchema(openapi3.NewStringSchema()), + }, + path: nil, + }, + want: openapi3.Content{ + "json": openapi3.NewMediaType().WithSchema(openapi3.NewStringSchema()), + "foo": openapi3.NewMediaType().WithSchema(openapi3.NewInt64Schema()), + "xml": openapi3.NewMediaType().WithSchema(openapi3.NewStringSchema()), + }, + want1: nil, + }, + { + name: "mutual contents solve conflicts", + args: args{ + content: openapi3.Content{ + "xml": openapi3.NewMediaType().WithSchema(openapi3.NewStringSchema()), + }, + content2: openapi3.Content{ + "xml": openapi3.NewMediaType().WithSchema(openapi3.NewInt64Schema()), + }, + path: field.NewPath("start"), + }, + want: openapi3.Content{ + "xml": openapi3.NewMediaType().WithSchema(openapi3.NewStringSchema()), + }, + want1: nil, + }, + { + name: "mutual contents with conflicts", + args: args{ + content: openapi3.Content{ + "xml": openapi3.NewMediaType().WithSchema(openapi3.NewBoolSchema()), + }, + content2: openapi3.Content{ + "xml": openapi3.NewMediaType().WithSchema(openapi3.NewInt64Schema()), + }, + path: field.NewPath("start"), + }, + want: openapi3.Content{ + "xml": openapi3.NewMediaType().WithSchema(openapi3.NewBoolSchema()), + }, + want1: []conflict{ + { + path: field.NewPath("start").Child("xml"), + obj1: openapi3.NewBoolSchema(), + obj2: openapi3.NewInt64Schema(), + msg: createConflictMsg(field.NewPath("start").Child("xml"), openapi3.TypeBoolean, openapi3.TypeInteger), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := mergeContent(tt.args.content, tt.args.content2, tt.args.path) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("mergeContent() got = %v, want %v", got, tt.want) + } + if !reflect.DeepEqual(got1, tt.want1) { + t.Errorf("mergeContent() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} diff --git a/speculator/pkg/apispec/operation.go b/speculator/pkg/apispec/operation.go new file mode 100644 index 0000000..50e23d5 --- /dev/null +++ b/speculator/pkg/apispec/operation.go @@ -0,0 +1,463 @@ +package apispec + +import ( + "encoding/json" + "fmt" + "mime" + "net/url" + "strings" + + "github.com/getkin/kin-openapi/openapi3" + "github.com/golang-jwt/jwt/v5" + log "github.com/sirupsen/logrus" + "github.com/spf13/cast" + "github.com/xeipuuv/gojsonschema" + + "github.com/5gsec/sentryflow/speculator/pkg/util" +) + +func getSchema(value interface{}) (schema *openapi3.Schema, err error) { + switch value.(type) { + case bool: + schema = openapi3.NewBoolSchema() + case string: + schema = getStringSchema(value) + case json.Number: + schema = getNumberSchema(value) + case map[string]interface{}: + schema, err = getObjectSchema(value) + if err != nil { + return nil, err + } + case []interface{}: + schema, err = getArraySchema(value) + if err != nil { + return nil, err + } + case nil: + // TODO: Not sure how to handle null. ex: {"size":3,"err":null} + schema = openapi3.NewStringSchema() + default: + // TODO: + // I've tested additionalProperties and it seems like properties - we will might have problems in the diff logic + // openapi3.MapProperty() + // openapi3.RefProperty() + // openapi3.RefSchema() + // openapi3.ComposedSchema() - discriminator? + return nil, fmt.Errorf("unexpected value type. value=%v, type=%T", value, value) + } + + return schema, nil +} + +func getStringSchema(value interface{}) (schema *openapi3.Schema) { + return openapi3.NewStringSchema().WithFormat(getStringFormat(value)) +} + +func getNumberSchema(value interface{}) (schema *openapi3.Schema) { + // https://swagger.io/docs/specification/data-models/data-types/#numbers + + // It is important to try first convert it to int + if _, err := value.(json.Number).Int64(); err != nil { + // if failed to convert to int it's a double + // TODO: we will set a 'double' and not a 'float' - is that ok? + schema = openapi3.NewFloat64Schema() + } else { + schema = openapi3.NewInt64Schema() + } + // TODO: Format + // openapi3.Int8Property() + // openapi3.Int16Property() + // openapi3.Int32Property() + // openapi3.Float64Property() + // openapi3.Float32Property() + return schema /*.WithExample(value)*/ +} + +func getObjectSchema(value interface{}) (schema *openapi3.Schema, err error) { + schema = openapi3.NewObjectSchema() + stringMapE, err := cast.ToStringMapE(value) + if err != nil { + return nil, fmt.Errorf("failed to cast to string map. value=%v: %w", value, err) + } + + for key, val := range stringMapE { + if s, err := getSchema(val); err != nil { + return nil, fmt.Errorf("failed to get schema from string map. key=%v, value=%v: %w", key, val, err) + } else { + schema = schema.WithProperty(escapeString(key), s) + } + } + + return schema, nil +} + +func escapeString(key string) string { + // need to escape double quotes if exists + if strings.Contains(key, "\"") { + key = strings.ReplaceAll(key, "\"", "\\\"") + } + return key +} + +func getArraySchema(value interface{}) (schema *openapi3.Schema, err error) { + sliceE, err := cast.ToSliceE(value) + if err != nil { + return nil, fmt.Errorf("failed to cast to slice. value=%v: %w", value, err) + } + + // in order to support mixed type array we will map all schemas by schema type + schemaTypeToSchema := make(map[string]*openapi3.Schema) + for i := range sliceE { + item, err := getSchema(sliceE[i]) + if err != nil { + return nil, fmt.Errorf("failed to get items schema from slice. value=%v: %w", sliceE[i], err) + } + if len(item.Type.Slice()) > 0 { + if _, ok := schemaTypeToSchema[item.Type.Slice()[0]]; !ok { + schemaTypeToSchema[item.Type.Slice()[0]] = item + } + } + } + + switch len(schemaTypeToSchema) { + case 0: + // array is empty, but we can't create an empty array property (Schemas with 'type: array', require a sibling 'items:' field) + // we will create string type items as a default value + schema = openapi3.NewArraySchema().WithItems(openapi3.NewStringSchema()) + case 1: + for _, s := range schemaTypeToSchema { + schema = openapi3.NewArraySchema().WithItems(s) + break + } + default: + // oneOf + // https://swagger.io/docs/specification/data-models/oneof-anyof-allof-not/ + var schemas []*openapi3.Schema + for _, s := range schemaTypeToSchema { + schemas = append(schemas, s) + } + schema = openapi3.NewOneOfSchema(schemas...) + } + + return schema, nil +} + +type HTTPInteractionData struct { + ReqBody, RespBody string + ReqHeaders, RespHeaders map[string]string + QueryParams url.Values + statusCode int +} + +func (h *HTTPInteractionData) getReqContentType() string { + return h.ReqHeaders[contentTypeHeaderName] +} + +func (h *HTTPInteractionData) getRespContentType() string { + return h.RespHeaders[contentTypeHeaderName] +} + +type OperationGeneratorConfig struct { + ResponseHeadersToIgnore []string + RequestHeadersToIgnore []string +} + +type OperationGenerator struct { + ResponseHeadersToIgnore map[string]struct{} + RequestHeadersToIgnore map[string]struct{} +} + +func NewOperationGenerator(config OperationGeneratorConfig) *OperationGenerator { + return &OperationGenerator{ + ResponseHeadersToIgnore: createHeadersToIgnore(config.ResponseHeadersToIgnore), + RequestHeadersToIgnore: createHeadersToIgnore(config.RequestHeadersToIgnore), + } +} + +// Note: SecuritySchemes might be updated. +func (o *OperationGenerator) GenerateSpecOperation(data *HTTPInteractionData, securitySchemes openapi3.SecuritySchemes) (*openapi3.Operation, error) { + operation := openapi3.NewOperation() + + if len(data.ReqBody) > 0 { + reqContentType := data.getReqContentType() + if reqContentType == "" { + log.Infof("Missing Content-Type header, ignoring request body. (%v)", data.ReqBody) + } else { + mediaType, mediaTypeParams, err := mime.ParseMediaType(reqContentType) + if err != nil { + return nil, fmt.Errorf("failed to parse request media type. Content-Type=%v: %w", reqContentType, err) + } + switch true { + case util.IsApplicationJSONMediaType(mediaType): + reqBodyJSON, err := gojsonschema.NewStringLoader(data.ReqBody).LoadJSON() + if err != nil { + return nil, fmt.Errorf("failed to load json from request body. body=%v: %w", data.ReqBody, err) + } + + reqSchema, err := getSchema(reqBodyJSON) + if err != nil { + return nil, fmt.Errorf("failed to get schema from request body. body=%v: %w", data.ReqBody, err) + } + + operationSetRequestBody(operation, openapi3.NewRequestBody().WithJSONSchema(reqSchema)) + case mediaType == mediaTypeApplicationForm: + operation, securitySchemes, err = handleApplicationFormURLEncodedBody(operation, securitySchemes, data.ReqBody) + if err != nil { + return nil, fmt.Errorf("failed to handle %s body: %v", mediaTypeApplicationForm, err) + } + case mediaType == mediaTypeMultipartFormData: + // Multipart requests combine one or more sets of data into a single body, separated by boundaries. + // You typically use these requests for file uploads and for transferring data of several types + // in a single request (for example, a file along with a JSON object). + // https://swagger.io/docs/specification/describing-request-body/multipart-requests/ + schema, err := getMultipartFormDataSchema(data.ReqBody, mediaTypeParams) + if err != nil { + return nil, fmt.Errorf("failed to get multipart form-data schema from request body. body=%v: %v", data.ReqBody, err) + } + operationSetRequestBody(operation, openapi3.NewRequestBody().WithFormDataSchema(schema)) + default: + log.Infof("Treating %v as default request content type (no schema)", reqContentType) + } + } + } + + for key, value := range data.ReqHeaders { + lowerKey := strings.ToLower(key) + if lowerKey == authorizationTypeHeaderName { + // https://datatracker.ietf.org/doc/html/rfc6750#section-2.1 + operation, securitySchemes = handleAuthReqHeader(operation, securitySchemes, value) + } else if APIKeyNames[lowerKey] { + schemeKey := APIKeyAuthSecuritySchemeKey + operation = addSecurity(operation, schemeKey) + securitySchemes = updateSecuritySchemes(securitySchemes, schemeKey, NewAPIKeySecuritySchemeInHeader(key)) + } else if lowerKey == cookieTypeHeaderName { + operation = o.addCookieParam(operation, value) + } else { + operation = o.addHeaderParam(operation, key, value) + } + } + + for key, values := range data.QueryParams { + lowerKey := strings.ToLower(key) + if lowerKey == AccessTokenParamKey { + // https://datatracker.ietf.org/doc/html/rfc6750#section-2.3 + operation, securitySchemes = handleAuthQueryParam(operation, securitySchemes, values) + } else if APIKeyNames[lowerKey] { + schemeKey := APIKeyAuthSecuritySchemeKey + operation = addSecurity(operation, schemeKey) + securitySchemes = updateSecuritySchemes(securitySchemes, schemeKey, NewAPIKeySecuritySchemeInQuery(key)) + } else { + operation = addQueryParam(operation, key, values) + } + } + + // https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.3.md#responseObject + // REQUIRED. A short description of the response. + response := openapi3.NewResponse().WithDescription("response") + if len(data.RespBody) > 0 { + respContentType := data.getRespContentType() + if respContentType == "" { + log.Infof("Missing Content-Type header, ignoring response body. (%v)", data.RespBody) + } else { + mediaType, _, err := mime.ParseMediaType(respContentType) + if err != nil { + return nil, fmt.Errorf("failed to parse response media type. Content-Type=%v: %w", respContentType, err) + } + switch true { + case util.IsApplicationJSONMediaType(mediaType): + respBodyJSON, err := gojsonschema.NewStringLoader(data.RespBody).LoadJSON() + if err != nil { + return nil, fmt.Errorf("failed to load json from response body. body=%v: %w", data.RespBody, err) + } + + respSchema, err := getSchema(respBodyJSON) + if err != nil { + return nil, fmt.Errorf("failed to get schema from response body. body=%v: %w", respBodyJSON, err) + } + + response = response.WithJSONSchema(respSchema) + default: + log.Infof("Treating %v as default response content type (no schema)", respContentType) + } + } + } + + for key, value := range data.RespHeaders { + response = o.addResponseHeader(response, key, value) + } + + operation.AddResponse(data.statusCode, response) + operation.AddResponse(0 /*"default"*/, openapi3.NewResponse().WithDescription("default")) + + return operation, nil +} + +func operationSetRequestBody(operation *openapi3.Operation, reqBody *openapi3.RequestBody) { + operation.RequestBody = &openapi3.RequestBodyRef{Value: reqBody} +} + +func CloneOperation(op *openapi3.Operation) (*openapi3.Operation, error) { + var out openapi3.Operation + + opB, err := json.Marshal(op) + if err != nil { + return nil, fmt.Errorf("failed to marshal operation (%+v): %v", op, err) + } + + if err := json.Unmarshal(opB, &out); err != nil { + return nil, fmt.Errorf("failed to unmarshal: %v", err) + } + + return &out, nil +} + +func getBearerAuthClaims(bearerToken string) (claims jwt.MapClaims, found bool) { + if len(bearerToken) == 0 { + log.Warnf("authZ token provided with no value.") + return nil, false + } + + // Parse the claims without validating (since we don't want to bother downloading a key) + parser := jwt.Parser{} + token, _, err := parser.ParseUnverified(bearerToken, jwt.MapClaims{}) + if err != nil { + log.Warnf("authZ token is not a JWT.") + return nil, false + } + + claims, ok := token.Claims.(jwt.MapClaims) + if !ok { + log.Infof("authZ token had unintelligble claims.") + return nil, false + } + + return claims, true +} + +func generateBearerAuthScheme(operation *openapi3.Operation, claims jwt.MapClaims, key string) (*openapi3.Operation, *openapi3.SecurityScheme) { + switch key { + case BearerAuthSecuritySchemeKey: + // https://swagger.io/docs/specification/authentication/bearer-authentication/ + return addSecurity(operation, key), openapi3.NewJWTSecurityScheme() + case OAuth2SecuritySchemeKey: + // https://swagger.io/docs/specification/authentication/oauth2/ + // we can't know the flow type (implicit, password, clientCredentials or authorizationCode) so we choose authorizationCode for now + scopes := getScopesFromJWTClaims(claims) + oAuth2SecurityScheme := NewOAuth2SecurityScheme(scopes) + return addSecurity(operation, key, scopes...), oAuth2SecurityScheme + default: + log.Warnf("Unsupported BearerAuth key: %v", key) + return operation, nil + } +} + +func getScopesFromJWTClaims(claims jwt.MapClaims) []string { + var scopes []string + if claims == nil { + return scopes + } + + if scope, ok := claims["scope"]; ok { + scopes = strings.Split(scope.(string), " ") + log.Debugf("found OAuth token scopes: %v", scopes) + } else { + log.Warnf("no scopes defined in this token") + } + return scopes +} + +func handleAuthQueryParam(operation *openapi3.Operation, securitySchemes openapi3.SecuritySchemes, values []string) (*openapi3.Operation, openapi3.SecuritySchemes) { + if len(values) > 1 { + // RFC 6750 does not prohibit multiple tokens, but we do not know whether + // they would be AND or OR so we just pick the latest. + log.Warnf("Found %v tokens in query parameters, using only the last", len(values)) + values = values[len(values)-1:] + } + + // Use scheme as security scheme name + securitySchemeKey := OAuth2SecuritySchemeKey + claims, _ := getBearerAuthClaims(values[0]) + + if hasSecurity(operation, securitySchemeKey) { + // RFC 6750 states multiple methods (form, uri query, header) cannot be used. + log.Errorf("OAuth tokens supplied with multiple methods, ignoring query param") + return operation, securitySchemes + } + + var scheme *openapi3.SecurityScheme + operation, scheme = generateBearerAuthScheme(operation, claims, securitySchemeKey) + if scheme != nil { + securitySchemes = updateSecuritySchemes(securitySchemes, securitySchemeKey, scheme) + } + return operation, securitySchemes +} + +func handleAuthReqHeader(operation *openapi3.Operation, securitySchemes openapi3.SecuritySchemes, value string) (*openapi3.Operation, openapi3.SecuritySchemes) { + if strings.HasPrefix(value, BasicAuthPrefix) { + // https://swagger.io/docs/specification/authentication/basic-authentication/ + // Use scheme as security scheme name + key := BasicAuthSecuritySchemeKey + operation = addSecurity(operation, key) + securitySchemes = updateSecuritySchemes(securitySchemes, key, NewBasicAuthSecurityScheme()) + } else if strings.HasPrefix(value, BearerAuthPrefix) { + // https://swagger.io/docs/specification/authentication/bearer-authentication/ + // https://datatracker.ietf.org/doc/html/rfc6750#section-2.1 + // Use scheme as security scheme name. For OAuth, we should consider checking + // supported scopes to allow multiple defs. + key := BearerAuthSecuritySchemeKey + claims, found := getBearerAuthClaims(strings.TrimPrefix(value, BearerAuthPrefix)) + if found { + key = OAuth2SecuritySchemeKey + } + + if hasSecurity(operation, key) { + // RFC 6750 states multiple methods (form, uri query, header) cannot be used. + log.Error("OAuth tokens supplied with multiple methods, ignoring header") + return operation, securitySchemes + } + + var scheme *openapi3.SecurityScheme + operation, scheme = generateBearerAuthScheme(operation, claims, key) + if scheme != nil { + securitySchemes = updateSecuritySchemes(securitySchemes, key, scheme) + } + } else { + log.Warnf("ignoring unknown authorization header value (%v)", value) + } + return operation, securitySchemes +} + +func addSecurity(op *openapi3.Operation, name string, scopes ...string) *openapi3.Operation { + // https://swagger.io/docs/specification/authentication/ + // We will treat multiple authentication types as an OR + // (Security schemes combined via OR are alternatives – any one can be used in the given context) + securityRequirement := openapi3.NewSecurityRequirement() + + if len(scopes) > 0 { + securityRequirement[name] = scopes + } else { + // We must use an empty array as the scopes, otherwise it will create invalid swagger + securityRequirement[name] = []string{} + } + + if op.Security == nil { + op.Security = openapi3.NewSecurityRequirements() + } + op.Security.With(securityRequirement) + + return op +} + +func hasSecurity(op *openapi3.Operation, name string) bool { + if op.Security == nil { + return false + } + + for _, securityScheme := range *op.Security { + if _, ok := securityScheme[name]; ok { + return true + } + } + return false +} diff --git a/speculator/pkg/apispec/operation_test.go b/speculator/pkg/apispec/operation_test.go new file mode 100644 index 0000000..55088f6 --- /dev/null +++ b/speculator/pkg/apispec/operation_test.go @@ -0,0 +1,605 @@ +package apispec + +import ( + "encoding/json" + "net/url" + "reflect" + "strings" + "testing" + "time" + + "github.com/getkin/kin-openapi/openapi3" + "github.com/golang-jwt/jwt/v5" + log "github.com/sirupsen/logrus" + "github.com/yudai/gojsondiff" +) + +var agentStatusBody = `{"active":true, +"certificateVersion":"86eb5278-676a-3b7c-b29d-4a57007dc7be", +"controllerInstanceInfo":{"replicaId":"portshift-agent-66fc77c848-tmmk8"}, +"policyAndAppVersion":1621477900361, +"statusCodes":["NO_METRICS_SERVER"], +"version":"1.147.1"}` + +var cvssBody = `{"cvss":[{"score":7.8,"vector":"AV:L/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H","version":"3"}]}` + +func generateDefaultOAuthToken(scopes []string) (string, string) { + mySigningKey := []byte("AllYourBase") + + var defaultOAuth2Claims jwt.Claims = OAuth2Claims{ + strings.Join(scopes, " "), + jwt.RegisteredClaims{ + ExpiresAt: jwt.NewNumericDate(time.Now().Add(24 * time.Hour)), + IssuedAt: jwt.NewNumericDate(time.Now()), + NotBefore: jwt.NewNumericDate(time.Now()), + Issuer: "test", + Subject: "somebody", + Audience: []string{"somebody_else"}, + }, + } + + token := jwt.NewWithClaims(jwt.SigningMethodHS256, defaultOAuth2Claims) + bearerToken, err := token.SignedString(mySigningKey) + if err != nil { + log.Errorf("Failed to create default OAuth2 Bearer Token: %v", err) + return bearerToken, "" + } + + oAuth2JSON := "" + encoded, err := json.Marshal(scopes) + if err != nil { + log.Errorf("Cannot encode token scopes: %v", scopes) + } else { + oAuth2JSON = string(encoded) + } + + return bearerToken, oAuth2JSON +} + +func generateQueryParams(t *testing.T, query string) url.Values { + t.Helper() + parseQuery, err := url.ParseQuery(query) + if err != nil { + t.Fatal(err) + } + return parseQuery +} + +func TestGenerateSpecOperation(t *testing.T) { + sd := openapi3.SecuritySchemes{} + opGen := CreateTestNewOperationGenerator() + operation, err := opGen.GenerateSpecOperation(&HTTPInteractionData{ + ReqBody: agentStatusBody, + RespBody: cvssBody, + ReqHeaders: map[string]string{ + "X-Request-ID": "77e1c83b-7bb0-437b-bc50-a7a58e5660ac", + "X-Float-Test": "12.2", + "X-Collection-Test": "a,b,c,d", + contentTypeHeaderName: mediaTypeApplicationJSON, + }, + RespHeaders: map[string]string{ + "X-RateLimit-Limit": "12", + "X-RateLimit-Reset": "2016-10-12T11:00:00Z", + contentTypeHeaderName: mediaTypeApplicationJSON, + }, + QueryParams: generateQueryParams(t, "offset=30&limit=10"), + statusCode: 200, + }, sd) + if err != nil { + t.Fatal(err) + } + + t.Log(marshal(operation)) + t.Log(marshal(sd)) +} + +func validateOperation(t *testing.T, got *openapi3.Operation, want string) bool { + t.Helper() + templateB, err := json.Marshal(got) + if err != nil { + t.Fatal(err) + } + + differ := gojsondiff.New() + diff, err := differ.Compare(templateB, []byte(want)) + if err != nil { + t.Fatal(err) + } + return diff.Modified() == false +} + +func TestGenerateSpecOperation1(t *testing.T) { + defaultOAuth2Scopes := []string{"admin", "write:pets"} + defaultOAuth2BearerToken, defaultOAuth2JSON := generateDefaultOAuthToken(defaultOAuth2Scopes) + defaultOAuthSecurityScheme := NewOAuth2SecurityScheme(defaultOAuth2Scopes) + defaultAPIKeyHeaderName := "" + for key := range APIKeyNames { + defaultAPIKeyHeaderName = key + break + } + type args struct { + data *HTTPInteractionData + } + opGen := CreateTestNewOperationGenerator() + tests := []struct { + name string + args args + want string + wantErr bool + expectedSd openapi3.SecuritySchemes + }{ + { + name: "Basic authorization req header", + args: args{ + data: &HTTPInteractionData{ + ReqBody: agentStatusBody, + RespBody: cvssBody, + ReqHeaders: map[string]string{ + contentTypeHeaderName: mediaTypeApplicationHalJSON, + authorizationTypeHeaderName: BasicAuthPrefix + "=token", + }, + RespHeaders: map[string]string{ + contentTypeHeaderName: mediaTypeApplicationHalJSON, + }, + statusCode: 200, + }, + }, + want: "{\"requestBody\":{\"content\":{\"application/json\":{\"schema\":{\"properties\":{\"active\":{\"type\":\"boolean\"},\"certificateVersion\":{\"format\":\"uuid\",\"type\":\"string\"},\"controllerInstanceInfo\":{\"properties\":{\"replicaId\":{\"type\":\"string\"}},\"type\":\"object\"},\"policyAndAppVersion\":{\"format\":\"int64\",\"type\":\"integer\"},\"statusCodes\":{\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"version\":{\"type\":\"string\"}},\"type\":\"object\"}}}},\"responses\":{\"200\":{\"content\":{\"application/json\":{\"schema\":{\"properties\":{\"cvss\":{\"items\":{\"properties\":{\"score\":{\"type\":\"number\"},\"vector\":{\"type\":\"string\"},\"version\":{\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"}}},\"description\":\"response\"},\"default\":{\"description\":\"default\"}},\"security\":[{\"BasicAuth\":[]}]}", + expectedSd: openapi3.SecuritySchemes{ + BasicAuthSecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewBasicAuthSecurityScheme()}, + }, + wantErr: false, + }, + { + name: "OAuth 2.0 authorization req header", + args: args{ + data: &HTTPInteractionData{ + ReqBody: agentStatusBody, + RespBody: cvssBody, + ReqHeaders: map[string]string{ + contentTypeHeaderName: mediaTypeApplicationJSON, + authorizationTypeHeaderName: BearerAuthPrefix + defaultOAuth2BearerToken, + }, + RespHeaders: map[string]string{ + contentTypeHeaderName: mediaTypeApplicationJSON, + }, + statusCode: 200, + }, + }, + want: "{\"requestBody\":{\"content\":{\"application/json\":{\"schema\":{\"properties\":{\"active\":{\"type\":\"boolean\"},\"certificateVersion\":{\"format\":\"uuid\",\"type\":\"string\"},\"controllerInstanceInfo\":{\"properties\":{\"replicaId\":{\"type\":\"string\"}},\"type\":\"object\"},\"policyAndAppVersion\":{\"format\":\"int64\",\"type\":\"integer\"},\"statusCodes\":{\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"version\":{\"type\":\"string\"}},\"type\":\"object\"}}}},\"responses\":{\"200\":{\"content\":{\"application/json\":{\"schema\":{\"properties\":{\"cvss\":{\"items\":{\"properties\":{\"score\":{\"type\":\"number\"},\"vector\":{\"type\":\"string\"},\"version\":{\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"}}},\"description\":\"response\"},\"default\":{\"description\":\"default\"}},\"security\":[{\"OAuth2\":[\"admin\",\"write:pets\"]}]}", + expectedSd: openapi3.SecuritySchemes{ + OAuth2SecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: defaultOAuthSecurityScheme}, + }, + wantErr: false, + }, + { + name: "OAuth 2.0 URI Query Parameter", + args: args{ + data: &HTTPInteractionData{ + ReqBody: agentStatusBody, + RespBody: cvssBody, + ReqHeaders: map[string]string{ + contentTypeHeaderName: mediaTypeApplicationJSON, + }, + RespHeaders: map[string]string{ + contentTypeHeaderName: mediaTypeApplicationJSON, + }, + QueryParams: generateQueryParams(t, AccessTokenParamKey+"="+defaultOAuth2BearerToken), + statusCode: 200, + }, + }, + want: "{\"requestBody\":{\"content\":{\"application/json\":{\"schema\":{\"properties\":{\"active\":{\"type\":\"boolean\"},\"certificateVersion\":{\"format\":\"uuid\",\"type\":\"string\"},\"controllerInstanceInfo\":{\"properties\":{\"replicaId\":{\"type\":\"string\"}},\"type\":\"object\"},\"policyAndAppVersion\":{\"format\":\"int64\",\"type\":\"integer\"},\"statusCodes\":{\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"version\":{\"type\":\"string\"}},\"type\":\"object\"}}}},\"responses\":{\"200\":{\"content\":{\"application/json\":{\"schema\":{\"properties\":{\"cvss\":{\"items\":{\"properties\":{\"score\":{\"type\":\"number\"},\"vector\":{\"type\":\"string\"},\"version\":{\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"}}},\"description\":\"response\"},\"default\":{\"description\":\"default\"}},\"security\":[{\"OAuth2\":[\"admin\",\"write:pets\"]}]}", + expectedSd: openapi3.SecuritySchemes{ + OAuth2SecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: defaultOAuthSecurityScheme}, + }, + wantErr: false, + }, + { + name: "OAuth 2.0 Form-Encoded Body Parameter", + args: args{ + data: &HTTPInteractionData{ + ReqBody: AccessTokenParamKey + "=" + defaultOAuth2BearerToken + "&key=val", + RespBody: cvssBody, + ReqHeaders: map[string]string{ + contentTypeHeaderName: mediaTypeApplicationForm, + }, + RespHeaders: map[string]string{ + contentTypeHeaderName: mediaTypeApplicationJSON, + }, + statusCode: 200, + }, + }, + want: "{\"requestBody\":{\"content\":{\"application/x-www-form-urlencoded\":{\"schema\":{\"properties\":{\"key\":{\"type\":\"string\"}},\"type\":\"object\"}}}},\"responses\":{\"200\":{\"content\":{\"application/json\":{\"schema\":{\"properties\":{\"cvss\":{\"items\":{\"properties\":{\"score\":{\"type\":\"number\"},\"vector\":{\"type\":\"string\"},\"version\":{\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"}}},\"description\":\"response\"},\"default\":{\"description\":\"default\"}},\"security\":[{\"OAuth2\":[\"admin\",\"write:pets\"]}]}", + expectedSd: openapi3.SecuritySchemes{ + OAuth2SecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: defaultOAuthSecurityScheme}, + }, + wantErr: false, + }, + { + name: "OAuth 2.0 Multiple parameters: Authorization Req Header and URI Query Parameter", + args: args{ + data: &HTTPInteractionData{ + ReqBody: agentStatusBody, + RespBody: cvssBody, + ReqHeaders: map[string]string{ + contentTypeHeaderName: mediaTypeApplicationJSON, + authorizationTypeHeaderName: BearerAuthPrefix + defaultOAuth2BearerToken, + }, + RespHeaders: map[string]string{ + contentTypeHeaderName: mediaTypeApplicationJSON, + }, + QueryParams: generateQueryParams(t, AccessTokenParamKey+"=bogus.key.material"), + statusCode: 200, + }, + }, + want: "{\"requestBody\":{\"content\":{\"application/json\":{\"schema\":{\"properties\":{\"active\":{\"type\":\"boolean\"},\"certificateVersion\":{\"format\":\"uuid\",\"type\":\"string\"},\"controllerInstanceInfo\":{\"properties\":{\"replicaId\":{\"type\":\"string\"}},\"type\":\"object\"},\"policyAndAppVersion\":{\"format\":\"int64\",\"type\":\"integer\"},\"statusCodes\":{\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"version\":{\"type\":\"string\"}},\"type\":\"object\"}}}},\"responses\":{\"200\":{\"content\":{\"application/json\":{\"schema\":{\"properties\":{\"cvss\":{\"items\":{\"properties\":{\"score\":{\"type\":\"number\"},\"vector\":{\"type\":\"string\"},\"version\":{\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"}}},\"description\":\"response\"},\"default\":{\"description\":\"default\"}},\"security\":[{\"OAuth2\":" + defaultOAuth2JSON + "}]}", + expectedSd: openapi3.SecuritySchemes{ + // Note: Auth Header will be used before Query Parameter is ignored. + OAuth2SecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: defaultOAuthSecurityScheme}, + }, + wantErr: false, + }, + { + name: "API Key in header", + args: args{ + data: &HTTPInteractionData{ + ReqBody: agentStatusBody, + RespBody: cvssBody, + ReqHeaders: map[string]string{ + contentTypeHeaderName: mediaTypeApplicationJSON, + defaultAPIKeyHeaderName: "mybogusapikey", + }, + RespHeaders: map[string]string{ + contentTypeHeaderName: mediaTypeApplicationJSON, + }, + statusCode: 200, + }, + }, + want: "{\"requestBody\":{\"content\":{\"application/json\":{\"schema\":{\"properties\":{\"active\":{\"type\":\"boolean\"},\"certificateVersion\":{\"format\":\"uuid\",\"type\":\"string\"},\"controllerInstanceInfo\":{\"properties\":{\"replicaId\":{\"type\":\"string\"}},\"type\":\"object\"},\"policyAndAppVersion\":{\"format\":\"int64\",\"type\":\"integer\"},\"statusCodes\":{\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"version\":{\"type\":\"string\"}},\"type\":\"object\"}}}},\"responses\":{\"200\":{\"content\":{\"application/json\":{\"schema\":{\"properties\":{\"cvss\":{\"items\":{\"properties\":{\"score\":{\"type\":\"number\"},\"vector\":{\"type\":\"string\"},\"version\":{\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"}}},\"description\":\"response\"},\"default\":{\"description\":\"default\"}},\"security\":[{\"ApiKeyAuth\":[]}]}", + expectedSd: openapi3.SecuritySchemes{ + APIKeyAuthSecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewAPIKeySecuritySchemeInHeader(defaultAPIKeyHeaderName)}, + }, + wantErr: false, + }, + { + name: "API Key URI Query Parameter", + args: args{ + data: &HTTPInteractionData{ + ReqBody: agentStatusBody, + RespBody: cvssBody, + ReqHeaders: map[string]string{ + contentTypeHeaderName: mediaTypeApplicationJSON, + }, + RespHeaders: map[string]string{ + contentTypeHeaderName: mediaTypeApplicationJSON, + }, + QueryParams: generateQueryParams(t, defaultAPIKeyHeaderName+"=mybogusapikey"), + statusCode: 200, + }, + }, + want: "{\"requestBody\":{\"content\":{\"application/json\":{\"schema\":{\"properties\":{\"active\":{\"type\":\"boolean\"},\"certificateVersion\":{\"format\":\"uuid\",\"type\":\"string\"},\"controllerInstanceInfo\":{\"properties\":{\"replicaId\":{\"type\":\"string\"}},\"type\":\"object\"},\"policyAndAppVersion\":{\"format\":\"int64\",\"type\":\"integer\"},\"statusCodes\":{\"items\":{\"type\":\"string\"},\"type\":\"array\"},\"version\":{\"type\":\"string\"}},\"type\":\"object\"}}}},\"responses\":{\"200\":{\"content\":{\"application/json\":{\"schema\":{\"properties\":{\"cvss\":{\"items\":{\"properties\":{\"score\":{\"type\":\"number\"},\"vector\":{\"type\":\"string\"},\"version\":{\"type\":\"string\"}},\"type\":\"object\"},\"type\":\"array\"}},\"type\":\"object\"}}},\"description\":\"response\"},\"default\":{\"description\":\"default\"}},\"security\":[{\"ApiKeyAuth\":[]}]}", + expectedSd: openapi3.SecuritySchemes{ + APIKeyAuthSecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewAPIKeySecuritySchemeInQuery(defaultAPIKeyHeaderName)}, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sd := openapi3.SecuritySchemes{} + got, err := opGen.GenerateSpecOperation(tt.args.data, sd) + if (err != nil) != tt.wantErr { + t.Errorf("GenerateSpecOperation() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if !validateOperation(t, got, tt.want) { + t.Errorf("GenerateSpecOperation() got = %v, want %v", marshal(got), marshal(tt.want)) + } + + assertEqual(t, sd, tt.expectedSd) + }) + } +} + +func Test_getStringSchema(t *testing.T) { + type args struct { + value interface{} + } + tests := []struct { + name string + args args + wantSchema *openapi3.Schema + }{ + { + name: "date", + args: args{ + value: "2017-07-21", + }, + wantSchema: openapi3.NewStringSchema().WithFormat("date"), + }, + { + name: "time", + args: args{ + value: "17:32:28", + }, + wantSchema: openapi3.NewStringSchema().WithFormat("time"), + }, + { + name: "date-time", + args: args{ + value: "2017-07-21T17:32:28Z", + }, + wantSchema: openapi3.NewDateTimeSchema(), + }, + { + name: "email", + args: args{ + value: "test@securecn.com", + }, + wantSchema: openapi3.NewStringSchema().WithFormat("email"), + }, + { + name: "ipv4", + args: args{ + value: "1.1.1.1", + }, + wantSchema: openapi3.NewStringSchema().WithFormat("ipv4"), + }, + { + name: "ipv6", + args: args{ + value: "2001:0db8:85a3:0000:0000:8a2e:0370:7334", + }, + wantSchema: openapi3.NewStringSchema().WithFormat("ipv6"), + }, + { + name: "uuid", + args: args{ + value: "123e4567-e89b-12d3-a456-426614174000", + }, + wantSchema: openapi3.NewStringSchema().WithFormat("uuid"), + }, + { + name: "json-pointer", + args: args{ + value: "/k%22l", + }, + wantSchema: openapi3.NewStringSchema().WithFormat("json-pointer"), + }, + { + name: "string", + args: args{ + value: "it is very hard to get a simple string", + }, + wantSchema: openapi3.NewStringSchema(), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if gotSchema := getStringSchema(tt.args.value); !reflect.DeepEqual(gotSchema, tt.wantSchema) { + t.Errorf("getStringSchema() = %v, want %v", gotSchema, tt.wantSchema) + } + }) + } +} + +func Test_getNumberSchema(t *testing.T) { + type args struct { + value interface{} + } + tests := []struct { + name string + args args + wantSchema *openapi3.Schema + }{ + { + name: "int", + args: args{ + value: json.Number("85"), + }, + wantSchema: openapi3.NewInt64Schema(), + }, + { + name: "float", + args: args{ + value: json.Number("85.1"), + }, + wantSchema: openapi3.NewFloat64Schema(), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if gotSchema := getNumberSchema(tt.args.value); !reflect.DeepEqual(gotSchema, tt.wantSchema) { + t.Errorf("getNumberSchema() = %v, want %v", gotSchema, tt.wantSchema) + } + }) + } +} + +func Test_escapeString(t *testing.T) { + type args struct { + key string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "nothing to strip", + args: args{ + key: "key", + }, + want: "key", + }, + { + name: "escape double quotes", + args: args{ + key: "{\"key1\":\"value1\", \"key2\":\"value2\"}", + }, + want: "{\\\"key1\\\":\\\"value1\\\", \\\"key2\\\":\\\"value2\\\"}", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := escapeString(tt.args.key); got != tt.want { + t.Errorf("stripKey() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestCloneOperation(t *testing.T) { + type args struct { + op *openapi3.Operation + } + tests := []struct { + name string + args args + want *openapi3.Operation + wantErr bool + }{ + { + name: "sanity", + args: args{ + op: createTestOperation(). + WithParameter(openapi3.NewHeaderParameter("header")). + WithResponse(200, openapi3.NewResponse().WithDescription("keep"). + WithJSONSchemaRef(openapi3.NewSchemaRef("", + openapi3.NewObjectSchema().WithProperty("test", openapi3.NewStringSchema())))).Op, + }, + want: createTestOperation(). + WithParameter(openapi3.NewHeaderParameter("header")). + WithResponse(200, openapi3.NewResponse().WithDescription("keep"). + WithJSONSchemaRef(openapi3.NewSchemaRef("", + openapi3.NewObjectSchema().WithProperty("test", openapi3.NewStringSchema())))).Op, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := CloneOperation(tt.args.op) + if (err != nil) != tt.wantErr { + t.Errorf("CloneOperation() error = %v, wantErr %v", err, tt.wantErr) + return + } + assertEqual(t, got, tt.want) + if got != nil { + got.Responses = nil + if tt.args.op.Responses == nil { + t.Errorf("CloneOperation() original object should not have been changed") + return + } + } + }) + } +} + +func Test_handleAuthReqHeader(t *testing.T) { + type args struct { + operation *openapi3.Operation + securitySchemes openapi3.SecuritySchemes + value string + } + defaultOAuth2Scopes := []string{"superman", "write:novel"} + defaultOAuth2BearerToken, _ := generateDefaultOAuthToken(defaultOAuth2Scopes) + defaultOAuthSecurityScheme := NewOAuth2SecurityScheme(defaultOAuth2Scopes) + tests := []struct { + name string + args args + wantOp *openapi3.Operation + wantSd openapi3.SecuritySchemes + }{ + { + name: "BearerAuthPrefix", + args: args{ + operation: openapi3.NewOperation(), + securitySchemes: openapi3.SecuritySchemes{}, + value: BearerAuthPrefix + defaultOAuth2BearerToken, + }, + wantOp: createTestOperation().WithSecurityRequirement(map[string][]string{OAuth2SecuritySchemeKey: defaultOAuth2Scopes}).Op, + wantSd: openapi3.SecuritySchemes{ + OAuth2SecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: defaultOAuthSecurityScheme}, + }, + }, + { + name: "BasicAuthPrefix", + args: args{ + operation: openapi3.NewOperation(), + securitySchemes: openapi3.SecuritySchemes{}, + value: BasicAuthPrefix + "token", + }, + wantOp: createTestOperation().WithSecurityRequirement(map[string][]string{BasicAuthSecuritySchemeKey: {}}).Op, + wantSd: openapi3.SecuritySchemes{ + BasicAuthSecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewBasicAuthSecurityScheme()}, + }, + }, + { + name: "ignoring unknown authorization header value", + args: args{ + operation: openapi3.NewOperation(), + securitySchemes: openapi3.SecuritySchemes{}, + value: "invalid token", + }, + wantOp: openapi3.NewOperation(), + wantSd: openapi3.SecuritySchemes{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := handleAuthReqHeader(tt.args.operation, tt.args.securitySchemes, tt.args.value) + if !reflect.DeepEqual(got, tt.wantOp) { + t.Errorf("handleAuthReqHeader() got = %v, want %v", got, tt.wantOp) + } + if !reflect.DeepEqual(got1, tt.wantSd) { + t.Errorf("handleAuthReqHeader() got1 = %v, want %v", got1, tt.wantSd) + } + }) + } +} + +func Test_getScopesFromJWTClaims(t *testing.T) { + type args struct { + claims jwt.MapClaims + } + tests := []struct { + name string + args args + want []string + }{ + { + name: "nil claims - expected nil scopes", + args: args{ + claims: nil, + }, + want: nil, + }, + { + name: "no scopes defined - expected nil scopes", + args: args{ + claims: jwt.MapClaims{ + "no-scopes": "123", + }, + }, + want: nil, + }, + { + name: "no scopes defined - expected nil scopes", + args: args{ + claims: jwt.MapClaims{ + "no-scope": "123", + "scope": "scope1 scope2 scope3", + }, + }, + want: []string{"scope1", "scope2", "scope3"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getScopesFromJWTClaims(tt.args.claims); !reflect.DeepEqual(got, tt.want) { + t.Errorf("getScopesFromJWTClaims() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/speculator/pkg/apispec/path_item.go b/speculator/pkg/apispec/path_item.go new file mode 100644 index 0000000..3067b3f --- /dev/null +++ b/speculator/pkg/apispec/path_item.go @@ -0,0 +1,76 @@ +package apispec + +import ( + "net/http" + + "github.com/getkin/kin-openapi/openapi3" +) + +func MergePathItems(dst, src *openapi3.PathItem) *openapi3.PathItem { + dst.Get, _ = mergeOperation(dst.Get, src.Get) + dst.Put, _ = mergeOperation(dst.Put, src.Put) + dst.Post, _ = mergeOperation(dst.Post, src.Post) + dst.Delete, _ = mergeOperation(dst.Delete, src.Delete) + dst.Options, _ = mergeOperation(dst.Options, src.Options) + dst.Head, _ = mergeOperation(dst.Head, src.Head) + dst.Patch, _ = mergeOperation(dst.Patch, src.Patch) + + // TODO what about merging parameters? + + return dst +} + +func CopyPathItemWithNewOperation(item *openapi3.PathItem, method string, operation *openapi3.Operation) *openapi3.PathItem { + // TODO - do we want to do : ret = *item? + ret := openapi3.PathItem{} + ret.Get = item.Get + ret.Put = item.Put + ret.Patch = item.Patch + ret.Post = item.Post + ret.Head = item.Head + ret.Delete = item.Delete + ret.Options = item.Options + ret.Parameters = item.Parameters + + AddOperationToPathItem(&ret, method, operation) + return &ret +} + +func GetOperationFromPathItem(item *openapi3.PathItem, method string) *openapi3.Operation { + switch method { + case http.MethodGet: + return item.Get + case http.MethodDelete: + return item.Delete + case http.MethodOptions: + return item.Options + case http.MethodPatch: + return item.Patch + case http.MethodHead: + return item.Head + case http.MethodPost: + return item.Post + case http.MethodPut: + return item.Put + } + return nil +} + +func AddOperationToPathItem(item *openapi3.PathItem, method string, operation *openapi3.Operation) { + switch method { + case http.MethodGet: + item.Get = operation + case http.MethodDelete: + item.Delete = operation + case http.MethodOptions: + item.Options = operation + case http.MethodPatch: + item.Patch = operation + case http.MethodHead: + item.Head = operation + case http.MethodPost: + item.Post = operation + case http.MethodPut: + item.Put = operation + } +} diff --git a/speculator/pkg/apispec/path_params.go b/speculator/pkg/apispec/path_params.go new file mode 100644 index 0000000..a865dd2 --- /dev/null +++ b/speculator/pkg/apispec/path_params.go @@ -0,0 +1,160 @@ +package apispec + +import ( + "fmt" + "regexp" + "strings" + "unicode" + + "github.com/getkin/kin-openapi/openapi3" + "github.com/gofrs/uuid" +) + +type PathParam struct { + *openapi3.Parameter +} + +func generateParamName(i int) string { + return fmt.Sprintf("param%v", i) +} + +var digitCheck = regexp.MustCompile(`^[0-9]+$`) + +func createParameterizedPath(path string) string { + var ParameterizedPathParts []string + paramCount := 0 + pathParts := strings.Split(path, "/") + + for _, part := range pathParts { + // if part is a suspect param, replace it with a param name, otherwise do nothing + if isSuspectPathParam(part) { + paramCount++ + paramName := generateParamName(paramCount) + ParameterizedPathParts = append(ParameterizedPathParts, "{"+paramName+"}") + } else { + ParameterizedPathParts = append(ParameterizedPathParts, part) + } + } + + parameterizedPath := strings.Join(ParameterizedPathParts, "/") + + return parameterizedPath +} + +type paramFormat string + +const ( + paramFormatUnset paramFormat = "paramFormatUnset" + paramFormatNumber paramFormat = "paramFormatNumber" + paramFormatUUID paramFormat = "paramFormatUUID" + paramFormatMixed paramFormat = "paramFormatMixed" +) + +// /api/1/foo, api/2/foo and index 1 will return: +// []string{1, 2}. +func getOnlyIndexedPartFromPaths(paths map[string]bool, i int) []string { + var ret []string + for path := range paths { + path = strings.TrimPrefix(path, "/") + splt := strings.Split(path, "/") + if len(splt) <= i { + continue + } + ret = append(ret, splt[i]) + } + return ret +} + +// If all params in paramList can be guessed as same schema, this schema will be returned, otherwise, +// if there is a couple of formats, string schema with no format will be returned. +func getParamSchema(paramsList []string) *openapi3.Schema { + parameterFormat := paramFormatUnset + + for _, pathPart := range paramsList { + if isNumber(pathPart) { + // in case there is a conflict, we will return string as the type and empty format + if parameterFormat != paramFormatNumber && parameterFormat != paramFormatUnset { + return openapi3.NewStringSchema() + } + parameterFormat = paramFormatNumber + continue + } + if isUUID(pathPart) { + if parameterFormat != paramFormatUUID && parameterFormat != paramFormatUnset { + return openapi3.NewStringSchema() + } + parameterFormat = paramFormatUUID + continue + } + if isMixed(pathPart) { + if parameterFormat != paramFormatMixed && parameterFormat != paramFormatUnset { + return openapi3.NewStringSchema() + } + parameterFormat = paramFormatMixed + } + } + + switch parameterFormat { + case paramFormatMixed: + return openapi3.NewStringSchema() + case paramFormatUUID: + return openapi3.NewUUIDSchema() + case paramFormatNumber: + return openapi3.NewInt64Schema() + case paramFormatUnset: + return openapi3.NewStringSchema() + } + + return openapi3.NewStringSchema() +} + +func isSuspectPathParam(pathPart string) bool { + if isNumber(pathPart) { + return true + } + if isUUID(pathPart) { + return true + } + if isMixed(pathPart) { + return true + } + return false +} + +func isNumber(pathPart string) bool { + return digitCheck.MatchString(pathPart) +} + +func isUUID(pathPart string) bool { + _, err := uuid.FromString(pathPart) + return err == nil +} + +// Check if a path part that is mixed from digits and chars can be considered as parameter following hard-coded heuristics. +// Temporary, we'll consider strings as parameters that are at least 8 chars longs and has at least 3 digits. +func isMixed(pathPart string) bool { + const maxLen = 8 + const minDigitsLen = 2 + + if len(pathPart) < maxLen { + return false + } + + return countDigitsInString(pathPart) > minDigitsLen +} + +func countDigitsInString(s string) int { + count := 0 + for _, c := range s { + if unicode.IsNumber(c) { + count++ + } + } + return count +} + +func createPathParam(name string, schema *openapi3.Schema) *PathParam { + return &PathParam{ + Parameter: openapi3.NewPathParameter(name).WithSchema(schema), + } +} diff --git a/speculator/pkg/apispec/path_params_test.go b/speculator/pkg/apispec/path_params_test.go new file mode 100644 index 0000000..cf8fa3b --- /dev/null +++ b/speculator/pkg/apispec/path_params_test.go @@ -0,0 +1,294 @@ +package apispec + +import ( + "reflect" + "sort" + "testing" + + "github.com/getkin/kin-openapi/openapi3" +) + +func Test_createParameterizedPath(t *testing.T) { + type args struct { + path string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "no suspect params", + args: args{ + path: "/api/user/hello", + }, + want: "/api/user/hello", + }, + { + name: "1 suspect param", + args: args{ + path: "/api/123/hello", + }, + want: "/api/{param1}/hello", + }, + { + name: "2 suspect param", + args: args{ + path: "/api/123/hello/234", + }, + want: "/api/{param1}/hello/{param2}", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := createParameterizedPath(tt.args.path); got != tt.want { + t.Errorf("createParameterizedPath() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_isSuspectPathParam(t *testing.T) { + type args struct { + pathPart string + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "number", + args: args{ + pathPart: "1234", + }, + want: true, + }, + { + name: "big number", + args: args{ + pathPart: "123456789001234567890023456789", + }, + want: true, + }, + { + name: "uuid", + args: args{ + pathPart: "3d9f2779-264f-4930-9196-e60c8a3610d2", + }, + want: true, + }, + { + name: "mixed type - numbers are more than 20%", + args: args{ + pathPart: "abcdefghij123", + }, + want: true, + }, + { + name: "mixed type - numbers are less than 20%", + args: args{ + pathPart: "abcdefghijk12", + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isSuspectPathParam(tt.args.pathPart); got != tt.want { + t.Errorf("isSuspectPathParam() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_countDigitsInString(t *testing.T) { + type args struct { + s string + } + tests := []struct { + name string + args args + want int + }{ + { + name: "4", + args: args{ + s: "abcdefg1234hijk", + }, + want: 4, + }, + { + name: "0", + args: args{ + s: "abcdefghijk", + }, + want: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := countDigitsInString(tt.args.s); got != tt.want { + t.Errorf("countDigitsInString() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getOnlyIndexedPartFromPaths(t *testing.T) { + type args struct { + paths map[string]bool + i int + } + tests := []struct { + name string + args args + want []string + }{ + { + name: "2 numbers", + args: args{ + paths: map[string]bool{ + "/api/1/foo": true, + "/api/2/foo": true, + }, + i: 1, + }, + want: []string{"1", "2"}, + }, + { + name: "number and string", + args: args{ + paths: map[string]bool{ + "/api/1/foo": true, + "/api/foo/2": true, + }, + i: 1, + }, + want: []string{"1", "foo"}, + }, + { + name: "get first part", + args: args{ + paths: map[string]bool{ + "/api/1/foo": true, + "/api/2/foo": true, + }, + i: 0, + }, + want: []string{"api", "api"}, + }, + { + name: "get last part", + args: args{ + paths: map[string]bool{ + "/api/1/foo": true, + "/api/2/foo": true, + }, + i: 2, + }, + want: []string{"foo", "foo"}, + }, + { + name: "index is bigger than paths len", + args: args{ + paths: map[string]bool{ + "/api/1/foo": true, + "/api/2/foo": true, + }, + i: 3, + }, + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := getOnlyIndexedPartFromPaths(tt.args.paths, tt.args.i) + sort.Slice(got, func(i, j int) bool { + return got[i] < got[j] + }) + sort.Slice(tt.want, func(i, j int) bool { + return tt.want[i] < tt.want[j] + }) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("getOnlyIndexedPartFromPaths() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getParamTypeAndFormat(t *testing.T) { + type args struct { + paramsList []string + } + tests := []struct { + name string + args args + want *openapi3.Schema + }{ + { + name: "mixed", + args: args{ + paramsList: []string{"str", "1234", "77e1c83b-7bb0-437b-bc50-a7a58e5660ac"}, + }, + want: openapi3.NewStringSchema(), + }, + { + name: "uuid", + args: args{ + paramsList: []string{"77e1c83b-7bb0-437b-bc50-a7a58e5660a3", "77e1c83b-7bb0-437b-bc50-a7a58e5660a8", "77e1c83b-7bb0-437b-bc50-a7a58e5660ac"}, + }, + want: openapi3.NewUUIDSchema(), + }, + { + name: "number", + args: args{ + paramsList: []string{"7776", "78", "123"}, + }, + want: openapi3.NewInt64Schema(), + }, + { + name: "string", + args: args{ + paramsList: []string{"strone", "strtwo", "strthree"}, + }, + want: openapi3.NewStringSchema(), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + schema := getParamSchema(tt.args.paramsList) + assertEqual(t, schema, tt.want) + }) + } +} + +func Test_createPathParam(t *testing.T) { + type args struct { + name string + schema *openapi3.Schema + } + tests := []struct { + name string + args args + want *PathParam + }{ + { + name: "create", + args: args{ + name: "param1", + schema: openapi3.NewUUIDSchema(), + }, + want: &PathParam{ + Parameter: openapi3.NewPathParameter("param1").WithSchema(openapi3.NewUUIDSchema()), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := createPathParam(tt.args.name, tt.args.schema); !reflect.DeepEqual(got, tt.want) { + t.Errorf("createPathParam() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/speculator/pkg/apispec/provided_spec.go b/speculator/pkg/apispec/provided_spec.go new file mode 100644 index 0000000..eeab112 --- /dev/null +++ b/speculator/pkg/apispec/provided_spec.go @@ -0,0 +1,354 @@ +package apispec + +import ( + "errors" + "fmt" + "strings" + + "github.com/getkin/kin-openapi/openapi3" + "github.com/ghodss/yaml" + log "github.com/sirupsen/logrus" + + "github.com/5gsec/sentryflow/speculator/pkg/pathtrie" +) + +// Sentinel errors for spec version issues. +var ( + ErrUnknownSpecVersion = errors.New("unknown spec version") + ErrUnsupportedSpecVersion = errors.New("unsupported spec version") +) + +type ProvidedSpec struct { + Doc *openapi3.T + OriginalSpecVersion OASVersion +} + +func (s *Spec) LoadProvidedSpec(providedSpec []byte, pathToPathID map[string]string) error { + doc, oasVersion, err := LoadAndValidateRawJSONSpec(providedSpec) + if err != nil { + return fmt.Errorf("failed to load and validate spec: %w", err) + } + + if s.ProvidedSpec == nil { + s.ProvidedSpec = &ProvidedSpec{} + } + // will save doc without refs for proper diff logic + s.ProvidedSpec.Doc = clearRefFromDoc(doc) + s.ProvidedSpec.OriginalSpecVersion = oasVersion + log.Debugf("Setting provided spec version %q", s.ProvidedSpec.GetSpecVersion()) + + // path trie need to be repopulated from start on each new spec + s.ProvidedPathTrie = pathtrie.New() + for path := range s.ProvidedSpec.Doc.Paths.Map() { + if pathID, ok := pathToPathID[path]; ok { + s.ProvidedPathTrie.Insert(path, pathID) + } + } + + return nil +} + +func LoadAndValidateRawJSONSpec(spec []byte) (*openapi3.T, OASVersion, error) { + // Convert YAML to JSON. Since JSON is a subset of YAML, passing JSON through + // this method should be a no-op. + jsonSpec, err := yaml.YAMLToJSON(spec) + if err != nil { + return nil, Unknown, fmt.Errorf("failed to convert provided spec into json: %s. %w", spec, err) + } + + oasVersion, err := GetJSONSpecVersion(jsonSpec) + if err != nil { + return nil, Unknown, fmt.Errorf("failed to get spec version: %s. %w", jsonSpec, err) + } + + var doc *openapi3.T + switch oasVersion { + case OASv2: + log.Debugf("OASv2 spec provided") + if doc, err = LoadAndValidateRawJSONSpecV3FromV2(jsonSpec); err != nil { + log.Errorf("provided spec is not valid OpenAPI 2.0: %s. %v", jsonSpec, err) + return nil, Unknown, fmt.Errorf("provided spec is not valid OpenAPI 2.0: %w", err) + } + case OASv3: + log.Debugf("OASv3 spec provided") + if doc, err = LoadAndValidateRawJSONSpecV3(jsonSpec); err != nil { + log.Errorf("provided spec is not valid OpenAPI 3.0: %s. %v", jsonSpec, err) + return nil, Unknown, fmt.Errorf("provided spec is not valid OpenAPI 3.0: %w", err) + } + case Unknown: + return nil, Unknown, fmt.Errorf("%w (%v)", ErrUnknownSpecVersion, oasVersion) + default: + return nil, Unknown, fmt.Errorf("%w (%v)", ErrUnsupportedSpecVersion, oasVersion) + } + + return doc, oasVersion, nil +} + +func (p *ProvidedSpec) GetPathItem(path string) *openapi3.PathItem { + return p.Doc.Paths.Find(path) +} + +func (p *ProvidedSpec) GetSpecVersion() OASVersion { + return p.OriginalSpecVersion +} + +func (p *ProvidedSpec) GetBasePath() string { + for _, server := range p.Doc.Servers { + if server.URL == "" || server.URL == "/" { + continue + } + + // strip scheme if exits + urlNoScheme := server.URL + schemeSplittedURL := strings.Split(server.URL, "://") + if len(schemeSplittedURL) > 1 { + urlNoScheme = schemeSplittedURL[1] + } + + // get path + var path string + splittedURLNoScheme := strings.SplitN(urlNoScheme, "/", 2) // nolint:gomnd + if len(splittedURLNoScheme) > 1 { + path = splittedURLNoScheme[1] + } + if path == "" { + continue + } + + return "/" + path + } + + return "" +} + +func clearRefFromDoc(doc *openapi3.T) *openapi3.T { + if doc == nil { + return doc + } + + for path, item := range doc.Paths.Map() { + doc.Paths.Set(path, clearRefFromPathItem(item)) + } + + return doc +} + +func clearRefFromPathItem(item *openapi3.PathItem) *openapi3.PathItem { + if item == nil { + return item + } + + for method, operation := range item.Operations() { + item.SetOperation(method, clearRefFromOperation(operation)) + } + + item.Parameters = clearRefFromParameters(item.Parameters) + + item.Ref = "" + + return item +} + +func clearRefFromParameters(parameters openapi3.Parameters) openapi3.Parameters { + if len(parameters) == 0 { + return parameters + } + + retParameters := make(openapi3.Parameters, len(parameters)) + for i, parameterRef := range parameters { + retParameters[i] = clearRefFromParameterRef(parameterRef) + } + + return retParameters +} + +func clearRefFromOperation(operation *openapi3.Operation) *openapi3.Operation { + if operation == nil { + return operation + } + + operation.Parameters = clearRefFromParameters(operation.Parameters) + operation.Responses = clearRefFromResponses(operation.Responses) + operation.RequestBody = clearRefFromRequestBody(operation.RequestBody) + + return operation +} + +func clearRefFromResponses(responses *openapi3.Responses) *openapi3.Responses { + if responses.Len() == 0 { + return responses + } + + retResponses := &openapi3.Responses{} + for i, parameterRef := range responses.Map() { + retResponses.Set(i, clearRefFromResponseRef(parameterRef)) + } + + return retResponses +} + +func clearRefFromRequestBody(requestBodyRef *openapi3.RequestBodyRef) *openapi3.RequestBodyRef { + if requestBodyRef == nil { + return requestBodyRef + } + + return &openapi3.RequestBodyRef{ + Value: clearRefFromRequestBodyRef(requestBodyRef.Value), + } +} + +func clearRefFromRequestBodyRef(requestBody *openapi3.RequestBody) *openapi3.RequestBody { + if requestBody == nil { + return requestBody + } + + requestBody.Content = clearRefFromContent(requestBody.Content) + + return requestBody +} + +func clearRefFromResponseRef(responseRef *openapi3.ResponseRef) *openapi3.ResponseRef { + if responseRef == nil { + return responseRef + } + + return &openapi3.ResponseRef{ + Value: clearRefFromResponse(responseRef.Value), + } +} + +func clearRefFromResponse(response *openapi3.Response) *openapi3.Response { + if response == nil { + return response + } + + response.Headers = clearRefFromHeaders(response.Headers) + response.Content = clearRefFromContent(response.Content) + + return response +} + +func clearRefFromHeaders(headers openapi3.Headers) openapi3.Headers { + if len(headers) == 0 { + return headers + } + + retHeaders := make(openapi3.Headers, len(headers)) + for key, headerRef := range headers { + retHeaders[key] = clearRefFromHeaderRef(headerRef) + } + return retHeaders +} + +func clearRefFromContent(content openapi3.Content) openapi3.Content { + if len(content) == 0 { + return content + } + + retContent := make(openapi3.Content, len(content)) + for key, mediaType := range content { + retContent[key] = clearRefFromMediaType(mediaType) + } + return retContent +} + +func clearRefFromMediaType(mediaType *openapi3.MediaType) *openapi3.MediaType { + if mediaType == nil { + return mediaType + } + + mediaType.Schema = clearRefFromSchemaRef(mediaType.Schema) + return mediaType +} + +func clearRefFromHeaderRef(headerRef *openapi3.HeaderRef) *openapi3.HeaderRef { + if headerRef == nil { + return headerRef + } + + return &openapi3.HeaderRef{ + Value: clearRefFromHeader(headerRef.Value), + } +} + +func clearRefFromHeader(header *openapi3.Header) *openapi3.Header { + if header == nil { + return header + } + + if parameter := clearRefFromParameter(&header.Parameter); parameter != nil { + header.Parameter = *parameter + } + + return header +} + +func clearRefFromParameterRef(parameterRef *openapi3.ParameterRef) *openapi3.ParameterRef { + if parameterRef == nil { + return parameterRef + } + + return &openapi3.ParameterRef{ + Value: clearRefFromParameter(parameterRef.Value), + } +} + +func clearRefFromParameter(parameter *openapi3.Parameter) *openapi3.Parameter { + if parameter == nil { + return parameter + } + + parameter.Schema = clearRefFromSchemaRef(parameter.Schema) + parameter.Content = clearRefFromContent(parameter.Content) + return parameter +} + +func clearRefFromSchemaRef(schemaRef *openapi3.SchemaRef) *openapi3.SchemaRef { + if schemaRef == nil { + return schemaRef + } + + return &openapi3.SchemaRef{ + Value: clearRefFromSchema(schemaRef.Value), + } +} + +func clearRefFromSchema(schema *openapi3.Schema) *openapi3.Schema { + if schema == nil { + return schema + } + + schema.OneOf = clearRefFromSchemaRefs(schema.OneOf) + schema.AnyOf = clearRefFromSchemaRefs(schema.AnyOf) + schema.AllOf = clearRefFromSchemaRefs(schema.AllOf) + schema.Not = clearRefFromSchemaRef(schema.Not) + schema.Items = clearRefFromSchemaRef(schema.Items) + schema.Properties = clearRefFromSchemas(schema.Properties) + //schema.AdditionalProperties = clearRefFromSchemaRef(schema.AdditionalProperties) + + return schema +} + +func clearRefFromSchemas(schemas openapi3.Schemas) openapi3.Schemas { + if len(schemas) == 0 { + return schemas + } + + retSchemas := make(openapi3.Schemas, len(schemas)) + for key, schemaRef := range schemas { + retSchemas[key] = clearRefFromSchemaRef(schemaRef) + } + return retSchemas +} + +func clearRefFromSchemaRefs(schemaRefs openapi3.SchemaRefs) openapi3.SchemaRefs { + if len(schemaRefs) == 0 { + return schemaRefs + } + + retSchemaRefs := make(openapi3.SchemaRefs, len(schemaRefs)) + for i, schemaRef := range schemaRefs { + retSchemaRefs[i] = clearRefFromSchemaRef(schemaRef) + } + return retSchemaRefs +} diff --git a/speculator/pkg/apispec/provided_spec_test.go b/speculator/pkg/apispec/provided_spec_test.go new file mode 100644 index 0000000..a4b80d3 --- /dev/null +++ b/speculator/pkg/apispec/provided_spec_test.go @@ -0,0 +1,1819 @@ +package apispec + +//import ( +// "encoding/json" +// "reflect" +// "testing" +// +// "github.com/getkin/kin-openapi/openapi3" +// "github.com/google/go-cmp/cmp/cmpopts" +// "gotest.tools/v3/assert" +// +// "github.com/5gsec/sentryflow/speculator/pkg/pathtrie" +//) +// +//func TestSpec_LoadProvidedSpec(t *testing.T) { +// jsonSpecV2 := "{\n \"swagger\": \"2.0\",\n \"info\": {\n \"version\": \"1.0.0\",\n \"title\": \"APIClarity APIs\"\n },\n \"basePath\": \"/api\",\n \"schemes\": [\n \"http\"\n ],\n \"consumes\": [\n \"application/json\"\n ],\n \"produces\": [\n \"application/json\"\n ],\n \"paths\": {\n \"/dashboard/apiUsage/mostUsed\": {\n \"get\": {\n \"summary\": \"Get most used APIs\",\n \"responses\": {\n \"200\": {\n \"description\": \"Success\",\n \"schema\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n }\n },\n \"default\": {\n \"$ref\": \"#/responses/UnknownError\"\n }\n }\n }\n }\n },\n \"schemas\": {\n \"ApiResponse\": {\n \"description\": \"An object that is return in all cases of failures.\",\n \"type\": \"object\",\n \"properties\": {\n \"message\": {\n \"type\": \"string\"\n }\n }\n }\n },\n \"responses\": {\n \"UnknownError\": {\n \"description\": \"unknown error\",\n \"schema\": {\n \"$ref\": \"\"\n }\n }\n }\n}" +// jsonSpecV2Invalid := "{\n \"info\": {\n \"version\": \"1.0.0\",\n \"title\": \"APIClarity APIs\"\n },\n \"basePath\": \"/api\",\n \"schemes\": [\n \"http\"\n ],\n \"consumes\": [\n \"application/json\"\n ],\n \"produces\": [\n \"application/json\"\n ],\n \"paths\": {\n \"/dashboard/apiUsage/mostUsed\": {\n \"get\": {\n \"summary\": \"Get most used APIs\",\n \"responses\": {\n \"200\": {\n \"description\": \"Success\",\n \"schema\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n }\n },\n \"default\": {\n \"$ref\": \"#/responses/UnknownError\"\n }\n }\n }\n }\n },\n \"schemas\": {\n \"ApiResponse\": {\n \"description\": \"An object that is return in all cases of failures.\",\n \"type\": \"object\",\n \"properties\": {\n \"message\": {\n \"type\": \"string\"\n }\n }\n }\n },\n \"responses\": {\n \"UnknownError\": {\n \"description\": \"unknown error\",\n \"schema\": {\n \"$ref\": \"#/schemas/ApiResponse\"\n }\n }\n }\n}" +// jsonSpec := "{\n \"openapi\": \"3.0.3\",\n \"info\": {\n \"version\": \"1.0.0\",\n \"title\": \"Simple API\",\n \"description\": \"A simple API to illustrate OpenAPI concepts\"\n },\n \"servers\": [\n {\n \"url\": \"https://example.io/v1\"\n }\n ],\n \"security\": [\n {\n \"BasicAuth\": []\n }\n ],\n \"paths\": {\n \"/artists\": {\n \"get\": {\n \"description\": \"Returns a list of artists\",\n \"parameters\": [\n {\n \"name\": \"limit\",\n \"in\": \"query\",\n \"description\": \"Limits the number of items on a page\",\n \"schema\": {\n \"type\": \"integer\"\n }\n },\n {\n \"name\": \"offset\",\n \"in\": \"query\",\n \"description\": \"Specifies the page number of the artists to be displayed\",\n \"schema\": {\n \"type\": \"integer\"\n }\n }\n ],\n \"responses\": {\n \"200\": {\n \"description\": \"Successfully returned a list of artists\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"required\": [\n \"username\"\n ],\n \"properties\": {\n \"artist_name\": {\n \"type\": \"string\"\n },\n \"artist_genre\": {\n \"type\": \"string\"\n },\n \"albums_recorded\": {\n \"type\": \"integer\"\n },\n \"username\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n },\n \"400\": {\n \"description\": \"Invalid request\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"message\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n }\n },\n \"post\": {\n \"description\": \"Lets a user post a new artist\",\n \"requestBody\": {\n \"required\": true,\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"required\": [\n \"username\"\n ],\n \"properties\": {\n \"artist_name\": {\n \"type\": \"string\"\n },\n \"artist_genre\": {\n \"type\": \"string\"\n },\n \"albums_recorded\": {\n \"type\": \"integer\"\n },\n \"username\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n },\n \"responses\": {\n \"200\": {\n \"description\": \"Successfully created a new artist\"\n },\n \"400\": {\n \"description\": \"Invalid request\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"message\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n }\n }\n },\n \"/artists/{username}\": {\n \"get\": {\n \"description\": \"Obtain information about an artist from his or her unique username\",\n \"parameters\": [\n {\n \"name\": \"username\",\n \"in\": \"path\",\n \"required\": true,\n \"schema\": {\n \"type\": \"string\"\n }\n }\n ],\n \"responses\": {\n \"200\": {\n \"description\": \"Successfully returned an artist\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"artist_name\": {\n \"type\": \"string\"\n },\n \"artist_genre\": {\n \"type\": \"string\"\n },\n \"albums_recorded\": {\n \"type\": \"integer\"\n }\n }\n }\n }\n }\n },\n \"400\": {\n \"description\": \"Invalid request\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"message\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n }\n }\n }\n },\n \"components\": {\n \"securitySchemes\": {\n \"BasicAuth\": {\n \"type\": \"http\",\n \"scheme\": \"basic\"\n }\n }\n }\n}" +// jsonSpecWithRef := "{\n \"openapi\": \"3.0.3\",\n \"info\": {\n \"version\": \"1.0.0\",\n \"title\": \"Simple API\",\n \"description\": \"A simple API to illustrate OpenAPI concepts\"\n },\n \"servers\": [\n {\n \"url\": \"https://example.io/v1\"\n }\n ],\n \"security\": [\n {\n \"BasicAuth\": []\n }\n ],\n \"paths\": {\n \"/artists\": {\n \"get\": {\n \"description\": \"Returns a list of artists\",\n \"parameters\": [\n {\n \"name\": \"limit\",\n \"in\": \"query\",\n \"description\": \"Limits the number of items on a page\",\n \"schema\": {\n \"type\": \"integer\"\n }\n },\n {\n \"name\": \"offset\",\n \"in\": \"query\",\n \"description\": \"Specifies the page number of the artists to be displayed\",\n \"schema\": {\n \"type\": \"integer\"\n }\n }\n ],\n \"responses\": {\n \"200\": {\n \"description\": \"Successfully returned a list of artists\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"required\": [\n \"username\"\n ],\n \"properties\": {\n \"artist_name\": {\n \"type\": \"string\"\n },\n \"artist_genre\": {\n \"type\": \"string\"\n },\n \"albums_recorded\": {\n \"type\": \"integer\"\n },\n \"username\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n },\n \"400\": {\n \"description\": \"Invalid request\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"message\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n }\n },\n \"post\": {\n \"description\": \"Lets a user post a new artist\",\n \"requestBody\": {\n \"required\": true,\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"required\": [\n \"username\"\n ],\n \"properties\": {\n \"artist_name\": {\n \"type\": \"string\"\n },\n \"artist_genre\": {\n \"type\": \"string\"\n },\n \"albums_recorded\": {\n \"type\": \"integer\"\n },\n \"username\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n },\n \"responses\": {\n \"200\": {\n \"description\": \"Successfully created a new artist\"\n },\n \"400\": {\n \"description\": \"Invalid request\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"message\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n }\n }\n },\n \"/artists/{username}\": {\n \"get\": {\n \"description\": \"Obtain information about an artist from his or her unique username\",\n \"parameters\": [\n {\n \"name\": \"username\",\n \"in\": \"path\",\n \"required\": true,\n \"schema\": {\n \"type\": \"string\"\n }\n }\n ],\n \"responses\": {\n \"200\": {\n \"description\": \"Successfully returned an artist\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"$ref\": \"#/components/schemas/Artists\"\n }\n }\n }\n },\n \"400\": {\n \"description\": \"Invalid request\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"message\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n }\n }\n }\n },\n \"components\": {\n \"schemas\": {\n \"Artists\": {\n \"type\": \"object\",\n \"properties\": {\n \"artist_name\": {\n \"type\": \"string\"\n },\n \"artist_genre\": {\n \"type\": \"string\"\n },\n \"albums_recorded\": {\n \"type\": \"integer\"\n }\n }\n }\n },\n \"securitySchemes\": {\n \"BasicAuth\": {\n \"type\": \"http\",\n \"scheme\": \"basic\"\n }\n }\n }\n}" +// jsonSpecWithRefAfterRemoveRef := "{\n \"openapi\": \"3.0.3\",\n \"info\": {\n \"version\": \"1.0.0\",\n \"title\": \"Simple API\",\n \"description\": \"A simple API to illustrate OpenAPI concepts\"\n },\n \"servers\": [\n {\n \"url\": \"https://example.io/v1\"\n }\n ],\n \"security\": [\n {\n \"BasicAuth\": []\n }\n ],\n \"paths\": {\n \"/artists\": {\n \"get\": {\n \"description\": \"Returns a list of artists\",\n \"parameters\": [\n {\n \"name\": \"limit\",\n \"in\": \"query\",\n \"description\": \"Limits the number of items on a page\",\n \"schema\": {\n \"type\": \"integer\"\n }\n },\n {\n \"name\": \"offset\",\n \"in\": \"query\",\n \"description\": \"Specifies the page number of the artists to be displayed\",\n \"schema\": {\n \"type\": \"integer\"\n }\n }\n ],\n \"responses\": {\n \"200\": {\n \"description\": \"Successfully returned a list of artists\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"required\": [\n \"username\"\n ],\n \"properties\": {\n \"artist_name\": {\n \"type\": \"string\"\n },\n \"artist_genre\": {\n \"type\": \"string\"\n },\n \"albums_recorded\": {\n \"type\": \"integer\"\n },\n \"username\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n },\n \"400\": {\n \"description\": \"Invalid request\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"message\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n }\n },\n \"post\": {\n \"description\": \"Lets a user post a new artist\",\n \"requestBody\": {\n \"required\": true,\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"required\": [\n \"username\"\n ],\n \"properties\": {\n \"artist_name\": {\n \"type\": \"string\"\n },\n \"artist_genre\": {\n \"type\": \"string\"\n },\n \"albums_recorded\": {\n \"type\": \"integer\"\n },\n \"username\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n },\n \"responses\": {\n \"200\": {\n \"description\": \"Successfully created a new artist\"\n },\n \"400\": {\n \"description\": \"Invalid request\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"message\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n }\n }\n },\n \"/artists/{username}\": {\n \"get\": {\n \"description\": \"Obtain information about an artist from his or her unique username\",\n \"parameters\": [\n {\n \"name\": \"username\",\n \"in\": \"path\",\n \"required\": true,\n \"schema\": {\n \"type\": \"string\"\n }\n }\n ],\n \"responses\": {\n \"200\": {\n \"description\": \"Successfully returned an artist\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"artist_name\": {\n \"type\": \"string\"\n },\n \"artist_genre\": {\n \"type\": \"string\"\n },\n \"albums_recorded\": {\n \"type\": \"integer\"\n }\n }\n }\n }\n }\n },\n \"400\": {\n \"description\": \"Invalid request\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"message\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n }\n }\n }\n },\n \"components\": {\n \"schemas\": {\n \"Artists\": {\n \"type\": \"object\",\n \"properties\": {\n \"artist_name\": {\n \"type\": \"string\"\n },\n \"artist_genre\": {\n \"type\": \"string\"\n },\n \"albums_recorded\": {\n \"type\": \"integer\"\n }\n }\n }\n },\n \"securitySchemes\": {\n \"BasicAuth\": {\n \"type\": \"http\",\n \"scheme\": \"basic\"\n }\n }\n }\n}" +// jsonSpecInvalid := "{\n \"openapi\": \"\",\n \"info\": {\n \"version\": \"1.0.0\",\n \"title\": \"Simple API\",\n \"description\": \"A simple API to illustrate OpenAPI concepts\"\n },\n \"servers\": [\n {\n \"url\": \"https://example.io/v1\"\n }\n ],\n \"security\": [\n {\n \"BasicAuth\": []\n }\n ],\n \"paths\": {\n \"/artists\": {\n \"get\": {\n \"description\": \"Returns a list of artists\",\n \"parameters\": [\n {\n \"name\": \"limit\",\n \"in\": \"query\",\n \"description\": \"Limits the number of items on a page\",\n \"schema\": {\n \"type\": \"integer\"\n }\n },\n {\n \"name\": \"offset\",\n \"in\": \"query\",\n \"description\": \"Specifies the page number of the artists to be displayed\",\n \"schema\": {\n \"type\": \"integer\"\n }\n }\n ],\n \"responses\": {\n \"200\": {\n \"description\": \"Successfully returned a list of artists\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"required\": [\n \"username\"\n ],\n \"properties\": {\n \"artist_name\": {\n \"type\": \"string\"\n },\n \"artist_genre\": {\n \"type\": \"string\"\n },\n \"albums_recorded\": {\n \"type\": \"integer\"\n },\n \"username\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n },\n \"400\": {\n \"description\": \"Invalid request\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"message\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n }\n },\n \"post\": {\n \"description\": \"Lets a user post a new artist\",\n \"requestBody\": {\n \"required\": true,\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"required\": [\n \"username\"\n ],\n \"properties\": {\n \"artist_name\": {\n \"type\": \"string\"\n },\n \"artist_genre\": {\n \"type\": \"string\"\n },\n \"albums_recorded\": {\n \"type\": \"integer\"\n },\n \"username\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n },\n \"responses\": {\n \"200\": {\n \"description\": \"Successfully created a new artist\"\n },\n \"400\": {\n \"description\": \"Invalid request\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"message\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n }\n }\n },\n \"/artists/{username}\": {\n \"get\": {\n \"description\": \"Obtain information about an artist from his or her unique username\",\n \"parameters\": [\n {\n \"name\": \"username\",\n \"in\": \"path\",\n \"required\": true,\n \"schema\": {\n \"type\": \"string\"\n }\n }\n ],\n \"responses\": {\n \"200\": {\n \"description\": \"Successfully returned an artist\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"artist_name\": {\n \"type\": \"string\"\n },\n \"artist_genre\": {\n \"type\": \"string\"\n },\n \"albums_recorded\": {\n \"type\": \"integer\"\n }\n }\n }\n }\n }\n },\n \"400\": {\n \"description\": \"Invalid request\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"message\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n }\n }\n }\n },\n \"components\": {\n \"securitySchemes\": {\n \"BasicAuth\": {\n \"type\": \"http\",\n \"scheme\": \"basic\"\n }\n }\n }\n}" +// yamlSpec := "openapi: 3.0.3\ninfo:\n version: 1.0.0\n title: Simple API\n description: A simple API to illustrate OpenAPI concepts\n\nservers:\n - url: https://example.io/v1\n\nsecurity:\n - BasicAuth: []\n\npaths:\n /artists:\n get:\n description: Returns a list of artists \n parameters:\n - name: limit\n in: query\n description: Limits the number of items on a page\n schema:\n type: integer\n - name: offset\n in: query\n description: Specifies the page number of the artists to be displayed\n schema:\n type: integer\n responses:\n '200':\n description: Successfully returned a list of artists\n content:\n application/json:\n schema:\n type: array\n items:\n type: object\n required:\n - username\n properties:\n artist_name:\n type: string\n artist_genre:\n type: string\n albums_recorded:\n type: integer\n username:\n type: string\n '400':\n description: Invalid request\n content:\n application/json:\n schema:\n type: object \n properties:\n message:\n type: string\n\n post:\n description: Lets a user post a new artist\n requestBody:\n required: true\n content:\n application/json:\n schema:\n type: array\n items:\n type: object\n required:\n - username\n properties:\n artist_name:\n type: string\n artist_genre:\n type: string\n albums_recorded:\n type: integer\n username:\n type: string\n responses:\n '200':\n description: Successfully created a new artist\n '400':\n description: Invalid request\n content:\n application/json:\n schema:\n type: object \n properties:\n message:\n type: string\n\n /artists/{username}:\n get:\n description: Obtain information about an artist from his or her unique username\n parameters:\n - name: username\n in: path\n required: true\n schema:\n type: string\n \n responses:\n '200':\n description: Successfully returned an artist\n content:\n application/json:\n schema:\n type: object\n properties:\n artist_name:\n type: string\n artist_genre:\n type: string\n albums_recorded:\n type: integer\n \n '400':\n description: Invalid request\n content:\n application/json:\n schema:\n type: object \n properties:\n message:\n type: string\n\ncomponents:\n securitySchemes:\n BasicAuth:\n type: http\n scheme: basic\n" +// +// v3, err := LoadAndValidateRawJSONSpecV3([]byte(jsonSpec)) +// assert.NilError(t, err) +// wantProvidedSpec := &ProvidedSpec{ +// Doc: v3, +// OriginalSpecVersion: OASv3, +// } +// +// v2, err := LoadAndValidateRawJSONSpecV3FromV2([]byte(jsonSpecV2)) +// assert.NilError(t, err) +// wantProvidedSpecV2 := &ProvidedSpec{ +// Doc: clearRefFromDoc(v2), +// OriginalSpecVersion: OASv2, +// } +// +// wantProvidedSpecWithRefAfterRemoveRef := &ProvidedSpec{ +// Doc: &openapi3.T{ +// Paths: openapi3.Paths{}, +// }, +// OriginalSpecVersion: OASv3, +// } +// err = json.Unmarshal([]byte(jsonSpecWithRefAfterRemoveRef), wantProvidedSpecWithRefAfterRemoveRef.Doc) +// assert.NilError(t, err) +// +// pathToPathID := map[string]string{ +// "/artists": "1", +// } +// wantProvidedPathTrie := createPathTrie(pathToPathID) +// +// pathToPathIDv2 := map[string]string{ +// "/dashboard/apiUsage/mostUsed": "1", +// } +// wantProvidedPathv2Trie := createPathTrie(pathToPathIDv2) +// emptyPathTrie := createPathTrie(nil) +// +// type fields struct { +// ProvidedSpec *ProvidedSpec +// } +// type args struct { +// providedSpec []byte +// pathToPathID map[string]string +// } +// tests := []struct { +// name string +// fields fields +// args args +// wantErr bool +// wantProvidedPathTrie pathtrie.PathTrie +// wantProvidedSpec *ProvidedSpec +// }{ +// { +// name: "json spec", +// fields: fields{ +// ProvidedSpec: nil, +// }, +// args: args{ +// providedSpec: []byte(jsonSpec), +// pathToPathID: pathToPathID, +// }, +// wantErr: false, +// wantProvidedPathTrie: wantProvidedPathTrie, +// wantProvidedSpec: wantProvidedSpec, +// }, +// { +// name: "json spec v2", +// fields: fields{ +// ProvidedSpec: nil, +// }, +// args: args{ +// providedSpec: []byte(jsonSpecV2), +// pathToPathID: pathToPathIDv2, +// }, +// wantErr: false, +// wantProvidedPathTrie: wantProvidedPathv2Trie, +// wantProvidedSpec: wantProvidedSpecV2, +// }, +// { +// name: "json spec with ref", +// fields: fields{ +// ProvidedSpec: nil, +// }, +// args: args{ +// providedSpec: []byte(jsonSpecWithRef), +// pathToPathID: pathToPathID, +// }, +// wantErr: false, +// wantProvidedPathTrie: wantProvidedPathTrie, +// wantProvidedSpec: wantProvidedSpecWithRefAfterRemoveRef, +// }, +// { +// name: "json spec with a missing path", +// fields: fields{ +// ProvidedSpec: nil, +// }, +// args: args{ +// providedSpec: []byte(jsonSpec), +// pathToPathID: map[string]string{}, +// }, +// wantErr: false, +// wantProvidedPathTrie: emptyPathTrie, +// wantProvidedSpec: wantProvidedSpec, +// }, +// { +// name: "yaml spec", +// fields: fields{ +// ProvidedSpec: nil, +// }, +// args: args{ +// providedSpec: []byte(yamlSpec), +// pathToPathID: pathToPathID, +// }, +// wantErr: false, +// wantProvidedPathTrie: wantProvidedPathTrie, +// wantProvidedSpec: wantProvidedSpec, +// }, +// { +// name: "invalid json", +// fields: fields{ +// ProvidedSpec: nil, +// }, +// args: args{ +// providedSpec: []byte("bad" + jsonSpec), +// }, +// wantErr: true, +// }, +// { +// name: "invalid spec v3", +// fields: fields{ +// ProvidedSpec: nil, +// }, +// args: args{ +// providedSpec: []byte(jsonSpecInvalid), +// }, +// wantErr: true, +// }, +// { +// name: "invalid spec v2", +// fields: fields{ +// ProvidedSpec: nil, +// }, +// args: args{ +// providedSpec: []byte(jsonSpecV2Invalid), +// }, +// wantErr: true, +// }, +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// s := &Spec{ +// SpecInfo: SpecInfo{ +// ProvidedSpec: tt.fields.ProvidedSpec, +// }, +// } +// if err := s.LoadProvidedSpec(tt.args.providedSpec, tt.args.pathToPathID); (err != nil) != tt.wantErr { +// t.Errorf("LoadProvidedSpec() error = %v, wantErr %v", err, tt.wantErr) +// } +// if !tt.wantErr { +// assert.DeepEqual(t, s.ProvidedSpec, tt.wantProvidedSpec, cmpopts.IgnoreUnexported(openapi3.Schema{}), cmpopts.IgnoreTypes(openapi3.ExtensionProps{})) +// if !reflect.DeepEqual(s.ProvidedPathTrie, tt.wantProvidedPathTrie) { +// t.Errorf("LoadProvidedSpec() got = %v, want %v", marshal(s.ProvidedPathTrie), marshal(tt.wantProvidedPathTrie)) +// } +// } +// }) +// } +//} +// +//func TestProvidedSpec_GetBasePath(t *testing.T) { +// type fields struct { +// Doc *openapi3.T +// } +// tests := []struct { +// name string +// fields fields +// want string +// }{ +// { +// name: "url templating", +// fields: fields{ +// Doc: &openapi3.T{ +// Servers: []*openapi3.Server{ +// { +// URL: "{protocol}://api.example.com/api", +// }, +// }, +// }, +// }, +// want: "/api", +// }, +// { +// name: "sanity", +// fields: fields{ +// Doc: &openapi3.T{ +// Servers: []*openapi3.Server{ +// { +// URL: "https://api.example.com:8443/v1/reports", +// }, +// }, +// }, +// }, +// want: "/v1/reports", +// }, +// { +// name: "no path", +// fields: fields{ +// Doc: &openapi3.T{ +// Servers: []*openapi3.Server{ +// { +// URL: "https://api.example.com", +// }, +// }, +// }, +// }, +// want: "", +// }, +// { +// name: "no url", +// fields: fields{ +// Doc: &openapi3.T{ +// Servers: []*openapi3.Server{ +// { +// URL: "", +// }, +// }, +// }, +// }, +// want: "", +// }, +// { +// name: "only path", +// fields: fields{ +// Doc: &openapi3.T{ +// Servers: []*openapi3.Server{ +// { +// URL: "/v1/reports", +// }, +// }, +// }, +// }, +// want: "/v1/reports", +// }, +// { +// name: "root path", +// fields: fields{ +// Doc: &openapi3.T{ +// Servers: []*openapi3.Server{ +// { +// URL: "/", +// }, +// }, +// }, +// }, +// want: "", +// }, +// { +// name: "ip", +// fields: fields{ +// Doc: &openapi3.T{ +// Servers: []*openapi3.Server{ +// { +// URL: "http://10.0.81.36/v1", +// }, +// }, +// }, +// }, +// want: "/v1", +// }, +// { +// name: "bad url", +// fields: fields{ +// Doc: &openapi3.T{ +// Servers: []*openapi3.Server{ +// { +// URL: "bad.url.dot.com.!@##", +// }, +// }, +// }, +// }, +// want: "", +// }, +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// p := &ProvidedSpec{ +// Doc: tt.fields.Doc, +// } +// if got := p.GetBasePath(); got != tt.want { +// t.Errorf("GetBasePath() = %v, want %v", got, tt.want) +// } +// }) +// } +//} +// +//func Test_clearRefFromDoc(t *testing.T) { +// type args struct { +// doc *openapi3.T +// } +// tests := []struct { +// name string +// args args +// want *openapi3.T +// }{ +// { +// name: "nil doc", +// args: args{ +// doc: nil, +// }, +// want: nil, +// }, +// { +// name: "no paths", +// args: args{ +// doc: &openapi3.T{ +// Paths: openapi3.Paths{}, +// }, +// }, +// want: &openapi3.T{ +// Paths: openapi3.Paths{}, +// }, +// }, +// { +// name: "multiple paths", +// args: args{ +// doc: &openapi3.T{ +// Paths: openapi3.Paths{ +// "path1": &openapi3.PathItem{ +// Get: createTestOperation(). +// WithRequestBody(openapi3.NewRequestBody(). +// WithJSONSchemaRef(openapi3.NewSchemaRef("array-string", +// openapi3.NewObjectSchema().WithItems(openapi3.NewStringSchema())))).Op, +// }, +// "path2": &openapi3.PathItem{ +// Put: createTestOperation(). +// WithRequestBody(openapi3.NewRequestBody(). +// WithJSONSchemaRef(openapi3.NewSchemaRef("array-int", +// openapi3.NewObjectSchema().WithItems(openapi3.NewInt64Schema())))).Op, +// }, +// }, +// }, +// }, +// want: &openapi3.T{ +// Paths: openapi3.Paths{ +// "path1": &openapi3.PathItem{ +// Get: createTestOperation(). +// WithRequestBody(openapi3.NewRequestBody(). +// WithJSONSchemaRef(openapi3.NewSchemaRef("", +// openapi3.NewObjectSchema().WithItems(openapi3.NewStringSchema())))).Op, +// }, +// "path2": &openapi3.PathItem{ +// Put: createTestOperation(). +// WithRequestBody(openapi3.NewRequestBody(). +// WithJSONSchemaRef(openapi3.NewSchemaRef("", +// openapi3.NewObjectSchema().WithItems(openapi3.NewInt64Schema())))).Op, +// }, +// }, +// }, +// }, +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// assert.DeepEqual(t, clearRefFromDoc(tt.args.doc), tt.want, cmpopts.IgnoreUnexported(openapi3.Schema{}), cmpopts.IgnoreTypes(openapi3.ExtensionProps{})) +// }) +// } +//} +// +//func Test_clearRefFromPathItem(t *testing.T) { +// type args struct { +// item *openapi3.PathItem +// } +// tests := []struct { +// name string +// args args +// want *openapi3.PathItem +// }{ +// { +// name: "nil item", +// args: args{ +// item: nil, +// }, +// want: nil, +// }, +// { +// name: "empty item", +// args: args{ +// item: &openapi3.PathItem{}, +// }, +// want: &openapi3.PathItem{}, +// }, +// { +// name: "ref item", +// args: args{ +// item: &openapi3.PathItem{ +// Ref: "ref", +// }, +// }, +// want: &openapi3.PathItem{ +// Ref: "", +// }, +// }, +// { +// name: "multiple operations", +// args: args{ +// item: &openapi3.PathItem{ +// Connect: createTestOperation(). +// WithRequestBody(openapi3.NewRequestBody(). +// WithJSONSchemaRef(openapi3.NewSchemaRef("array-string", +// openapi3.NewObjectSchema().WithItems(openapi3.NewStringSchema())))).Op, +// Delete: createTestOperation(). +// WithRequestBody(openapi3.NewRequestBody(). +// WithJSONSchemaRef(openapi3.NewSchemaRef("array-int", +// openapi3.NewObjectSchema().WithItems(openapi3.NewInt64Schema())))).Op, +// }, +// }, +// want: &openapi3.PathItem{ +// Connect: createTestOperation(). +// WithRequestBody(openapi3.NewRequestBody(). +// WithJSONSchemaRef(openapi3.NewSchemaRef("", +// openapi3.NewObjectSchema().WithItems(openapi3.NewStringSchema())))).Op, +// Delete: createTestOperation(). +// WithRequestBody(openapi3.NewRequestBody(). +// WithJSONSchemaRef(openapi3.NewSchemaRef("", +// openapi3.NewObjectSchema().WithItems(openapi3.NewInt64Schema())))).Op, +// }, +// }, +// { +// name: "multiple parameters", +// args: args{ +// item: &openapi3.PathItem{ +// Parameters: openapi3.Parameters{ +// { +// Ref: "ref-path", +// Value: openapi3.NewPathParameter("path"), +// }, +// { +// Ref: "ref-query", +// Value: openapi3.NewQueryParameter("query"), +// }, +// }, +// }, +// }, +// want: &openapi3.PathItem{ +// Parameters: openapi3.Parameters{ +// { +// Ref: "", +// Value: openapi3.NewPathParameter("path"), +// }, +// { +// Ref: "", +// Value: openapi3.NewQueryParameter("query"), +// }, +// }, +// }, +// }, +// { +// name: "multiple operations and parameters", +// args: args{ +// item: &openapi3.PathItem{ +// Connect: createTestOperation(). +// WithRequestBody(openapi3.NewRequestBody(). +// WithJSONSchemaRef(openapi3.NewSchemaRef("array-string", +// openapi3.NewObjectSchema().WithItems(openapi3.NewStringSchema())))).Op, +// Delete: createTestOperation(). +// WithRequestBody(openapi3.NewRequestBody(). +// WithJSONSchemaRef(openapi3.NewSchemaRef("array-int", +// openapi3.NewObjectSchema().WithItems(openapi3.NewInt64Schema())))).Op, +// Parameters: openapi3.Parameters{ +// { +// Ref: "ref-path", +// Value: openapi3.NewPathParameter("path"), +// }, +// { +// Ref: "ref-query", +// Value: openapi3.NewQueryParameter("query"), +// }, +// }, +// }, +// }, +// want: &openapi3.PathItem{ +// Connect: createTestOperation(). +// WithRequestBody(openapi3.NewRequestBody(). +// WithJSONSchemaRef(openapi3.NewSchemaRef("", +// openapi3.NewObjectSchema().WithItems(openapi3.NewStringSchema())))).Op, +// Delete: createTestOperation(). +// WithRequestBody(openapi3.NewRequestBody(). +// WithJSONSchemaRef(openapi3.NewSchemaRef("", +// openapi3.NewObjectSchema().WithItems(openapi3.NewInt64Schema())))).Op, +// Parameters: openapi3.Parameters{ +// { +// Ref: "", +// Value: openapi3.NewPathParameter("path"), +// }, +// { +// Ref: "", +// Value: openapi3.NewQueryParameter("query"), +// }, +// }, +// }, +// }, +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// assert.DeepEqual(t, clearRefFromPathItem(tt.args.item), tt.want, cmpopts.IgnoreUnexported(openapi3.Schema{}), cmpopts.IgnoreTypes(openapi3.ExtensionProps{})) +// }) +// } +//} +// +//func Test_clearRefFromParameters(t *testing.T) { +// type args struct { +// parameters openapi3.Parameters +// } +// tests := []struct { +// name string +// args args +// want openapi3.Parameters +// }{ +// { +// name: "nil parameters", +// args: args{ +// parameters: nil, +// }, +// want: nil, +// }, +// { +// name: "empty parameters", +// args: args{ +// parameters: openapi3.NewParameters(), +// }, +// want: openapi3.NewParameters(), +// }, +// { +// name: "multiple parameters", +// args: args{ +// parameters: openapi3.Parameters{ +// { +// Ref: "ref-path", +// Value: openapi3.NewPathParameter("path"), +// }, +// { +// Ref: "ref-query", +// Value: openapi3.NewQueryParameter("query"), +// }, +// }, +// }, +// want: openapi3.Parameters{ +// { +// Ref: "", +// Value: openapi3.NewPathParameter("path"), +// }, +// { +// Ref: "", +// Value: openapi3.NewQueryParameter("query"), +// }, +// }, +// }, +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// assert.DeepEqual(t, clearRefFromParameters(tt.args.parameters), tt.want, cmpopts.IgnoreUnexported(openapi3.Schema{}), cmpopts.IgnoreTypes(openapi3.ExtensionProps{})) +// }) +// } +//} +// +//func Test_clearRefFromOperation(t *testing.T) { +// type args struct { +// operation *openapi3.Operation +// } +// tests := []struct { +// name string +// args args +// want *openapi3.Operation +// }{ +// { +// name: "nil operation", +// args: args{ +// operation: nil, +// }, +// want: nil, +// }, +// { +// name: "empty operation", +// args: args{ +// operation: openapi3.NewOperation(), +// }, +// want: openapi3.NewOperation(), +// }, +// { +// name: "multiple parameters", +// args: args{ +// operation: &openapi3.Operation{ +// Parameters: openapi3.Parameters{ +// { +// Ref: "ref-path", +// Value: openapi3.NewPathParameter("path"), +// }, +// { +// Ref: "ref-query", +// Value: openapi3.NewQueryParameter("query"), +// }, +// }, +// }, +// }, +// want: &openapi3.Operation{ +// Parameters: openapi3.Parameters{ +// { +// Ref: "", +// Value: openapi3.NewPathParameter("path"), +// }, +// { +// Ref: "", +// Value: openapi3.NewQueryParameter("query"), +// }, +// }, +// }, +// }, +// { +// name: "multiple responses", +// args: args{ +// operation: &openapi3.Operation{ +// Responses: openapi3.Responses{ +// "response1": { +// Ref: "ref-response1", +// Value: openapi3.NewResponse(). +// WithJSONSchemaRef(openapi3.NewSchemaRef("ref-int", +// openapi3.NewObjectSchema().WithItems(openapi3.NewInt64Schema()))), +// }, +// "response2": { +// Ref: "ref-response2", +// Value: openapi3.NewResponse(). +// WithJSONSchemaRef(openapi3.NewSchemaRef("ref-string", +// openapi3.NewObjectSchema().WithItems(openapi3.NewStringSchema()))), +// }, +// }, +// }, +// }, +// want: &openapi3.Operation{ +// Responses: openapi3.Responses{ +// "response1": { +// Ref: "", +// Value: openapi3.NewResponse(). +// WithJSONSchemaRef(openapi3.NewSchemaRef("", +// openapi3.NewObjectSchema().WithItems(openapi3.NewInt64Schema()))), +// }, +// "response2": { +// Ref: "", +// Value: openapi3.NewResponse(). +// WithJSONSchemaRef(openapi3.NewSchemaRef("", +// openapi3.NewObjectSchema().WithItems(openapi3.NewStringSchema()))), +// }, +// }, +// }, +// }, +// { +// name: "request body", +// args: args{ +// operation: &openapi3.Operation{ +// RequestBody: &openapi3.RequestBodyRef{ +// Ref: "ref-request-body", +// Value: openapi3.NewRequestBody(). +// WithJSONSchemaRef(openapi3.NewSchemaRef("ref", +// openapi3.NewObjectSchema().WithItems(openapi3.NewStringSchema()))), +// }, +// }, +// }, +// want: &openapi3.Operation{ +// RequestBody: &openapi3.RequestBodyRef{ +// Ref: "", +// Value: openapi3.NewRequestBody(). +// WithJSONSchemaRef(openapi3.NewSchemaRef("", +// openapi3.NewObjectSchema().WithItems(openapi3.NewStringSchema()))), +// }, +// }, +// }, +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// assert.DeepEqual(t, clearRefFromOperation(tt.args.operation), tt.want, cmpopts.IgnoreUnexported(openapi3.Schema{}), cmpopts.IgnoreTypes(openapi3.ExtensionProps{})) +// }) +// } +//} +// +//func Test_clearRefFromResponses(t *testing.T) { +// type args struct { +// responses openapi3.Responses +// } +// tests := []struct { +// name string +// args args +// want openapi3.Responses +// }{ +// { +// name: "nil responses", +// args: args{ +// responses: nil, +// }, +// want: nil, +// }, +// { +// name: "empty responses", +// args: args{ +// responses: openapi3.NewResponses(), +// }, +// want: openapi3.NewResponses(), +// }, +// { +// name: "multiple responses", +// args: args{ +// responses: openapi3.Responses{ +// "response1": { +// Ref: "ref-response1", +// Value: openapi3.NewResponse(). +// WithJSONSchemaRef(openapi3.NewSchemaRef("ref-int", +// openapi3.NewObjectSchema().WithItems(openapi3.NewInt64Schema()))), +// }, +// "response2": { +// Ref: "ref-response2", +// Value: openapi3.NewResponse(). +// WithJSONSchemaRef(openapi3.NewSchemaRef("ref-string", +// openapi3.NewObjectSchema().WithItems(openapi3.NewStringSchema()))), +// }, +// }, +// }, +// want: openapi3.Responses{ +// "response1": { +// Ref: "", +// Value: openapi3.NewResponse(). +// WithJSONSchemaRef(openapi3.NewSchemaRef("", +// openapi3.NewObjectSchema().WithItems(openapi3.NewInt64Schema()))), +// }, +// "response2": { +// Ref: "", +// Value: openapi3.NewResponse(). +// WithJSONSchemaRef(openapi3.NewSchemaRef("", +// openapi3.NewObjectSchema().WithItems(openapi3.NewStringSchema()))), +// }, +// }, +// }, +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// assert.DeepEqual(t, clearRefFromResponses(tt.args.responses), tt.want, cmpopts.IgnoreUnexported(openapi3.Schema{}), cmpopts.IgnoreTypes(openapi3.ExtensionProps{})) +// }) +// } +//} +// +//func Test_clearRefFromRequestBody(t *testing.T) { +// type args struct { +// requestBodyRef *openapi3.RequestBodyRef +// } +// tests := []struct { +// name string +// args args +// want *openapi3.RequestBodyRef +// }{ +// { +// name: "nil requestBodyRef", +// args: args{ +// requestBodyRef: nil, +// }, +// want: nil, +// }, +// { +// name: "empty requestBodyRef", +// args: args{ +// requestBodyRef: &openapi3.RequestBodyRef{}, +// }, +// want: &openapi3.RequestBodyRef{}, +// }, +// { +// name: "sanity requestBodyRef", +// args: args{ +// requestBodyRef: &openapi3.RequestBodyRef{ +// Ref: "ref", +// Value: openapi3.NewRequestBody(). +// WithJSONSchemaRef(openapi3.NewSchemaRef("ref-string", +// openapi3.NewObjectSchema().WithItems(openapi3.NewStringSchema()))), +// }, +// }, +// want: &openapi3.RequestBodyRef{ +// Ref: "", +// Value: openapi3.NewRequestBody(). +// WithJSONSchemaRef(openapi3.NewSchemaRef("", +// openapi3.NewObjectSchema().WithItems(openapi3.NewStringSchema()))), +// }, +// }, +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// assert.DeepEqual(t, clearRefFromRequestBody(tt.args.requestBodyRef), tt.want, cmpopts.IgnoreUnexported(openapi3.Schema{}), cmpopts.IgnoreTypes(openapi3.ExtensionProps{})) +// }) +// } +//} +// +//func Test_clearRefFromRequestBodyRef(t *testing.T) { +// type args struct { +// requestBody *openapi3.RequestBody +// } +// tests := []struct { +// name string +// args args +// want *openapi3.RequestBody +// }{ +// { +// name: "nil RequestBody", +// args: args{ +// requestBody: nil, +// }, +// want: nil, +// }, +// { +// name: "empty RequestBody", +// args: args{ +// requestBody: &openapi3.RequestBody{}, +// }, +// want: &openapi3.RequestBody{}, +// }, +// { +// name: "multiple contents", +// args: args{ +// requestBody: openapi3.NewRequestBody(). +// WithSchemaRef(openapi3.NewSchemaRef("ref-string", +// openapi3.NewObjectSchema().WithItems(openapi3.NewStringSchema())), +// []string{"content1", "content2"}), +// }, +// want: openapi3.NewRequestBody(). +// WithSchemaRef(openapi3.NewSchemaRef("", +// openapi3.NewObjectSchema().WithItems(openapi3.NewStringSchema())), +// []string{"content1", "content2"}), +// }, +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// assert.DeepEqual(t, clearRefFromRequestBodyRef(tt.args.requestBody), tt.want, cmpopts.IgnoreUnexported(openapi3.Schema{}), cmpopts.IgnoreTypes(openapi3.ExtensionProps{})) +// }) +// } +//} +// +//func Test_clearRefFromResponseRef(t *testing.T) { +// type args struct { +// responseRef *openapi3.ResponseRef +// } +// tests := []struct { +// name string +// args args +// want *openapi3.ResponseRef +// }{ +// { +// name: "nil ResponseRef", +// args: args{ +// responseRef: nil, +// }, +// want: nil, +// }, +// { +// name: "empty ResponseRef", +// args: args{ +// responseRef: &openapi3.ResponseRef{}, +// }, +// want: &openapi3.ResponseRef{}, +// }, +// { +// name: "sanity ResponseRef", +// args: args{ +// responseRef: &openapi3.ResponseRef{ +// Ref: "ref-response", +// Value: openapi3.NewResponse(). +// WithJSONSchemaRef(openapi3.NewSchemaRef("ref-string", +// openapi3.NewObjectSchema().WithItems(openapi3.NewStringSchema()))), +// }, +// }, +// want: &openapi3.ResponseRef{ +// Ref: "", +// Value: openapi3.NewResponse(). +// WithJSONSchemaRef(openapi3.NewSchemaRef("", +// openapi3.NewObjectSchema().WithItems(openapi3.NewStringSchema()))), +// }, +// }, +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// assert.DeepEqual(t, clearRefFromResponseRef(tt.args.responseRef), tt.want, cmpopts.IgnoreUnexported(openapi3.Schema{}), cmpopts.IgnoreTypes(openapi3.ExtensionProps{})) +// }) +// } +//} +// +//func Test_clearRefFromResponse(t *testing.T) { +// type args struct { +// response *openapi3.Response +// } +// tests := []struct { +// name string +// args args +// want *openapi3.Response +// }{ +// { +// name: "nil response", +// args: args{ +// response: nil, +// }, +// want: nil, +// }, +// { +// name: "empty response", +// args: args{ +// response: openapi3.NewResponse(), +// }, +// want: openapi3.NewResponse(), +// }, +// { +// name: "multiple headers", +// args: args{ +// response: &openapi3.Response{ +// Headers: openapi3.Headers{ +// "header1": &openapi3.HeaderRef{ +// Ref: "header1-ref", +// Value: &openapi3.Header{ +// Parameter: *openapi3.NewHeaderParameter("test1"). +// WithSchema(&openapi3.Schema{ +// Properties: openapi3.Schemas{ +// "prop1": openapi3.NewSchemaRef("prop1-ref", openapi3.NewStringSchema()), +// }, +// }), +// }, +// }, +// "header2": &openapi3.HeaderRef{ +// Ref: "header2-ref", +// Value: &openapi3.Header{ +// Parameter: *openapi3.NewHeaderParameter("test2"). +// WithSchema(&openapi3.Schema{ +// Properties: openapi3.Schemas{ +// "prop2": openapi3.NewSchemaRef("prop2-ref", openapi3.NewStringSchema()), +// }, +// }), +// }, +// }, +// }, +// }, +// }, +// want: &openapi3.Response{ +// Headers: openapi3.Headers{ +// "header1": &openapi3.HeaderRef{ +// Ref: "", +// Value: &openapi3.Header{ +// Parameter: *openapi3.NewHeaderParameter("test1"). +// WithSchema(&openapi3.Schema{ +// Properties: openapi3.Schemas{ +// "prop1": openapi3.NewSchemaRef("", openapi3.NewStringSchema()), +// }, +// }), +// }, +// }, +// "header2": &openapi3.HeaderRef{ +// Ref: "", +// Value: &openapi3.Header{ +// Parameter: *openapi3.NewHeaderParameter("test2"). +// WithSchema(&openapi3.Schema{ +// Properties: openapi3.Schemas{ +// "prop2": openapi3.NewSchemaRef("", openapi3.NewStringSchema()), +// }, +// }), +// }, +// }, +// }, +// }, +// }, +// { +// name: "multiple contents", +// args: args{ +// response: &openapi3.Response{ +// Content: openapi3.Content{ +// "content1": openapi3.NewMediaType(). +// WithSchemaRef(openapi3.NewSchemaRef("ref1-string", +// openapi3.NewObjectSchema().WithItems(openapi3.NewStringSchema()))), +// "content2": openapi3.NewMediaType(). +// WithSchemaRef(openapi3.NewSchemaRef("ref2-int", +// openapi3.NewObjectSchema().WithItems(openapi3.NewInt64Schema()))), +// }, +// }, +// }, +// want: &openapi3.Response{ +// Content: openapi3.Content{ +// "content1": openapi3.NewMediaType(). +// WithSchemaRef(openapi3.NewSchemaRef("", +// openapi3.NewObjectSchema().WithItems(openapi3.NewStringSchema()))), +// "content2": openapi3.NewMediaType(). +// WithSchemaRef(openapi3.NewSchemaRef("", +// openapi3.NewObjectSchema().WithItems(openapi3.NewInt64Schema()))), +// }, +// }, +// }, +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// assert.DeepEqual(t, clearRefFromResponse(tt.args.response), tt.want, cmpopts.IgnoreUnexported(openapi3.Schema{}), cmpopts.IgnoreTypes(openapi3.ExtensionProps{})) +// }) +// } +//} +// +//func Test_clearRefFromMediaType(t *testing.T) { +// type args struct { +// mediaType *openapi3.MediaType +// } +// tests := []struct { +// name string +// args args +// want *openapi3.MediaType +// }{ +// { +// name: "nil mediaType", +// args: args{ +// mediaType: nil, +// }, +// want: nil, +// }, +// { +// name: "empty mediaType", +// args: args{ +// mediaType: openapi3.NewMediaType(), +// }, +// want: openapi3.NewMediaType(), +// }, +// { +// name: "sanity mediaType", +// args: args{ +// mediaType: openapi3.NewMediaType(). +// WithSchemaRef(openapi3.NewSchemaRef("ref", +// openapi3.NewArraySchema().WithItems(openapi3.NewUUIDSchema()))), +// }, +// want: openapi3.NewMediaType(). +// WithSchemaRef(openapi3.NewSchemaRef("", +// openapi3.NewArraySchema().WithItems(openapi3.NewUUIDSchema()))), +// }, +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// assert.DeepEqual(t, clearRefFromMediaType(tt.args.mediaType), tt.want, cmpopts.IgnoreUnexported(openapi3.Schema{}), cmpopts.IgnoreTypes(openapi3.ExtensionProps{})) +// }) +// } +//} +// +//func Test_clearRefFromHeaderRef(t *testing.T) { +// type args struct { +// headerRef *openapi3.HeaderRef +// } +// tests := []struct { +// name string +// args args +// want *openapi3.HeaderRef +// }{ +// { +// name: "nil headerRef", +// args: args{ +// headerRef: nil, +// }, +// want: nil, +// }, +// { +// name: "empty headerRef", +// args: args{ +// headerRef: &openapi3.HeaderRef{}, +// }, +// want: &openapi3.HeaderRef{}, +// }, +// { +// name: "sanity headerRef", +// args: args{ +// headerRef: &openapi3.HeaderRef{ +// Ref: "header-ref", +// Value: &openapi3.Header{ +// Parameter: *openapi3.NewHeaderParameter("test"). +// WithSchema(&openapi3.Schema{ +// Properties: openapi3.Schemas{ +// "prop": openapi3.NewSchemaRef("prop-ref", openapi3.NewStringSchema()), +// }, +// }), +// }, +// }, +// }, +// want: &openapi3.HeaderRef{ +// Ref: "", +// Value: &openapi3.Header{ +// Parameter: *openapi3.NewHeaderParameter("test"). +// WithSchema(&openapi3.Schema{ +// Properties: openapi3.Schemas{ +// "prop": openapi3.NewSchemaRef("", openapi3.NewStringSchema()), +// }, +// }), +// }, +// }, +// }, +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// assert.DeepEqual(t, clearRefFromHeaderRef(tt.args.headerRef), tt.want, cmpopts.IgnoreUnexported(openapi3.Schema{}), cmpopts.IgnoreTypes(openapi3.ExtensionProps{})) +// }) +// } +//} +// +//func Test_clearRefFromHeader(t *testing.T) { +// type args struct { +// header *openapi3.Header +// } +// tests := []struct { +// name string +// args args +// want *openapi3.Header +// }{ +// { +// name: "nil header", +// args: args{ +// header: nil, +// }, +// want: nil, +// }, +// { +// name: "empty header", +// args: args{ +// header: &openapi3.Header{}, +// }, +// want: &openapi3.Header{}, +// }, +// { +// name: "empty header param", +// args: args{ +// header: &openapi3.Header{ +// Parameter: openapi3.Parameter{}, +// }, +// }, +// want: &openapi3.Header{ +// Parameter: openapi3.Parameter{}, +// }, +// }, +// { +// name: "sanity header", +// args: args{ +// header: &openapi3.Header{ +// Parameter: *openapi3.NewHeaderParameter("test"). +// WithSchema(&openapi3.Schema{ +// Properties: openapi3.Schemas{ +// "prop": openapi3.NewSchemaRef("prop-ref", openapi3.NewStringSchema()), +// }, +// }), +// }, +// }, +// want: &openapi3.Header{ +// Parameter: *openapi3.NewHeaderParameter("test"). +// WithSchema(&openapi3.Schema{ +// Properties: openapi3.Schemas{ +// "prop": openapi3.NewSchemaRef("", openapi3.NewStringSchema()), +// }, +// }), +// }, +// }, +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// assert.DeepEqual(t, clearRefFromHeader(tt.args.header), tt.want, cmpopts.IgnoreUnexported(openapi3.Schema{}), cmpopts.IgnoreTypes(openapi3.ExtensionProps{})) +// }) +// } +//} +// +//func Test_clearRefFromParameterRef(t *testing.T) { +// type args struct { +// parameterRef *openapi3.ParameterRef +// } +// tests := []struct { +// name string +// args args +// want *openapi3.ParameterRef +// }{ +// { +// name: "nil parameterRef", +// args: args{ +// parameterRef: nil, +// }, +// want: nil, +// }, +// { +// name: "empty parameterRef", +// args: args{ +// parameterRef: &openapi3.ParameterRef{}, +// }, +// want: &openapi3.ParameterRef{}, +// }, +// { +// name: "sanity parameterRef with Schema", +// args: args{ +// parameterRef: &openapi3.ParameterRef{ +// Ref: "param-ref", +// Value: openapi3.NewHeaderParameter("test"). +// WithSchema(&openapi3.Schema{ +// Properties: openapi3.Schemas{ +// "prop": openapi3.NewSchemaRef("prop-ref", openapi3.NewStringSchema()), +// }, +// }), +// }, +// }, +// want: &openapi3.ParameterRef{ +// Ref: "", +// Value: openapi3.NewHeaderParameter("test"). +// WithSchema(&openapi3.Schema{ +// Properties: openapi3.Schemas{ +// "prop": openapi3.NewSchemaRef("", openapi3.NewStringSchema()), +// }, +// }), +// }, +// }, +// { +// name: "sanity parameterRef with multiple contents", +// args: args{ +// parameterRef: &openapi3.ParameterRef{ +// Ref: "param-ref", +// Value: &openapi3.Parameter{ +// Content: openapi3.Content{ +// "content1": openapi3.NewMediaType(). +// WithSchemaRef(openapi3.NewSchemaRef("ref-string", +// openapi3.NewObjectSchema().WithItems(openapi3.NewStringSchema()))), +// "content2": openapi3.NewMediaType(). +// WithSchemaRef(openapi3.NewSchemaRef("ref2-int", +// openapi3.NewObjectSchema().WithItems(openapi3.NewInt64Schema()))), +// }, +// }, +// }, +// }, +// want: &openapi3.ParameterRef{ +// Ref: "", +// Value: &openapi3.Parameter{ +// Content: openapi3.Content{ +// "content1": openapi3.NewMediaType(). +// WithSchemaRef(openapi3.NewSchemaRef("", +// openapi3.NewObjectSchema().WithItems(openapi3.NewStringSchema()))), +// "content2": openapi3.NewMediaType(). +// WithSchemaRef(openapi3.NewSchemaRef("", +// openapi3.NewObjectSchema().WithItems(openapi3.NewInt64Schema()))), +// }, +// }, +// }, +// }, +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// assert.DeepEqual(t, clearRefFromParameterRef(tt.args.parameterRef), tt.want, cmpopts.IgnoreUnexported(openapi3.Schema{}), cmpopts.IgnoreTypes(openapi3.ExtensionProps{})) +// }) +// } +//} +// +//func Test_clearRefFromSchemaRef(t *testing.T) { +// type args struct { +// schemaRef *openapi3.SchemaRef +// } +// tests := []struct { +// name string +// args args +// want *openapi3.SchemaRef +// }{ +// { +// name: "nil schemaRef", +// args: args{ +// schemaRef: nil, +// }, +// want: nil, +// }, +// { +// name: "empty schemaRef", +// args: args{ +// schemaRef: &openapi3.SchemaRef{}, +// }, +// want: &openapi3.SchemaRef{}, +// }, +// { +// name: "sanity schemaRef", +// args: args{ +// schemaRef: &openapi3.SchemaRef{ +// Ref: "param-ref", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "prop-ref", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "prop2-ref", +// Value: openapi3.NewStringSchema(), +// }), +// }), +// }, +// }, +// want: &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewStringSchema(), +// }), +// }), +// }, +// }, +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// assert.DeepEqual(t, clearRefFromSchemaRef(tt.args.schemaRef), tt.want, cmpopts.IgnoreUnexported(openapi3.Schema{}), cmpopts.IgnoreTypes(openapi3.ExtensionProps{})) +// }) +// } +//} +// +//func Test_clearRefFromSchema(t *testing.T) { +// type args struct { +// schema *openapi3.Schema +// } +// tests := []struct { +// name string +// args args +// want *openapi3.Schema +// }{ +// { +// name: "nil schema", +// args: args{ +// schema: nil, +// }, +// want: nil, +// }, +// { +// name: "empty schema", +// args: args{ +// schema: &openapi3.Schema{}, +// }, +// want: &openapi3.Schema{}, +// }, +// { +// name: "schema oneof", +// args: args{ +// schema: &openapi3.Schema{ +// OneOf: openapi3.SchemaRefs{ +// { +// Ref: "ref1", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "prop-ref", +// Value: openapi3.NewArraySchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "prop2-ref", +// Value: openapi3.NewStringSchema(), +// }), +// }), +// }, +// { +// Ref: "ref2", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "prop-ref", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "prop2-ref", +// Value: openapi3.NewStringSchema(), +// }), +// }), +// }, +// }, +// }, +// }, +// want: &openapi3.Schema{ +// OneOf: openapi3.SchemaRefs{ +// { +// Ref: "", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewArraySchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewStringSchema(), +// }), +// }), +// }, +// { +// Ref: "", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewStringSchema(), +// }), +// }), +// }, +// }, +// }, +// }, +// { +// name: "schema AnyOf", +// args: args{ +// schema: &openapi3.Schema{ +// AnyOf: openapi3.SchemaRefs{ +// { +// Ref: "ref1", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "prop-ref", +// Value: openapi3.NewArraySchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "prop2-ref", +// Value: openapi3.NewStringSchema(), +// }), +// }), +// }, +// { +// Ref: "ref2", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "prop-ref", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "prop2-ref", +// Value: openapi3.NewStringSchema(), +// }), +// }), +// }, +// }, +// }, +// }, +// want: &openapi3.Schema{ +// AnyOf: openapi3.SchemaRefs{ +// { +// Ref: "", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewArraySchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewStringSchema(), +// }), +// }), +// }, +// { +// Ref: "", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewStringSchema(), +// }), +// }), +// }, +// }, +// }, +// }, +// { +// name: "schema AllOf", +// args: args{ +// schema: &openapi3.Schema{ +// AllOf: openapi3.SchemaRefs{ +// { +// Ref: "ref1", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "prop-ref", +// Value: openapi3.NewArraySchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "prop2-ref", +// Value: openapi3.NewStringSchema(), +// }), +// }), +// }, +// { +// Ref: "ref2", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "prop-ref", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "prop2-ref", +// Value: openapi3.NewStringSchema(), +// }), +// }), +// }, +// }, +// }, +// }, +// want: &openapi3.Schema{ +// AllOf: openapi3.SchemaRefs{ +// { +// Ref: "", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewArraySchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewStringSchema(), +// }), +// }), +// }, +// { +// Ref: "", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewStringSchema(), +// }), +// }), +// }, +// }, +// }, +// }, +// { +// name: "schema Not", +// args: args{ +// schema: &openapi3.Schema{ +// Not: &openapi3.SchemaRef{ +// Ref: "ref1", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "prop-ref", +// Value: openapi3.NewArraySchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "prop2-ref", +// Value: openapi3.NewStringSchema(), +// }), +// }), +// }, +// }, +// }, +// want: &openapi3.Schema{ +// Not: &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewArraySchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewStringSchema(), +// }), +// }), +// }, +// }, +// }, +// { +// name: "schema Items", +// args: args{ +// schema: &openapi3.Schema{ +// Items: &openapi3.SchemaRef{ +// Ref: "ref1", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "prop-ref", +// Value: openapi3.NewArraySchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "prop2-ref", +// Value: openapi3.NewStringSchema(), +// }), +// }), +// }, +// }, +// }, +// want: &openapi3.Schema{ +// Items: &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewArraySchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewStringSchema(), +// }), +// }), +// }, +// }, +// }, +// { +// name: "schema Properties", +// args: args{ +// schema: &openapi3.Schema{ +// Properties: openapi3.Schemas{ +// "prop": openapi3.NewSchemaRef("ref", openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "prop-ref", +// Value: openapi3.NewStringSchema(), +// })), +// "prop2": openapi3.NewSchemaRef("ref2", openapi3.NewArraySchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "prop2-ref", +// Value: openapi3.NewStringSchema(), +// })), +// }, +// }, +// }, +// want: &openapi3.Schema{ +// Properties: openapi3.Schemas{ +// "prop": openapi3.NewSchemaRef("", openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewStringSchema(), +// })), +// "prop2": openapi3.NewSchemaRef("", openapi3.NewArraySchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewStringSchema(), +// })), +// }, +// }, +// }, +// { +// name: "schema AdditionalProperties", +// args: args{ +// schema: &openapi3.Schema{ +// AdditionalProperties: &openapi3.SchemaRef{ +// Ref: "ref1", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "prop-ref", +// Value: openapi3.NewArraySchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "prop2-ref", +// Value: openapi3.NewStringSchema(), +// }), +// }), +// }, +// }, +// }, +// want: &openapi3.Schema{ +// AdditionalProperties: &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewArraySchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewStringSchema(), +// }), +// }), +// }, +// }, +// }, +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// assert.DeepEqual(t, clearRefFromSchema(tt.args.schema), tt.want, cmpopts.IgnoreUnexported(openapi3.Schema{}), cmpopts.IgnoreTypes(openapi3.ExtensionProps{})) +// }) +// } +//} +// +//func Test_clearRefFromSchemas(t *testing.T) { +// type args struct { +// schemas openapi3.Schemas +// } +// tests := []struct { +// name string +// args args +// want openapi3.Schemas +// }{ +// { +// name: "nil schemas", +// args: args{ +// schemas: nil, +// }, +// want: nil, +// }, +// { +// name: "empty schemas", +// args: args{ +// schemas: openapi3.Schemas{}, +// }, +// want: openapi3.Schemas{}, +// }, +// { +// name: "sanity schemas", +// args: args{ +// schemas: openapi3.Schemas{ +// "prop": openapi3.NewSchemaRef("ref1", openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "prop-ref", +// Value: openapi3.NewStringSchema(), +// })), +// "prop2": openapi3.NewSchemaRef("ref2", openapi3.NewArraySchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "prop2-ref", +// Value: openapi3.NewStringSchema(), +// })), +// }, +// }, +// want: openapi3.Schemas{ +// "prop": openapi3.NewSchemaRef("", openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewStringSchema(), +// })), +// "prop2": openapi3.NewSchemaRef("", openapi3.NewArraySchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewStringSchema(), +// })), +// }, +// }, +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// assert.DeepEqual(t, clearRefFromSchemas(tt.args.schemas), tt.want, cmpopts.IgnoreUnexported(openapi3.Schema{}), cmpopts.IgnoreTypes(openapi3.ExtensionProps{})) +// }) +// } +//} +// +//func Test_clearRefFromSchemaRefs(t *testing.T) { +// type args struct { +// schemaRefs openapi3.SchemaRefs +// } +// tests := []struct { +// name string +// args args +// want openapi3.SchemaRefs +// }{ +// { +// name: "nil schemaRefs", +// args: args{ +// schemaRefs: nil, +// }, +// want: nil, +// }, +// { +// name: "empty schemaRefs", +// args: args{ +// schemaRefs: openapi3.SchemaRefs{}, +// }, +// want: openapi3.SchemaRefs{}, +// }, +// { +// name: "sanity schemaRefs", +// args: args{ +// schemaRefs: openapi3.SchemaRefs{ +// { +// Ref: "ref1", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "prop-ref", +// Value: openapi3.NewArraySchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "prop2-ref", +// Value: openapi3.NewStringSchema(), +// }), +// }), +// }, +// { +// Ref: "ref2", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "prop-ref2", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "prop2-ref2", +// Value: openapi3.NewStringSchema(), +// }), +// }), +// }, +// }, +// }, +// want: openapi3.SchemaRefs{ +// { +// Ref: "", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewArraySchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewStringSchema(), +// }), +// }), +// }, +// { +// Ref: "", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewObjectSchema(). +// WithPropertyRef("prop2", &openapi3.SchemaRef{ +// Ref: "", +// Value: openapi3.NewStringSchema(), +// }), +// }), +// }, +// }, +// }, +// } +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// assert.DeepEqual(t, clearRefFromSchemaRefs(tt.args.schemaRefs), tt.want, cmpopts.IgnoreUnexported(openapi3.Schema{}), cmpopts.IgnoreTypes(openapi3.ExtensionProps{})) +// }) +// } +//} diff --git a/speculator/pkg/apispec/query.go b/speculator/pkg/apispec/query.go new file mode 100644 index 0000000..c30f114 --- /dev/null +++ b/speculator/pkg/apispec/query.go @@ -0,0 +1,38 @@ +package apispec + +import ( + "fmt" + "net/url" + + "github.com/getkin/kin-openapi/openapi3" +) + +func addQueryParam(operation *openapi3.Operation, key string, values []string) *openapi3.Operation { + operation.AddParameter(openapi3.NewQueryParameter(key).WithSchema(getSchemaFromQueryValues(values))) + return operation +} + +func getSchemaFromQueryValues(values []string) *openapi3.Schema { + var schema *openapi3.Schema + if len(values) == 0 || values[0] == "" { + schema = openapi3.NewBoolSchema() + schema.AllowEmptyValue = true + } else { + schema = getSchemaFromValues(values, true, openapi3.ParameterInQuery) + } + return schema +} + +func extractQueryParams(path string) (url.Values, error) { + _, query := GetPathAndQuery(path) + if query == "" { + return nil, nil + } + + values, err := url.ParseQuery(query) + if err != nil { + return nil, fmt.Errorf("failed to parse query: %v", err) + } + + return values, nil +} diff --git a/speculator/pkg/apispec/query_test.go b/speculator/pkg/apispec/query_test.go new file mode 100644 index 0000000..2a31b5e --- /dev/null +++ b/speculator/pkg/apispec/query_test.go @@ -0,0 +1,96 @@ +package apispec + +import ( + "net/url" + "reflect" + "testing" + + "github.com/getkin/kin-openapi/openapi3" +) + +func Test_extractQueryParams(t *testing.T) { + type args struct { + path string + } + tests := []struct { + name string + args args + want url.Values + wantErr bool + }{ + { + name: "no query params", + args: args{ + path: "path", + }, + want: nil, + wantErr: false, + }, + { + name: "no query params with ?", + args: args{ + path: "path?", + }, + want: nil, + wantErr: false, + }, + { + name: "with query params", + args: args{ + path: "path?foo=bar&foo=bar2", + }, + want: map[string][]string{"foo": {"bar", "bar2"}}, + wantErr: false, + }, + { + name: "invalid query params", + args: args{ + path: "path?foo%2", + }, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := extractQueryParams(tt.args.path) + if (err != nil) != tt.wantErr { + t.Errorf("extractQueryParams() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("extractQueryParams() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_addQueryParam(t *testing.T) { + type args struct { + operation *openapi3.Operation + key string + values []string + } + tests := []struct { + name string + args args + want *openapi3.Operation + }{ + { + name: "sanity", + args: args{ + operation: openapi3.NewOperation(), + key: "key", + values: []string{"val1"}, + }, + want: createTestOperation().WithParameter(openapi3.NewQueryParameter("key").WithSchema(openapi3.NewStringSchema())).Op, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := addQueryParam(tt.args.operation, tt.args.key, tt.args.values); !reflect.DeepEqual(got, tt.want) { + t.Errorf("addQueryParam() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/speculator/pkg/apispec/review.go b/speculator/pkg/apispec/review.go new file mode 100644 index 0000000..b7365c9 --- /dev/null +++ b/speculator/pkg/apispec/review.go @@ -0,0 +1,158 @@ +package apispec + +import ( + "fmt" + "strings" + + "github.com/getkin/kin-openapi/openapi3" + log "github.com/sirupsen/logrus" + + "github.com/5gsec/sentryflow/speculator/pkg/util" +) + +type SuggestedSpecReview struct { + PathItemsReview []*SuggestedSpecReviewPathItem + PathToPathItem map[string]*openapi3.PathItem +} + +type ApprovedSpecReview struct { + PathItemsReview []*ApprovedSpecReviewPathItem + PathToPathItem map[string]*openapi3.PathItem +} + +type ApprovedSpecReviewPathItem struct { + ReviewPathItem + PathUUID string +} + +type SuggestedSpecReviewPathItem struct { + ReviewPathItem +} + +type ReviewPathItem struct { + // ParameterizedPath represents the parameterized path grouping Paths + ParameterizedPath string + // Paths group of paths ParametrizedPath is representing + Paths map[string]bool +} + +// CreateSuggestedReview group all paths that have suspect parameter (with a certain template), +// into one path which is parameterized, and then add this path params to the spec. +func (s *Spec) CreateSuggestedReview() *SuggestedSpecReview { + s.lock.Lock() + defer s.lock.Unlock() + + ret := &SuggestedSpecReview{ + PathToPathItem: s.LearningSpec.PathItems, + } + + learningParametrizedPaths := s.createLearningParametrizedPaths() + + for parametrizedPath, paths := range learningParametrizedPaths.Paths { + pathReview := &SuggestedSpecReviewPathItem{} + pathReview.ParameterizedPath = parametrizedPath + + pathReview.Paths = paths + + ret.PathItemsReview = append(ret.PathItemsReview, pathReview) + } + return ret +} + +func (s *Spec) createLearningParametrizedPaths() *LearningParametrizedPaths { + var learningParametrizedPaths LearningParametrizedPaths + + learningParametrizedPaths.Paths = make(map[string]map[string]bool) + + for path := range s.LearningSpec.PathItems { + parameterizedPath := createParameterizedPath(path) + if _, ok := learningParametrizedPaths.Paths[parameterizedPath]; !ok { + learningParametrizedPaths.Paths[parameterizedPath] = make(map[string]bool) + } + learningParametrizedPaths.Paths[parameterizedPath][path] = true + } + return &learningParametrizedPaths +} + +func (s *Spec) ApplyApprovedReview(approvedReviews *ApprovedSpecReview, version OASVersion) error { + s.lock.Lock() + defer s.lock.Unlock() + + // first update the review into a copy of the state, in case the validation will fail + clonedSpec, err := s.SpecInfoClone() + if err != nil { + return fmt.Errorf("failed to clone spec. %v", err) + } + + for _, pathItemReview := range approvedReviews.PathItemsReview { + mergedPathItem := &openapi3.PathItem{} + for path := range pathItemReview.Paths { + pathItem, ok := approvedReviews.PathToPathItem[path] + if !ok { + log.Errorf("path: %v was not found in learning spec", path) + continue + } + mergedPathItem = MergePathItems(mergedPathItem, pathItem) + + // delete path from learning spec + delete(clonedSpec.LearningSpec.PathItems, path) + } + + addPathParamsToPathItem(mergedPathItem, pathItemReview.ParameterizedPath, pathItemReview.Paths) + + // add modified path and merged path item to ApprovedSpec + clonedSpec.ApprovedSpec.PathItems[pathItemReview.ParameterizedPath] = mergedPathItem + + // add the modified path to the path tree + isNewPath := clonedSpec.ApprovedPathTrie.Insert(pathItemReview.ParameterizedPath, pathItemReview.PathUUID) + if !isNewPath { + log.Warnf("Path was updated, a new path should be created in a normal case. path=%v, uuid=%v", pathItemReview.ParameterizedPath, pathItemReview.PathUUID) + } + + // populate SecuritySchemes from the approved merged path item + clonedSpec.ApprovedSpec.SecuritySchemes = updateSecuritySchemesFromPathItem(clonedSpec.ApprovedSpec.SecuritySchemes, mergedPathItem) + } + + if _, err := clonedSpec.GenerateOASJson(version); err != nil { + return fmt.Errorf("failed to generate Open API Spec. %w", err) + } + + clonedSpec.ApprovedSpec.SpecVersion = version + + s.SpecInfo = clonedSpec.SpecInfo + log.Debugf("Setting approved spec with version %q for %s:%s", s.ApprovedSpec.GetSpecVersion(), s.Host, s.Port) + + return nil +} + +func updateSecuritySchemesFromPathItem(sd openapi3.SecuritySchemes, item *openapi3.PathItem) openapi3.SecuritySchemes { + sd = updateSecuritySchemesFromOperation(sd, item.Get) + sd = updateSecuritySchemesFromOperation(sd, item.Put) + sd = updateSecuritySchemesFromOperation(sd, item.Post) + sd = updateSecuritySchemesFromOperation(sd, item.Delete) + sd = updateSecuritySchemesFromOperation(sd, item.Options) + sd = updateSecuritySchemesFromOperation(sd, item.Head) + sd = updateSecuritySchemesFromOperation(sd, item.Patch) + + return sd +} + +func addPathParamsToPathItem(pathItem *openapi3.PathItem, suggestedPath string, paths map[string]bool) { + // get all parameters names from path + suggestedPathTrimed := strings.TrimPrefix(suggestedPath, "/") + parts := strings.Split(suggestedPathTrimed, "/") + + for i, part := range parts { + if !util.IsPathParam(part) { + continue + } + + part = strings.TrimPrefix(part, util.ParamPrefix) + part = strings.TrimSuffix(part, util.ParamSuffix) + paramList := getOnlyIndexedPartFromPaths(paths, i) + paramInfo := createPathParam(part, getParamSchema(paramList)) + pathItem.Parameters = append(pathItem.Parameters, &openapi3.ParameterRef{ + Value: paramInfo.Parameter, + }) + } +} diff --git a/speculator/pkg/apispec/review_test.go b/speculator/pkg/apispec/review_test.go new file mode 100644 index 0000000..0574f45 --- /dev/null +++ b/speculator/pkg/apispec/review_test.go @@ -0,0 +1,904 @@ +package apispec + +import ( + "net/http" + "reflect" + "sort" + "sync" + "testing" + + "github.com/getkin/kin-openapi/openapi3" + "github.com/gofrs/uuid" + + "github.com/5gsec/sentryflow/speculator/pkg/pathtrie" +) + +var dataCombined = &HTTPInteractionData{ + ReqBody: combinedReq, + RespBody: combinedRes, + ReqHeaders: map[string]string{ + contentTypeHeaderName: mediaTypeApplicationJSON, + }, + RespHeaders: map[string]string{ + contentTypeHeaderName: mediaTypeApplicationJSON, + }, + statusCode: 200, +} + +func TestSpec_ApplyApprovedReview(t *testing.T) { + host := "host" + port := "8080" + uuidVar, _ := uuid.NewV4() + + type fields struct { + Host string + Port string + ID uuid.UUID + ApprovedSpec *ApprovedSpec + LearningSpec *LearningSpec + Mutex sync.Mutex + } + type args struct { + approvedReviews *ApprovedSpecReview + specVersion OASVersion + } + tests := []struct { + name string + fields fields + args args + wantSpec *Spec + wantErr bool + }{ + { + name: "1 reviewed path item. modified path param. same path item. 2 Paths", + fields: fields{ + Host: host, + Port: port, + ID: uuidVar, + ApprovedSpec: &ApprovedSpec{ + PathItems: map[string]*openapi3.PathItem{}, + }, + LearningSpec: &LearningSpec{ + PathItems: map[string]*openapi3.PathItem{ + "/api/1": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + "/api/2": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data2).Op).PathItem, + }, + }, + }, + args: args{ + specVersion: OASv3, + approvedReviews: &ApprovedSpecReview{ + PathToPathItem: map[string]*openapi3.PathItem{ + "/api/1": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + "/api/2": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data2).Op).PathItem, + }, + PathItemsReview: []*ApprovedSpecReviewPathItem{ + { + ReviewPathItem: ReviewPathItem{ + ParameterizedPath: "/api/{param1}", + Paths: map[string]bool{ + "/api/1": true, + "/api/2": true, + }, + }, + PathUUID: "1", + }, + }, + }, + }, + wantSpec: &Spec{ + SpecInfo: SpecInfo{ + ID: uuidVar, + Host: host, + Port: port, + ApprovedSpec: &ApprovedSpec{ + PathItems: map[string]*openapi3.PathItem{ + "/api/{param1}": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, dataCombined).Op). + WithPathParams("param1", openapi3.NewInt64Schema()).PathItem, + }, + SpecVersion: OASv3, + }, + LearningSpec: &LearningSpec{ + PathItems: map[string]*openapi3.PathItem{}, + }, + ApprovedPathTrie: createPathTrie(map[string]string{ + "/api/{param1}": "1", + }), + }, + }, + wantErr: false, + }, + { + name: "user took out one path out of the parameterized path, and also one more path has learned between review and approve (should ignore it and not delete)", + fields: fields{ + Host: host, + Port: port, + ID: uuidVar, + ApprovedSpec: &ApprovedSpec{ + PathItems: map[string]*openapi3.PathItem{}, + }, + LearningSpec: &LearningSpec{ + PathItems: map[string]*openapi3.PathItem{ + "api/3/foo": &NewTestPathItem().PathItem, + "/api/1": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + "/api/2": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data2).Op).PathItem, + }, + }, + }, + args: args{ + specVersion: OASv2, + approvedReviews: &ApprovedSpecReview{ + PathToPathItem: map[string]*openapi3.PathItem{ + "/api/1": &NewTestPathItem(). + WithOperation(http.MethodPost, NewOperation(t, Data).Op).PathItem, + "/api/2": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data2).Op).PathItem, + }, + PathItemsReview: []*ApprovedSpecReviewPathItem{ + { + ReviewPathItem: ReviewPathItem{ + ParameterizedPath: "/api/{param1}", + Paths: map[string]bool{ + "/api/2": true, + }, + }, + PathUUID: "1", + }, + { + ReviewPathItem: ReviewPathItem{ + ParameterizedPath: "/api/1", + Paths: map[string]bool{ + "/api/1": true, + }, + }, + PathUUID: "2", + }, + }, + }, + }, + wantSpec: &Spec{ + SpecInfo: SpecInfo{ + Host: host, + Port: port, + ID: uuidVar, + ApprovedSpec: &ApprovedSpec{ + PathItems: map[string]*openapi3.PathItem{ + "/api/{param1}": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data2).Op). + WithPathParams("param1", openapi3.NewInt64Schema()).PathItem, + "/api/1": &NewTestPathItem(). + WithOperation(http.MethodPost, NewOperation(t, Data).Op).PathItem, + }, + SpecVersion: OASv2, + }, + LearningSpec: &LearningSpec{ + PathItems: map[string]*openapi3.PathItem{ + "api/3/foo": &NewTestPathItem().PathItem, + }, + }, + ApprovedPathTrie: createPathTrie(map[string]string{ + "/api/{param1}": "1", + "/api/1": "2", + }), + }, + }, + wantErr: false, + }, + { + name: "multiple methods", + fields: fields{ + Host: host, + Port: port, + ID: uuidVar, + ApprovedSpec: &ApprovedSpec{ + PathItems: map[string]*openapi3.PathItem{}, + }, + LearningSpec: &LearningSpec{ + PathItems: map[string]*openapi3.PathItem{ + "/anything": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op). + WithOperation(http.MethodPost, NewOperation(t, Data).Op).PathItem, + "/headers": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + "/user-agent": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + }, + }, + }, + args: args{ + specVersion: OASv3, + approvedReviews: &ApprovedSpecReview{ + PathToPathItem: map[string]*openapi3.PathItem{ + "/anything": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op). + WithOperation(http.MethodPost, NewOperation(t, Data).Op).PathItem, + "/headers": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + "/user-agent": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + }, + PathItemsReview: []*ApprovedSpecReviewPathItem{ + { + ReviewPathItem: ReviewPathItem{ + ParameterizedPath: "/api/{test}", + Paths: map[string]bool{ + "/anything": true, + "/headers": true, + "/user-agent": true, + }, + }, + PathUUID: "1", + }, + }, + }, + }, + wantSpec: &Spec{ + SpecInfo: SpecInfo{ + Host: host, + Port: port, + ID: uuidVar, + ApprovedSpec: &ApprovedSpec{ + PathItems: map[string]*openapi3.PathItem{ + "/api/{test}": &NewTestPathItem(). + WithOperation(http.MethodPost, NewOperation(t, Data).Op). + WithOperation(http.MethodGet, NewOperation(t, Data).Op). + WithPathParams("test", openapi3.NewStringSchema()).PathItem, + }, + SpecVersion: OASv3, + }, + LearningSpec: &LearningSpec{ + PathItems: map[string]*openapi3.PathItem{}, + }, + ApprovedPathTrie: createPathTrie(map[string]string{ + "/api/{test}": "1", + }), + }, + }, + wantErr: false, + }, + { + name: "new parameterized path, unmerge of path item", + fields: fields{ + Host: host, + Port: port, + ID: uuidVar, + ApprovedSpec: &ApprovedSpec{ + PathItems: map[string]*openapi3.PathItem{}, + }, + LearningSpec: &LearningSpec{ + PathItems: map[string]*openapi3.PathItem{ + "/api/1": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + "/api/2": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + "/api/foo": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + "/user/1/bar/2": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + }, + }, + }, + args: args{ + specVersion: OASv3, + approvedReviews: &ApprovedSpecReview{ + PathToPathItem: map[string]*openapi3.PathItem{ + "/api/1": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + "/api/2": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + "/api/foo": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + "/user/1/bar/2": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + }, + PathItemsReview: []*ApprovedSpecReviewPathItem{ + { + ReviewPathItem: ReviewPathItem{ + ParameterizedPath: "/api/{param1}", + Paths: map[string]bool{ + "/api/1": true, + "/api/2": true, + }, + }, + PathUUID: "1", + }, + { + ReviewPathItem: ReviewPathItem{ + ParameterizedPath: "/api/foo", + Paths: map[string]bool{ + "/api/foo": true, + }, + }, + PathUUID: "2", + }, + { + ReviewPathItem: ReviewPathItem{ + ParameterizedPath: "/user/{param1}/bar/{param2}", + Paths: map[string]bool{ + "/user/1/bar/2": true, + }, + }, + PathUUID: "3", + }, + }, + }, + }, + wantSpec: &Spec{ + SpecInfo: SpecInfo{ + Host: host, + Port: port, + ID: uuidVar, + ApprovedSpec: &ApprovedSpec{ + PathItems: map[string]*openapi3.PathItem{ + "/api/{param1}": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op). + WithPathParams("param1", openapi3.NewInt64Schema()).PathItem, + "/api/foo": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + "/user/{param1}/bar/{param2}": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op). + WithPathParams("param1", openapi3.NewInt64Schema()). + WithPathParams("param2", openapi3.NewInt64Schema()).PathItem, + }, + SpecVersion: OASv3, + }, + LearningSpec: &LearningSpec{ + PathItems: map[string]*openapi3.PathItem{}, + }, + ApprovedPathTrie: createPathTrie(map[string]string{ + "/api/{param1}": "1", + "/api/foo": "2", + "/user/{param1}/bar/{param2}": "3", + }), + }, + }, + wantErr: false, + }, + { + name: "new parameterized path, unmerge of path item with security", + fields: fields{ + Host: host, + Port: port, + ID: uuidVar, + ApprovedSpec: &ApprovedSpec{ + PathItems: map[string]*openapi3.PathItem{}, + SecuritySchemes: openapi3.SecuritySchemes{}, + }, + LearningSpec: &LearningSpec{ + PathItems: map[string]*openapi3.PathItem{ + "/api/1": &NewTestPathItem().WithOperation(http.MethodGet, + NewOperation(t, Data).WithSecurityRequirement(openapi3.SecurityRequirement{BasicAuthSecuritySchemeKey: {}}).Op).PathItem, + "/api/2": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + "/api/foo": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + "/user/1/bar/2": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + }, + }, + }, + args: args{ + specVersion: OASv3, + approvedReviews: &ApprovedSpecReview{ + PathToPathItem: map[string]*openapi3.PathItem{ + "/api/1": &NewTestPathItem().WithOperation(http.MethodGet, + NewOperation(t, Data).WithSecurityRequirement(openapi3.SecurityRequirement{BasicAuthSecuritySchemeKey: {}}).Op).PathItem, + "/api/2": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + "/api/foo": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + "/user/1/bar/2": &NewTestPathItem().WithOperation(http.MethodGet, + NewOperation(t, Data).WithSecurityRequirement(openapi3.SecurityRequirement{OAuth2SecuritySchemeKey: {}}).Op).PathItem, + }, + PathItemsReview: []*ApprovedSpecReviewPathItem{ + { + ReviewPathItem: ReviewPathItem{ + ParameterizedPath: "/api/{param1}", + Paths: map[string]bool{ + "/api/1": true, + "/api/2": true, + }, + }, + PathUUID: "1", + }, + { + ReviewPathItem: ReviewPathItem{ + ParameterizedPath: "/api/foo", + Paths: map[string]bool{ + "/api/foo": true, + }, + }, + PathUUID: "2", + }, + { + ReviewPathItem: ReviewPathItem{ + ParameterizedPath: "/user/{param1}/bar/{param2}", + Paths: map[string]bool{ + "/user/1/bar/2": true, + }, + }, + PathUUID: "3", + }, + }, + }, + }, + wantSpec: &Spec{ + SpecInfo: SpecInfo{ + Host: host, + Port: port, + ID: uuidVar, + ApprovedSpec: &ApprovedSpec{ + PathItems: map[string]*openapi3.PathItem{ + "/api/{param1}": &NewTestPathItem().WithOperation(http.MethodGet, + NewOperation(t, Data). + WithSecurityRequirement(openapi3.SecurityRequirement{BasicAuthSecuritySchemeKey: {}}).Op). + WithPathParams("param1", openapi3.NewInt64Schema()).PathItem, + "/api/foo": &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + "/user/{param1}/bar/{param2}": &NewTestPathItem().WithOperation(http.MethodGet, + NewOperation(t, Data). + WithSecurityRequirement(openapi3.SecurityRequirement{OAuth2SecuritySchemeKey: {}}).Op). + WithPathParams("param1", openapi3.NewInt64Schema()). + WithPathParams("param2", openapi3.NewInt64Schema()).PathItem, + }, + SecuritySchemes: openapi3.SecuritySchemes{ + BasicAuthSecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewBasicAuthSecurityScheme()}, + OAuth2SecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewOAuth2SecurityScheme(nil)}, + }, + SpecVersion: OASv3, + }, + LearningSpec: &LearningSpec{ + PathItems: map[string]*openapi3.PathItem{}, + }, + ApprovedPathTrie: createPathTrie(map[string]string{ + "/api/{param1}": "1", + "/api/foo": "2", + "/user/{param1}/bar/{param2}": "3", + }), + }, + }, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &Spec{ + SpecInfo: SpecInfo{ + Host: tt.fields.Host, + Port: tt.fields.Port, + ID: tt.fields.ID, + ApprovedSpec: tt.fields.ApprovedSpec, + LearningSpec: tt.fields.LearningSpec, + ApprovedPathTrie: pathtrie.New(), + }, + } + err := s.ApplyApprovedReview(tt.args.approvedReviews, tt.args.specVersion) + if (err != nil) != tt.wantErr { + t.Errorf("Error response not as expected. want error: %v. error: %v", tt.wantErr, err) + return + } + + //assert.DeepEqual(t, s, tt.wantSpec, cmpopts.IgnoreUnexported(openapi3.Schema{}, Spec{}), cmpopts.IgnoreTypes(openapi3.ExtensionProps{})) + assertEqual(t, s, tt.wantSpec) + }) + } +} + +func TestSpec_CreateSuggestedReview(t *testing.T) { + type fields struct { + ID uuid.UUID + ApprovedSpec *ApprovedSpec + LearningSpec *LearningSpec + LearningParametrizedPaths *LearningParametrizedPaths + Mutex sync.Mutex + } + tests := []struct { + name string + fields fields + want *SuggestedSpecReview + }{ + { + name: "2 paths - map to one parameterized path", + fields: fields{ + ID: uuid.UUID{}, + LearningSpec: &LearningSpec{ + PathItems: map[string]*openapi3.PathItem{ + "/api/1": &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + "/api/2": &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data2).Op).PathItem, + }, + }, + LearningParametrizedPaths: &LearningParametrizedPaths{ + Paths: map[string]map[string]bool{ + "/api/{param1}": {"/api/1": true, "/api/2": true}, + }, + }, + }, + want: &SuggestedSpecReview{ + PathToPathItem: map[string]*openapi3.PathItem{ + "/api/1": &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + "/api/2": &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data2).Op).PathItem, + }, + PathItemsReview: []*SuggestedSpecReviewPathItem{ + { + ReviewPathItem: ReviewPathItem{ + ParameterizedPath: "/api/{param1}", + Paths: map[string]bool{ + "/api/1": true, + "/api/2": true, + }, + }, + }, + }, + }, + }, + { + name: "4 paths - 2 under one parameterized path with one param, one is not parameterized, one is parameterized path with 2 params", + fields: fields{ + ID: uuid.UUID{}, + LearningSpec: &LearningSpec{ + PathItems: map[string]*openapi3.PathItem{ + "/api/1": &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + "/api/2": &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data2).Op).PathItem, + "/api/foo": &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + "/api/foo/1/bar/2": &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + }, + }, + LearningParametrizedPaths: &LearningParametrizedPaths{ + Paths: map[string]map[string]bool{ + "/api/{param1}": {"/api/1": true, "/api/2": true}, + "/api/foo/{param1}/bar/{param2}": {"/api/foo/1/bar/2": true}, + "/api/foo": {"/api/foo": true}, + }, + }, + }, + want: &SuggestedSpecReview{ + PathToPathItem: map[string]*openapi3.PathItem{ + "/api/1": &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + "/api/2": &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data2).Op).PathItem, + "/api/foo": &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + "/api/foo/1/bar/2": &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + }, + PathItemsReview: []*SuggestedSpecReviewPathItem{ + { + ReviewPathItem: ReviewPathItem{ + ParameterizedPath: "/api/{param1}", + Paths: map[string]bool{ + "/api/1": true, + "/api/2": true, + }, + }, + }, + { + ReviewPathItem: ReviewPathItem{ + ParameterizedPath: "/api/foo/{param1}/bar/{param2}", + Paths: map[string]bool{ + "/api/foo/1/bar/2": true, + }, + }, + }, + { + ReviewPathItem: ReviewPathItem{ + ParameterizedPath: "/api/foo", + Paths: map[string]bool{ + "/api/foo": true, + }, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &Spec{ + SpecInfo: SpecInfo{ + ID: tt.fields.ID, + ApprovedSpec: tt.fields.ApprovedSpec, + LearningSpec: tt.fields.LearningSpec, + }, + } + got := s.CreateSuggestedReview() + sort.Slice(got.PathItemsReview, func(i, j int) bool { + return got.PathItemsReview[i].ParameterizedPath > got.PathItemsReview[j].ParameterizedPath + }) + sort.Slice(tt.want.PathItemsReview, func(i, j int) bool { + return tt.want.PathItemsReview[i].ParameterizedPath > tt.want.PathItemsReview[j].ParameterizedPath + }) + gotB := marshal(got) + wantB := marshal(tt.want) + if gotB != wantB { + t.Errorf("CreateSuggestedReview() got = %v, want %v", gotB, wantB) + } + }) + } +} + +func TestSpec_createLearningParametrizedPaths(t *testing.T) { + type fields struct { + Host string + ID uuid.UUID + ApprovedSpec *ApprovedSpec + LearningSpec *LearningSpec + } + tests := []struct { + name string + fields fields + want *LearningParametrizedPaths + }{ + { + name: "", + fields: fields{ + LearningSpec: &LearningSpec{ + PathItems: map[string]*openapi3.PathItem{ + "/api/1": &NewTestPathItem().PathItem, + }, + }, + }, + want: &LearningParametrizedPaths{ + Paths: map[string]map[string]bool{ + "/api/{param1}": {"/api/1": true}, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &Spec{ + SpecInfo: SpecInfo{ + Host: tt.fields.Host, + ID: tt.fields.ID, + ApprovedSpec: tt.fields.ApprovedSpec, + LearningSpec: tt.fields.LearningSpec, + }, + } + if got := s.createLearningParametrizedPaths(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("createLearningParametrizedPaths() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_addPathParamsToPathItem(t *testing.T) { + type args struct { + pathItem *openapi3.PathItem + suggestedPath string + paths map[string]bool + } + tests := []struct { + name string + args args + wantPathItem *openapi3.PathItem + }{ + { + name: "1 param", + args: args{ + pathItem: &NewTestPathItem().PathItem, + suggestedPath: "/api/{param1}/foo", + paths: map[string]bool{ + "api/1/foo": true, + "api/2/foo": true, + }, + }, + wantPathItem: &NewTestPathItem().WithPathParams("param1", openapi3.NewInt64Schema()).PathItem, + }, + { + name: "2 params", + args: args{ + pathItem: &NewTestPathItem().PathItem, + suggestedPath: "/api/{param1}/foo/{param2}", + paths: map[string]bool{ + "api/1/foo/2": true, + "api/2/foo/345": true, + }, + }, + wantPathItem: &NewTestPathItem(). + WithPathParams("param1", openapi3.NewInt64Schema()). + WithPathParams("param2", openapi3.NewInt64Schema()).PathItem, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + addPathParamsToPathItem(tt.args.pathItem, tt.args.suggestedPath, tt.args.paths) + assertEqual(t, tt.args.pathItem, tt.wantPathItem) + }) + } +} + +func Test_updateSecurityDefinitionsFromPathItem(t *testing.T) { + type args struct { + securitySchemes openapi3.SecuritySchemes + item *openapi3.PathItem + } + tests := []struct { + name string + args args + want openapi3.SecuritySchemes + }{ + { + name: "Get operation", + args: args{ + securitySchemes: openapi3.SecuritySchemes{}, + item: &openapi3.PathItem{ + Get: createOperationWithSecurity(&openapi3.SecurityRequirements{ + { + BasicAuthSecuritySchemeKey: {}, + }, + }), + }, + }, + want: openapi3.SecuritySchemes{ + BasicAuthSecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewBasicAuthSecurityScheme()}, + }, + }, + { + name: "Put operation", + args: args{ + securitySchemes: openapi3.SecuritySchemes{}, + item: &openapi3.PathItem{ + Put: createOperationWithSecurity(&openapi3.SecurityRequirements{ + { + OAuth2SecuritySchemeKey: {"admin"}, + }, + { + BasicAuthSecuritySchemeKey: {}, + }, + }), + }, + }, + want: openapi3.SecuritySchemes{ + OAuth2SecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewOAuth2SecurityScheme(nil)}, + BasicAuthSecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewBasicAuthSecurityScheme()}, + }, + }, + { + name: "Post operation", + args: args{ + securitySchemes: openapi3.SecuritySchemes{}, + item: &openapi3.PathItem{ + Post: createOperationWithSecurity(&openapi3.SecurityRequirements{ + { + OAuth2SecuritySchemeKey: {"admin"}, + }, + { + BasicAuthSecuritySchemeKey: {}, + }, + }), + }, + }, + want: openapi3.SecuritySchemes{ + OAuth2SecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewOAuth2SecurityScheme(nil)}, + BasicAuthSecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewBasicAuthSecurityScheme()}, + }, + }, + { + name: "Delete operation", + args: args{ + securitySchemes: openapi3.SecuritySchemes{}, + item: &openapi3.PathItem{ + Delete: createOperationWithSecurity(&openapi3.SecurityRequirements{ + { + OAuth2SecuritySchemeKey: {"admin"}, + }, + { + BasicAuthSecuritySchemeKey: {}, + }, + }), + }, + }, + want: openapi3.SecuritySchemes{ + OAuth2SecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewOAuth2SecurityScheme(nil)}, + BasicAuthSecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewBasicAuthSecurityScheme()}, + }, + }, + { + name: "Options operation", + args: args{ + securitySchemes: openapi3.SecuritySchemes{}, + item: &openapi3.PathItem{ + Options: createOperationWithSecurity(&openapi3.SecurityRequirements{ + { + OAuth2SecuritySchemeKey: {"admin"}, + }, + { + BasicAuthSecuritySchemeKey: {}, + }, + }), + }, + }, + want: openapi3.SecuritySchemes{ + OAuth2SecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewOAuth2SecurityScheme(nil)}, + BasicAuthSecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewBasicAuthSecurityScheme()}, + }, + }, + { + name: "Head operation", + args: args{ + securitySchemes: openapi3.SecuritySchemes{}, + item: &openapi3.PathItem{ + Head: createOperationWithSecurity(&openapi3.SecurityRequirements{ + { + OAuth2SecuritySchemeKey: {"admin"}, + }, + { + BasicAuthSecuritySchemeKey: {}, + }, + }), + }, + }, + want: openapi3.SecuritySchemes{ + OAuth2SecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewOAuth2SecurityScheme(nil)}, + BasicAuthSecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewBasicAuthSecurityScheme()}, + }, + }, + { + name: "Patch operation", + args: args{ + securitySchemes: openapi3.SecuritySchemes{}, + item: &openapi3.PathItem{ + Patch: createOperationWithSecurity(&openapi3.SecurityRequirements{ + { + OAuth2SecuritySchemeKey: {"admin"}, + }, + { + BasicAuthSecuritySchemeKey: {}, + }, + }), + }, + }, + want: openapi3.SecuritySchemes{ + OAuth2SecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewOAuth2SecurityScheme(nil)}, + BasicAuthSecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewBasicAuthSecurityScheme()}, + }, + }, + { + name: "Multiple operations", + args: args{ + securitySchemes: openapi3.SecuritySchemes{}, + item: &openapi3.PathItem{ + Get: createOperationWithSecurity(&openapi3.SecurityRequirements{ + { + BasicAuthSecuritySchemeKey: {}, + }, + }), + Put: createOperationWithSecurity(&openapi3.SecurityRequirements{ + { + OAuth2SecuritySchemeKey: {"read"}, + }, + }), + Post: createOperationWithSecurity(&openapi3.SecurityRequirements{ + { + "unsupported": {"read"}, + }, + }), + Delete: createOperationWithSecurity(&openapi3.SecurityRequirements{ + { + OAuth2SecuritySchemeKey: {"admin"}, + }, + { + BasicAuthSecuritySchemeKey: {}, + }, + }), + Options: createOperationWithSecurity(nil), + }, + }, + want: openapi3.SecuritySchemes{ + OAuth2SecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewOAuth2SecurityScheme(nil)}, + BasicAuthSecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewBasicAuthSecurityScheme()}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := updateSecuritySchemesFromPathItem(tt.args.securitySchemes, tt.args.item); !reflect.DeepEqual(got, tt.want) { + t.Errorf("updateSecuritySchemesFromPathItem() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/speculator/pkg/apispec/schema.go b/speculator/pkg/apispec/schema.go new file mode 100644 index 0000000..85d6f7f --- /dev/null +++ b/speculator/pkg/apispec/schema.go @@ -0,0 +1,146 @@ +package apispec + +import ( + "strconv" + "strings" + + "github.com/getkin/kin-openapi/openapi3" + log "github.com/sirupsen/logrus" +) + +var supportedQueryParamsSerializationStyles = []string{ + openapi3.SerializationForm, openapi3.SerializationSpaceDelimited, openapi3.SerializationPipeDelimited, +} + +var supportedHeaderParamsSerializationStyles = []string{ + openapi3.SerializationSimple, +} + +var supportedCookieParamsSerializationStyles = []string{ + openapi3.SerializationForm, +} + +// splitByStyle splits a string by a known style: +// +// form: comma separated value (default) +// spaceDelimited: space separated value +// pipeDelimited: pipe (|) separated value +// +// https://swagger.io/docs/specification/serialization/ +func splitByStyle(data, style string) []string { + if data == "" { + return nil + } + var sep string + switch style { + case openapi3.SerializationForm, openapi3.SerializationSimple: + sep = "," + case openapi3.SerializationSpaceDelimited: + sep = " " + case openapi3.SerializationPipeDelimited: + sep = "|" + default: + log.Warnf("Unsupported serialization style: %v", style) + return nil + } + var result []string + for _, s := range strings.Split(data, sep) { + if ts := strings.TrimSpace(s); ts != "" { + result = append(result, ts) + } + } + return result +} + +func getNewArraySchema(value string, paramInType string) (schema *openapi3.Schema, style string) { + var supportedSerializationStyles []string + + switch paramInType { + case openapi3.ParameterInHeader: + supportedSerializationStyles = supportedHeaderParamsSerializationStyles + case openapi3.ParameterInQuery: + supportedSerializationStyles = supportedQueryParamsSerializationStyles + case openapi3.ParameterInCookie: + supportedSerializationStyles = supportedCookieParamsSerializationStyles + default: + log.Errorf("Unsupported paramInType %v", paramInType) + return nil, "" + } + + for _, style = range supportedSerializationStyles { + byStyle := splitByStyle(value, style) + // Will create an array only if more than a single element exists + if len(byStyle) > 1 { + return getSchemaFromValues(byStyle, false, paramInType), style + } + } + + return nil, "" +} + +func getSchemaFromValues(values []string, shouldTryArraySchema bool, paramInType string) *openapi3.Schema { + valuesLen := len(values) + + if valuesLen == 0 { + return nil + } + + if valuesLen == 1 { + return getSchemaFromValue(values[0], shouldTryArraySchema, paramInType) + } + + // find the most common schema for the items type + return openapi3.NewArraySchema().WithItems(getCommonSchema(values, paramInType)) +} + +func getSchemaFromValue(value string, shouldTryArraySchema bool, paramInType string) *openapi3.Schema { + if isDateFormat(value) { + return openapi3.NewStringSchema() + } + + if shouldTryArraySchema { + schema, _ := getNewArraySchema(value, paramInType) + if schema != nil { + return schema + } + } + + // nolint:gomnd + if _, err := strconv.ParseInt(value, 10, 64); err == nil { + return openapi3.NewInt64Schema() + } + + // nolint:gomnd + if _, err := strconv.ParseFloat(value, 64); err == nil { + return openapi3.NewFloat64Schema() + } + + // TODO: not sure that `strconv.ParseBool` will do the job, it depends what is considers as boolean string + // The Go implementation for example uses `strconv.FormatBool(value)` ==> true/false + // But if we look at swag.ConvertBool - `checked` is evaluated as true so `unchecked` will be false? + // Also when using `strconv.ParseBool` 1 is considered as true so we must check for int before running it + if _, err := strconv.ParseBool(value); err == nil { + return openapi3.NewBoolSchema() + } + + return openapi3.NewStringSchema().WithFormat(getStringFormat(value)) +} + +func getCommonSchema(values []string, paramInType string) *openapi3.Schema { + var schemaType string + var schema *openapi3.Schema + + for _, value := range values { + schema = getSchemaFromValue(value, false, paramInType) + if schemaType == "" { + // first value, save schema type + schemaType = schema.Type.Slice()[0] + } else if schemaType != schema.Type.Slice()[0] { + // different schema type found, defaults to string schema + return openapi3.NewStringSchema() + } + } + + // identical schema type found + return schema +} diff --git a/speculator/pkg/apispec/schema_test.go b/speculator/pkg/apispec/schema_test.go new file mode 100644 index 0000000..3a6603f --- /dev/null +++ b/speculator/pkg/apispec/schema_test.go @@ -0,0 +1,84 @@ +package apispec + +import ( + "reflect" + "testing" + + "github.com/getkin/kin-openapi/openapi3" +) + +func Test_splitByStyle(t *testing.T) { + type args struct { + data string + style string + } + tests := []struct { + name string + args args + want []string + }{ + { + name: "empty data", + args: args{ + data: "", + style: "", + }, + want: nil, + }, + { + name: "unsupported serialization style", + args: args{ + data: "", + style: "Unsupported", + }, + want: nil, + }, + { + name: "SerializationForm", + args: args{ + data: "1, 2, 3", + style: openapi3.SerializationForm, + }, + want: []string{"1", "2", "3"}, + }, + { + name: "SerializationSimple", + args: args{ + data: "1, 2, 3", + style: openapi3.SerializationSimple, + }, + want: []string{"1", "2", "3"}, + }, + { + name: "SerializationSpaceDelimited", + args: args{ + data: "1 2 3", + style: openapi3.SerializationSpaceDelimited, + }, + want: []string{"1", "2", "3"}, + }, + { + name: "SerializationPipeDelimited", + args: args{ + data: "1|2|3", + style: openapi3.SerializationPipeDelimited, + }, + want: []string{"1", "2", "3"}, + }, + { + name: "SerializationPipeDelimited with empty space in the middle", + args: args{ + data: "1| |3", + style: openapi3.SerializationPipeDelimited, + }, + want: []string{"1", "3"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := splitByStyle(tt.args.data, tt.args.style); !reflect.DeepEqual(got, tt.want) { + t.Errorf("splitByStyle() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/speculator/pkg/apispec/schemas.go b/speculator/pkg/apispec/schemas.go new file mode 100644 index 0000000..7710eee --- /dev/null +++ b/speculator/pkg/apispec/schemas.go @@ -0,0 +1,157 @@ +package apispec + +import ( + "encoding/json" + "fmt" + "regexp" + "sort" + "strings" + + "github.com/getkin/kin-openapi/openapi3" + log "github.com/sirupsen/logrus" + "github.com/yudai/gojsondiff" +) + +const ( + schemasRefPrefix = "#/components/schemas/" + maxSchemaToRefDepth = 20 +) + +// will return a map of SchemaRef and update the operation accordingly. +func updateSchemas(schemas openapi3.Schemas, op *openapi3.Operation) (retSchemas openapi3.Schemas, retOperation *openapi3.Operation) { + if op == nil { + return schemas, op + } + + for i, response := range op.Responses.Map() { + if response.Value == nil { + continue + } + for content, mediaType := range response.Value.Content { + schemas, mediaType.Schema = schemaToRef(schemas, mediaType.Schema.Value, "", 0) + op.Responses.Value(i).Value.Content[content] = mediaType + } + } + + for i, parameter := range op.Parameters { + if parameter.Value == nil { + continue + } + for content, mediaType := range parameter.Value.Content { + schemas, mediaType.Schema = schemaToRef(schemas, mediaType.Schema.Value, "", 0) + op.Parameters[i].Value.Content[content] = mediaType + } + } + + if op.RequestBody != nil && op.RequestBody.Value != nil { + for content, mediaType := range op.RequestBody.Value.Content { + schemas, mediaType.Schema = schemaToRef(schemas, mediaType.Schema.Value, "", 0) + op.RequestBody.Value.Content[content] = mediaType + } + } + + return schemas, op +} + +func schemaToRef(schemas openapi3.Schemas, schema *openapi3.Schema, schemeNameHint string, depth int) (retSchemes openapi3.Schemas, schemaRef *openapi3.SchemaRef) { + if schema == nil { + return schemas, nil + } + + if depth >= maxSchemaToRefDepth { + log.Warnf("Maximum depth was reached") + return schemas, openapi3.NewSchemaRef("", schema) + } + + if schema.Type.Is(openapi3.TypeArray) { + if schema.Items == nil { + // no need to create definition for an empty array + return schemas, openapi3.NewSchemaRef("", schema) + } + // remove plural from def name hint when it's an array type (if exist) + schemas, schema.Items = schemaToRef(schemas, schema.Items.Value, strings.TrimSuffix(schemeNameHint, "s"), depth+1) + return schemas, openapi3.NewSchemaRef("", schema) + } + + if !schema.Type.Is(openapi3.TypeObject) { + return schemas, openapi3.NewSchemaRef("", schema) + } + + if schema.Properties == nil || len(schema.Properties) == 0 { + // no need to create ref for an empty object + return schemas, openapi3.NewSchemaRef("", schema) + } + + // go over all properties in the object and convert each one to ref if needed + var propNames []string + for propName := range schema.Properties { + var ref *openapi3.SchemaRef + schemas, ref = schemaToRef(schemas, schema.Properties[propName].Value, propName, depth+1) + if ref != nil { + schema.Properties[propName] = ref + propNames = append(propNames, propName) + } + } + + // look for schema in schemas with identical schema + schemeName, exist := findScheme(schemas, schema) + if !exist { + // generate new definition + schemeName = schemeNameHint + if schemeName == "" { + schemeName = generateDefNameFromPropNames(propNames) + } + if schemas == nil { + schemas = make(openapi3.Schemas) + } + if existingSchema, ok := schemas[schemeName]; ok { + log.Debugf("Security scheme name exist with different schema. existingSchema=%+v, schema=%+v", existingSchema, schema) + schemeName = getUniqueSchemeName(schemas, schemeName) + } + schemas[schemeName] = openapi3.NewSchemaRef("", schema) + } + + return schemas, openapi3.NewSchemaRef(schemasRefPrefix+schemeName, nil) +} + +func generateDefNameFromPropNames(propNames []string) string { + // generate name based on properties names when 'defNameHint' is missing + // sort the slice to get more stable test results + sort.Strings(propNames) + propString := strings.Join(propNames, "_") + return regexp.MustCompile(`[^a-zA-Z0-9._-]+`).ReplaceAllString(propString, "") +} + +func getUniqueSchemeName(schemes openapi3.Schemas, name string) string { + counter := 0 + for { + suggestedName := fmt.Sprintf("%s_%d", name, counter) + if _, ok := schemes[suggestedName]; !ok { + // found a unique name + return suggestedName + } + // suggestedName already exist - increase counter and look again + counter++ + } +} + +// will look for identical scheme in schemes map. +func findScheme(schemas openapi3.Schemas, schema *openapi3.Schema) (schemeName string, exist bool) { + schemaBytes, _ := json.Marshal(schema) + differ := gojsondiff.New() + for name, defSchema := range schemas { + defSchemaBytes, _ := json.Marshal(defSchema) + diff, err := differ.Compare(defSchemaBytes, schemaBytes) + if err != nil { + log.Errorf("Failed to compare schemas: %v", err) + continue + } + if !diff.Modified() { + log.Debugf("Schema was found in schemas. schema=%+v, def name=%v", schema, name) + return name, true + } + } + + log.Debugf("Schema was not found in schemas. schema=%+v", schema) + return "", false +} diff --git a/speculator/pkg/apispec/schemas_test.go b/speculator/pkg/apispec/schemas_test.go new file mode 100644 index 0000000..25ffebf --- /dev/null +++ b/speculator/pkg/apispec/schemas_test.go @@ -0,0 +1,579 @@ +package apispec + +import ( + "encoding/json" + "testing" + + "github.com/getkin/kin-openapi/openapi3" +) + +var ( + stringNumberObject = createObjectSchema(map[string]*openapi3.Schema{ + openapi3.TypeString: openapi3.NewStringSchema(), + openapi3.TypeNumber: openapi3.NewFloat64Schema(), + }) + stringBooleanObject = createObjectSchema(map[string]*openapi3.Schema{ + openapi3.TypeString: openapi3.NewStringSchema(), + openapi3.TypeBoolean: openapi3.NewBoolSchema(), + }) + stringIntegerObject = createObjectSchema(map[string]*openapi3.Schema{ + openapi3.TypeString: openapi3.NewStringSchema(), + openapi3.TypeInteger: openapi3.NewInt64Schema(), + }) +) + +func marshal(obj interface{}) string { + objB, _ := json.Marshal(obj) + return string(objB) +} + +func createObjectSchema(properties map[string]*openapi3.Schema) *openapi3.Schema { + return openapi3.NewObjectSchema().WithProperties(properties) +} + +func createObjectSchemaWithRef(properties map[string]*openapi3.SchemaRef) *openapi3.Schema { + objectSchema := openapi3.NewObjectSchema() + for name, ref := range properties { + objectSchema.WithPropertyRef(name, ref) + } + + return objectSchema +} + +func Test_findDefinition(t *testing.T) { + type args struct { + schemas openapi3.Schemas + schema *openapi3.Schema + } + tests := []struct { + name string + args args + wantDefName string + wantExist bool + }{ + { + name: "identical string schema exist", + args: args{ + schemas: openapi3.Schemas{ + "string": &openapi3.SchemaRef{Value: openapi3.NewStringSchema()}, + }, + schema: openapi3.NewStringSchema(), + }, + wantDefName: "string", + wantExist: true, + }, + { + name: "identical string schema does not exist", + args: args{ + schemas: openapi3.Schemas{ + "string": &openapi3.SchemaRef{Value: openapi3.NewStringSchema().WithFormat("format")}, + }, + schema: openapi3.NewStringSchema(), + }, + wantDefName: "", + wantExist: false, + }, + { + name: "identical object schema exist (object order is different)", + args: args{ + schemas: openapi3.Schemas{ + "object": &openapi3.SchemaRef{Value: openapi3.NewObjectSchema().WithProperties(map[string]*openapi3.Schema{ + openapi3.TypeObject: stringIntegerObject, + openapi3.TypeString: openapi3.NewStringSchema(), + })}, + }, + schema: createObjectSchema( + map[string]*openapi3.Schema{ + openapi3.TypeString: openapi3.NewStringSchema(), + openapi3.TypeObject: createObjectSchema( + map[string]*openapi3.Schema{ + openapi3.TypeInteger: openapi3.NewInt64Schema(), + openapi3.TypeString: openapi3.NewStringSchema(), + }, + ), + }, + ), + }, + wantDefName: "object", + wantExist: true, + }, + { + name: "identical object schema does not exist", + args: args{ + schemas: openapi3.Schemas{ + "object": &openapi3.SchemaRef{Value: openapi3.NewObjectSchema().WithProperties(map[string]*openapi3.Schema{ + openapi3.TypeString: openapi3.NewStringSchema(), + openapi3.TypeObject: stringIntegerObject, + })}, + }, + schema: createObjectSchema( + map[string]*openapi3.Schema{ + openapi3.TypeString: openapi3.NewStringSchema(), + openapi3.TypeObject: stringNumberObject, + }, + ), + }, + wantDefName: "", + wantExist: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotDefName, gotExist := findScheme(tt.args.schemas, tt.args.schema) + if gotDefName != tt.wantDefName { + t.Errorf("findScheme() gotDefName = %v, want %v", gotDefName, tt.wantDefName) + } + if gotExist != tt.wantExist { + t.Errorf("findScheme() gotExist = %v, want %v", gotExist, tt.wantExist) + } + }) + } +} + +func Test_getUniqueDefName(t *testing.T) { + type args struct { + schemas openapi3.Schemas + name string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "name does not exist", + args: args{ + schemas: openapi3.Schemas{ + "string": &openapi3.SchemaRef{Value: stringIntegerObject}, + }, + name: "no-test", + }, + want: "no-test_0", + }, + { + name: "name exist once", + args: args{ + schemas: openapi3.Schemas{ + "test_0": &openapi3.SchemaRef{Value: stringIntegerObject}, + }, + name: "test", + }, + want: "test_1", + }, + { + name: "name exist multiple times", + args: args{ + schemas: openapi3.Schemas{ + "test": &openapi3.SchemaRef{Value: stringIntegerObject}, + "test_0": &openapi3.SchemaRef{Value: stringNumberObject}, + "test_1": &openapi3.SchemaRef{Value: stringBooleanObject}, + }, + name: "test", + }, + want: "test_2", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getUniqueSchemeName(tt.args.schemas, tt.args.name); got != tt.want { + t.Errorf("getUniqueSchemeName() = %v, want %v", got, tt.want) + } + }) + } +} + +func createArraySchemaWithRefItems(name string) *openapi3.Schema { + arraySchemaWithRefItems := openapi3.NewArraySchema() + arraySchemaWithRefItems.Items = openapi3.NewSchemaRef(schemasRefPrefix+name, nil) + return arraySchemaWithRefItems +} + +func Test_schemaToRef(t *testing.T) { + arraySchemaWithNilItems := openapi3.NewArraySchema() + arraySchemaWithNilItems.Items = nil + type args struct { + schemas openapi3.Schemas + schema *openapi3.Schema + defNameHint string + depth int + } + tests := []struct { + name string + args args + wantRetSchemas openapi3.Schemas + wantRetSchema *openapi3.SchemaRef + }{ + { + name: "nil schema", + args: args{ + schemas: openapi3.Schemas{ + "test": &openapi3.SchemaRef{Value: openapi3.NewBoolSchema()}, + }, + schema: nil, + defNameHint: "", + }, + wantRetSchemas: openapi3.Schemas{ + "test": &openapi3.SchemaRef{Value: openapi3.NewBoolSchema()}, + }, + wantRetSchema: nil, + }, + { + name: "array schema with nil items", + args: args{ + schemas: openapi3.Schemas{ + "test": &openapi3.SchemaRef{Value: openapi3.NewBoolSchema()}, + }, + schema: openapi3.NewArraySchema(), + defNameHint: "", + }, + wantRetSchemas: openapi3.Schemas{ + "test": &openapi3.SchemaRef{Value: openapi3.NewBoolSchema()}, + }, + wantRetSchema: openapi3.NewSchemaRef("", openapi3.NewArraySchema()), + }, + { + name: "array schema with non object items - no change for schemas", + args: args{ + schemas: openapi3.Schemas{ + "test": &openapi3.SchemaRef{Value: openapi3.NewBoolSchema()}, + }, + schema: openapi3.NewArraySchema().WithItems(openapi3.NewBoolSchema()), + defNameHint: "", + }, + wantRetSchemas: openapi3.Schemas{ + "test": &openapi3.SchemaRef{Value: openapi3.NewBoolSchema()}, + }, + wantRetSchema: openapi3.NewSchemaRef("", openapi3.NewArraySchema().WithItems(openapi3.NewBoolSchema())), + }, + { + name: "array schema with object items - use hint name", + args: args{ + schemas: openapi3.Schemas{ + "test": &openapi3.SchemaRef{Value: openapi3.NewBoolSchema()}, + }, + schema: openapi3.NewArraySchema().WithItems(stringNumberObject), + defNameHint: "hint", + }, + wantRetSchemas: openapi3.Schemas{ + "test": &openapi3.SchemaRef{Value: openapi3.NewBoolSchema()}, + "hint": &openapi3.SchemaRef{Value: stringNumberObject}, + }, + wantRetSchema: openapi3.NewSchemaRef("", createArraySchemaWithRefItems("hint")), + }, + { + name: "array schema with object items - hint name already exist", + args: args{ + schemas: openapi3.Schemas{ + "hint": &openapi3.SchemaRef{Value: openapi3.NewBoolSchema()}, + }, + schema: openapi3.NewArraySchema().WithItems(stringNumberObject), + defNameHint: "hint", + }, + wantRetSchemas: openapi3.Schemas{ + "hint": &openapi3.SchemaRef{Value: openapi3.NewBoolSchema()}, + "hint_0": &openapi3.SchemaRef{Value: stringNumberObject}, + }, + wantRetSchema: openapi3.NewSchemaRef("", createArraySchemaWithRefItems("hint_0")), + }, + { + name: "primitive type", + args: args{ + schemas: openapi3.Schemas{ + "test": &openapi3.SchemaRef{Value: openapi3.NewBoolSchema()}, + }, + schema: openapi3.NewInt64Schema(), + }, + wantRetSchemas: openapi3.Schemas{ + "test": &openapi3.SchemaRef{Value: openapi3.NewBoolSchema()}, + }, + wantRetSchema: openapi3.NewSchemaRef("", openapi3.NewInt64Schema()), + }, + { + name: "empty object - no new schemas", + args: args{ + schemas: openapi3.Schemas{ + "test": &openapi3.SchemaRef{Value: stringNumberObject}, + }, + schema: openapi3.NewObjectSchema(), + }, + wantRetSchemas: openapi3.Schemas{ + "test": &openapi3.SchemaRef{Value: stringNumberObject}, + }, + wantRetSchema: openapi3.NewSchemaRef("", openapi3.NewObjectSchema()), + }, + { + name: "object - definition exist", + args: args{ + schemas: openapi3.Schemas{ + "test": &openapi3.SchemaRef{Value: stringNumberObject}, + }, + schema: stringNumberObject, + }, + wantRetSchemas: openapi3.Schemas{ + "test": &openapi3.SchemaRef{Value: stringNumberObject}, + }, + wantRetSchema: openapi3.NewSchemaRef(schemasRefPrefix+"test", nil), + }, + { + name: "object - definition does not exist", + args: args{ + schemas: openapi3.Schemas{ + "test": &openapi3.SchemaRef{Value: stringBooleanObject}, + }, + schema: stringNumberObject, + }, + wantRetSchemas: openapi3.Schemas{ + "test": &openapi3.SchemaRef{Value: stringBooleanObject}, + "number_string": &openapi3.SchemaRef{Value: stringNumberObject}, + }, + wantRetSchema: openapi3.NewSchemaRef(schemasRefPrefix+"number_string", nil), + }, + { + name: "object - definition does not exist - use hint", + args: args{ + schemas: openapi3.Schemas{ + "test": &openapi3.SchemaRef{Value: stringBooleanObject}, + }, + schema: stringNumberObject, + defNameHint: "hint", + }, + wantRetSchemas: openapi3.Schemas{ + "test": &openapi3.SchemaRef{Value: stringBooleanObject}, + "hint": &openapi3.SchemaRef{Value: stringNumberObject}, + }, + wantRetSchema: openapi3.NewSchemaRef(schemasRefPrefix+"hint", nil), + }, + { + name: "object in object", + args: args{ + schemas: nil, + schema: createObjectSchema( + map[string]*openapi3.Schema{ + openapi3.TypeString: openapi3.NewStringSchema(), + openapi3.TypeObject: stringNumberObject, + }, + ), + }, + wantRetSchemas: openapi3.Schemas{ + "object": openapi3.NewSchemaRef("", stringNumberObject), + "object_string": openapi3.NewSchemaRef("", createObjectSchemaWithRef( + map[string]*openapi3.SchemaRef{ + openapi3.TypeObject: openapi3.NewSchemaRef(schemasRefPrefix+"object", nil), + openapi3.TypeString: openapi3.NewSchemaRef("", openapi3.NewStringSchema()), + }, + )), + }, + wantRetSchema: openapi3.NewSchemaRef(schemasRefPrefix+"object_string", nil), + }, + { + name: "array of object in an object", + args: args{ + schemas: nil, + schema: createObjectSchema( + map[string]*openapi3.Schema{ + openapi3.TypeBoolean: openapi3.NewBoolSchema(), + + /*use plural to check the removal of the "s"*/ + "objects": openapi3.NewArraySchema().WithItems(stringNumberObject), + }, + ), + }, + wantRetSchemas: openapi3.Schemas{ + "object": openapi3.NewSchemaRef("", stringNumberObject), + "boolean_objects": openapi3.NewSchemaRef("", createObjectSchemaWithRef( + map[string]*openapi3.SchemaRef{ + openapi3.TypeBoolean: openapi3.NewSchemaRef("", openapi3.NewBoolSchema()), + "objects": openapi3.NewSchemaRef("", createArraySchemaWithRefItems("object")), + }, + )), + }, + wantRetSchema: openapi3.NewSchemaRef(schemasRefPrefix+"boolean_objects", nil), + }, + { + name: "object in object in object - max depth was reached after 1 object - ref was not created", + args: args{ + schemas: nil, + schema: createObjectSchema( + map[string]*openapi3.Schema{ + "obj1": createObjectSchema( + map[string]*openapi3.Schema{ + "obj2": stringNumberObject, + }, + ), + }, + ), + depth: maxSchemaToRefDepth - 1, + }, + wantRetSchemas: openapi3.Schemas{ + "obj1": openapi3.NewSchemaRef("", createObjectSchema( + map[string]*openapi3.Schema{ + "obj1": createObjectSchema( + map[string]*openapi3.Schema{ + "obj2": stringNumberObject, + }, + ), + }, + ), + ), + }, + wantRetSchema: openapi3.NewSchemaRef(schemasRefPrefix+"obj1", nil), + }, + { + name: "object in object in object - max depth was reached after 2 objects - ref was not created", + args: args{ + schemas: nil, + schema: createObjectSchema( + map[string]*openapi3.Schema{ + "obj1": createObjectSchema( + map[string]*openapi3.Schema{ + "obj2": stringNumberObject, + "string": openapi3.NewStringSchema(), + }, + ), + }, + ), + depth: maxSchemaToRefDepth - 2, + }, + wantRetSchemas: openapi3.Schemas{ + "obj1": openapi3.NewSchemaRef("", createObjectSchema( + map[string]*openapi3.Schema{ + "obj2": stringNumberObject, + "string": openapi3.NewStringSchema(), + }, + ), + ), + "obj1_0": openapi3.NewSchemaRef("", createObjectSchemaWithRef( + map[string]*openapi3.SchemaRef{ + "obj1": openapi3.NewSchemaRef(schemasRefPrefix+"obj1", nil), + }, + ), + ), + }, + wantRetSchema: openapi3.NewSchemaRef(schemasRefPrefix+"obj1_0", nil), + }, + { + name: "max depth was reached - ref was not created", + args: args{ + schemas: nil, + schema: createObjectSchema( + map[string]*openapi3.Schema{ + openapi3.TypeBoolean: openapi3.NewBoolSchema(), + + /*use plural to check the removal of the "s"*/ + "objects": openapi3.NewArraySchema().WithItems(stringNumberObject), + }, + ), + depth: maxSchemaToRefDepth, + }, + wantRetSchemas: nil, + wantRetSchema: openapi3.NewSchemaRef("", createObjectSchema( + map[string]*openapi3.Schema{ + openapi3.TypeBoolean: openapi3.NewBoolSchema(), + + /*use plural to check the removal of the "s"*/ + "objects": openapi3.NewArraySchema().WithItems(stringNumberObject), + }, + )), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotRetSchemas, gotRetSchema := schemaToRef(tt.args.schemas, tt.args.schema, tt.args.defNameHint, tt.args.depth) + assertEqual(t, gotRetSchemas, tt.wantRetSchemas) + assertEqual(t, gotRetSchema, tt.wantRetSchema) + }) + } +} + +var interactionReqBody = `{"active":true, +"certificateVersion":"86eb5278-676a-3b7c-b29d-4a57007dc7be", +"controllerInstanceInfo":{"replicaId":"portshift-agent-66fc77c848-tmmk8"}, +"policyAndAppVersion":1621477900361, +"version":"1.147.1"}` + +var interactionRespBody = `{"cvss":[{"score":7.8,"vector":"AV:L/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H"}]}` + +var interaction = &HTTPInteractionData{ + ReqBody: interactionReqBody, + RespBody: interactionRespBody, + ReqHeaders: map[string]string{ + contentTypeHeaderName: mediaTypeApplicationJSON, + }, + RespHeaders: map[string]string{ + contentTypeHeaderName: mediaTypeApplicationJSON, + }, + statusCode: 200, +} + +func createArraySchemaWithRef(ref string) *openapi3.Schema { + arraySchema := openapi3.NewArraySchema() + arraySchema.Items = &openapi3.SchemaRef{Ref: ref} + return arraySchema +} + +func Test_updateSchemas(t *testing.T) { + op := NewOperation(t, interaction).Op + retOp := NewOperation(t, interaction).Op + retOp.RequestBody = &openapi3.RequestBodyRef{Value: openapi3.NewRequestBody().WithJSONSchemaRef(&openapi3.SchemaRef{ + Ref: schemasRefPrefix + "active_certificateVersion_controllerInstanceInfo_policyAndAppVersion_version", + })} + retOp.Responses.Set("200", &openapi3.ResponseRef{ + Value: openapi3.NewResponse().WithDescription("response").WithJSONSchemaRef(&openapi3.SchemaRef{ + Ref: schemasRefPrefix + "cvss", + }), + }) + + type args struct { + schemas openapi3.Schemas + op *openapi3.Operation + } + tests := []struct { + name string + args args + wantRetSchemas openapi3.Schemas + wantRetOperation *openapi3.Operation + }{ + { + name: "sanity", + args: args{ + schemas: nil, + op: op, + }, + wantRetSchemas: openapi3.Schemas{ + "controllerInstanceInfo": openapi3.NewSchemaRef("", createObjectSchema( + map[string]*openapi3.Schema{ + "replicaId": openapi3.NewStringSchema(), + }, + )), + "active_certificateVersion_controllerInstanceInfo_policyAndAppVersion_version": openapi3.NewSchemaRef("", createObjectSchemaWithRef( + map[string]*openapi3.SchemaRef{ + "active": openapi3.NewSchemaRef("", openapi3.NewBoolSchema()), + "certificateVersion": openapi3.NewSchemaRef("", openapi3.NewUUIDSchema()), + "controllerInstanceInfo": openapi3.NewSchemaRef(schemasRefPrefix+"controllerInstanceInfo", nil), + "policyAndAppVersion": openapi3.NewSchemaRef("", openapi3.NewInt64Schema()), + "version": openapi3.NewSchemaRef("", openapi3.NewStringSchema()), + }, + )), + "cvs": openapi3.NewSchemaRef("", createObjectSchema( + map[string]*openapi3.Schema{ + "score": openapi3.NewFloat64Schema(), + "vector": openapi3.NewStringSchema(), + }, + )), + "cvss": openapi3.NewSchemaRef("", createObjectSchema( + map[string]*openapi3.Schema{ + "cvss": createArraySchemaWithRef(schemasRefPrefix + "cvs"), + }, + )), + }, + wantRetOperation: retOp, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotRetSchemas, gotRetOperation := updateSchemas(tt.args.schemas, tt.args.op) + assertEqual(t, gotRetSchemas, tt.wantRetSchemas) + assertEqual(t, gotRetOperation, tt.wantRetOperation) + }) + } +} diff --git a/speculator/pkg/apispec/security.go b/speculator/pkg/apispec/security.go new file mode 100644 index 0000000..268eae5 --- /dev/null +++ b/speculator/pkg/apispec/security.go @@ -0,0 +1,150 @@ +package apispec + +import ( + "github.com/getkin/kin-openapi/openapi3" + "github.com/golang-jwt/jwt/v5" + log "github.com/sirupsen/logrus" +) + +type OAuth2Claims struct { + Scope string `json:"scope"` + jwt.RegisteredClaims +} + +const ( + BasicAuthSecuritySchemeKey = "BasicAuth" + APIKeyAuthSecuritySchemeKey = "ApiKeyAuth" + OAuth2SecuritySchemeKey = "OAuth2" + BearerAuthSecuritySchemeKey = "BearerAuth" + + BearerAuthPrefix = "Bearer " + BasicAuthPrefix = "Basic " + + AccessTokenParamKey = "access_token" + + tknURL = "https://example.com/oauth2/token" + authorizationURL = "https://example.com/oauth2/authorize" + + apiKeyType = "apiKey" + basicAuthType = "http" + basicAuthScheme = "basic" + oauth2Type = "oauth2" +) + +// APIKeyNames is set of names of headers or query params defining API keys. +// This should be runtime configurable, of course. +// Note: keys should be lowercase. +var APIKeyNames = map[string]bool{ + "key": true, // Google + "api_key": true, +} + +func newAPIKeySecurityScheme(name string) *openapi3.SecurityScheme { + // https://swagger.io/docs/specification/authentication/api-keys/ + return &openapi3.SecurityScheme{ + Type: apiKeyType, + Name: name, + } +} + +func NewAPIKeySecuritySchemeInHeader(name string) *openapi3.SecurityScheme { + return newAPIKeySecurityScheme(name).WithIn(openapi3.ParameterInHeader) +} + +func NewAPIKeySecuritySchemeInQuery(name string) *openapi3.SecurityScheme { + return newAPIKeySecurityScheme(name).WithIn(openapi3.ParameterInQuery) +} + +func NewBasicAuthSecurityScheme() *openapi3.SecurityScheme { + // https://swagger.io/docs/specification/authentication/basic-authentication/ + return &openapi3.SecurityScheme{ + Type: basicAuthType, + Scheme: basicAuthScheme, + } +} + +func NewOAuth2SecurityScheme(scopes []string) *openapi3.SecurityScheme { + // https://swagger.io/docs/specification/authentication/oauth2/ + // we can't know the flow type (implicit, password, clientCredentials or authorizationCode) + // so we choose authorizationCode for now + return &openapi3.SecurityScheme{ + Type: oauth2Type, + Flows: &openapi3.OAuthFlows{ + AuthorizationCode: &openapi3.OAuthFlow{ + AuthorizationURL: authorizationURL, + TokenURL: tknURL, + Scopes: createOAuthFlowScopes(scopes, []string{}), + }, + }, + } +} + +func updateSecuritySchemesFromOperation(securitySchemes openapi3.SecuritySchemes, op *openapi3.Operation) openapi3.SecuritySchemes { + if op == nil || op.Security == nil { + return securitySchemes + } + + // Note: usage goes in the other direction; i.e., the security schemes do contain more detail, and operations + // (security requirements) reference those schemes. The reference is required to be valid (i.e., the + // name in the operation MUST be present in the security schemes) for OAuth openapi3 v2.0. Here we assume + // schemes are generic to push the operation's security requirements into the general security schemes. + for _, securityGroup := range *op.Security { + for key := range securityGroup { + var scheme *openapi3.SecurityScheme + switch key { + case BasicAuthSecuritySchemeKey: + scheme = NewBasicAuthSecurityScheme() + case OAuth2SecuritySchemeKey: + // we can't know the flow type (implicit, password, clientCredentials or authorizationCode) so + // we choose authorizationCode for now + scheme = NewOAuth2SecurityScheme(nil) + case BearerAuthSecuritySchemeKey: + scheme = openapi3.NewJWTSecurityScheme() + case APIKeyAuthSecuritySchemeKey: + // Use random key since it is not specified + for apiKeyName := range APIKeyNames { + scheme = NewAPIKeySecuritySchemeInHeader(apiKeyName) + break + } + default: + log.Warnf("Unsupported security definition key: %v", key) + } + securitySchemes = updateSecuritySchemes(securitySchemes, key, scheme) + } + } + + return securitySchemes +} + +func updateSecuritySchemes(securitySchemes openapi3.SecuritySchemes, key string, securityScheme *openapi3.SecurityScheme) openapi3.SecuritySchemes { + // we can override SecuritySchemes if exists since it has the same key and value + switch key { + case BasicAuthSecuritySchemeKey, OAuth2SecuritySchemeKey, APIKeyAuthSecuritySchemeKey: + securitySchemes[key] = &openapi3.SecuritySchemeRef{Value: securityScheme} + default: + log.Warnf("Unsupported security definition key: %v", key) + } + + return securitySchemes +} + +func createOAuthFlowScopes(scopes []string, descriptions []string) map[string]string { + flowScopes := make(map[string]string) + if len(descriptions) > 0 { + if len(descriptions) < len(scopes) { + log.Errorf("too few descriptions (%v) supplied for security scheme scopes (%v)", len(descriptions), len(scopes)) + } + for idx, scope := range scopes { + if idx < len(descriptions) { + flowScopes[scope] = descriptions[idx] + } else { + flowScopes[scope] = "" + } + } + } else { + for _, scope := range scopes { + flowScopes[scope] = "" + } + } + return flowScopes +} diff --git a/speculator/pkg/apispec/security_test.go b/speculator/pkg/apispec/security_test.go new file mode 100644 index 0000000..b55c068 --- /dev/null +++ b/speculator/pkg/apispec/security_test.go @@ -0,0 +1,127 @@ +package apispec + +import ( + "reflect" + "testing" + + "github.com/getkin/kin-openapi/openapi3" +) + +func createOperationWithSecurity(sec *openapi3.SecurityRequirements) *openapi3.Operation { + operation := openapi3.NewOperation() + operation.Security = sec + return operation +} + +func Test_updateSecurityDefinitionsFromOperation(t *testing.T) { + type args struct { + securitySchemes openapi3.SecuritySchemes + op *openapi3.Operation + } + tests := []struct { + name string + args args + want openapi3.SecuritySchemes + }{ + { + name: "OAuth2 OR BasicAuth", + args: args{ + securitySchemes: openapi3.SecuritySchemes{}, + op: createOperationWithSecurity(&openapi3.SecurityRequirements{ + { + OAuth2SecuritySchemeKey: {"admin"}, + }, + { + BasicAuthSecuritySchemeKey: {}, + }, + }), + }, + want: openapi3.SecuritySchemes{ + OAuth2SecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewOAuth2SecurityScheme(nil)}, + BasicAuthSecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewBasicAuthSecurityScheme()}, + }, + }, + { + name: "OAuth2 AND BasicAuth", + args: args{ + securitySchemes: openapi3.SecuritySchemes{}, + op: createOperationWithSecurity(&openapi3.SecurityRequirements{ + { + OAuth2SecuritySchemeKey: {"admin"}, + BasicAuthSecuritySchemeKey: {}, + }, + }), + }, + want: openapi3.SecuritySchemes{ + OAuth2SecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewOAuth2SecurityScheme(nil)}, + BasicAuthSecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewBasicAuthSecurityScheme()}, + }, + }, + { + name: "OAuth2 AND BasicAuth OR BasicAuth", + args: args{ + securitySchemes: openapi3.SecuritySchemes{}, + op: createOperationWithSecurity(&openapi3.SecurityRequirements{ + { + OAuth2SecuritySchemeKey: {"admin"}, + BasicAuthSecuritySchemeKey: {}, + }, + { + BasicAuthSecuritySchemeKey: {}, + }, + }), + }, + want: openapi3.SecuritySchemes{ + OAuth2SecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewOAuth2SecurityScheme(nil)}, + BasicAuthSecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewBasicAuthSecurityScheme()}, + }, + }, + { + name: "Unsupported SecurityDefinition key - no change to securitySchemes", + args: args{ + securitySchemes: openapi3.SecuritySchemes{ + OAuth2SecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewOAuth2SecurityScheme(nil)}, + }, + op: createOperationWithSecurity(&openapi3.SecurityRequirements{ + { + "unsupported": {"admin"}, + }, + }), + }, + want: openapi3.SecuritySchemes{ + OAuth2SecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewOAuth2SecurityScheme(nil)}, + }, + }, + { + name: "nil operation - no change to securitySchemes", + args: args{ + securitySchemes: openapi3.SecuritySchemes{ + OAuth2SecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewOAuth2SecurityScheme(nil)}, + }, + op: nil, + }, + want: openapi3.SecuritySchemes{ + OAuth2SecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewOAuth2SecurityScheme(nil)}, + }, + }, + { + name: "operation without security - no change to securitySchemes", + args: args{ + securitySchemes: openapi3.SecuritySchemes{ + OAuth2SecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewOAuth2SecurityScheme(nil)}, + }, + op: createOperationWithSecurity(nil), + }, + want: openapi3.SecuritySchemes{ + OAuth2SecuritySchemeKey: &openapi3.SecuritySchemeRef{Value: NewOAuth2SecurityScheme(nil)}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := updateSecuritySchemesFromOperation(tt.args.securitySchemes, tt.args.op); !reflect.DeepEqual(got, tt.want) { + t.Errorf("updateSecuritySchemesFromOperation() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/speculator/pkg/apispec/spec.go b/speculator/pkg/apispec/spec.go new file mode 100644 index 0000000..ae137ad --- /dev/null +++ b/speculator/pkg/apispec/spec.go @@ -0,0 +1,354 @@ +package apispec + +import ( + "context" + "encoding/json" + "fmt" + "sync" + + "github.com/getkin/kin-openapi/openapi2" + "github.com/getkin/kin-openapi/openapi2conv" + "github.com/getkin/kin-openapi/openapi3" + "github.com/ghodss/yaml" + "github.com/gofrs/uuid" + log "github.com/sirupsen/logrus" + + "github.com/5gsec/sentryflow/speculator/pkg/pathtrie" + "github.com/5gsec/sentryflow/speculator/pkg/util/errors" +) + +type SpecSource string + +const ( + SpecSourceReconstructed SpecSource = "RECONSTRUCTED" + SpecSourceProvided SpecSource = "PROVIDED" +) + +type Spec struct { + SpecInfo + + OpGenerator *OperationGenerator + + lock sync.Mutex +} + +type SpecInfo struct { + // Host of the spec + Host string + + Port string + // Spec ID + ID uuid.UUID + // Provided Spec + ProvidedSpec *ProvidedSpec + // Merged & approved state (can be generated into spec YAML) + ApprovedSpec *ApprovedSpec + // Upon learning, this will be updated (not the ApprovedSpec field) + LearningSpec *LearningSpec + + ApprovedPathTrie pathtrie.PathTrie + ProvidedPathTrie pathtrie.PathTrie +} + +type LearningParametrizedPaths struct { + // map parameterized paths into a list of paths included in it. + // e.g: /api/{param1} -> /api/1, /api/2 + // non parameterized path will map to itself + Paths map[string]map[string]bool +} + +type Telemetry struct { + DestinationAddress string `json:"destinationAddress,omitempty"` + DestinationNamespace string `json:"destinationNamespace,omitempty"` + Request *Request `json:"request,omitempty"` + RequestID string `json:"requestID,omitempty"` + Response *Response `json:"response,omitempty"` + Scheme string `json:"scheme,omitempty"` + SourceAddress string `json:"sourceAddress,omitempty"` +} + +type Request struct { + Common *Common `json:"common,omitempty"` + Host string `json:"host,omitempty"` + Method string `json:"method,omitempty"` + Path string `json:"path,omitempty"` +} + +type Response struct { + Common *Common `json:"common,omitempty"` + StatusCode string `json:"statusCode,omitempty"` +} + +type Common struct { + TruncatedBody bool `json:"TruncatedBody,omitempty"` + Body []byte `json:"body,omitempty"` + Headers []*Header `json:"headers"` + Version string `json:"version,omitempty"` +} + +type Header struct { + Key string `json:"key,omitempty"` + Value string `json:"value,omitempty"` +} + +func (s *Spec) HasApprovedSpec() bool { + if s.ApprovedSpec == nil || len(s.ApprovedSpec.PathItems) == 0 { + return false + } + + return true +} + +func (s *Spec) HasProvidedSpec() bool { + if s.ProvidedSpec == nil || s.ProvidedSpec.Doc == nil || s.ProvidedSpec.Doc.Paths == nil { + return false + } + + return true +} + +func (s *Spec) UnsetApprovedSpec() { + s.lock.Lock() + defer s.lock.Unlock() + + s.ApprovedSpec = &ApprovedSpec{ + PathItems: make(map[string]*openapi3.PathItem), + SecuritySchemes: make(openapi3.SecuritySchemes), + } + s.LearningSpec = &LearningSpec{ + PathItems: make(map[string]*openapi3.PathItem), + SecuritySchemes: make(openapi3.SecuritySchemes), + } + s.ApprovedPathTrie = pathtrie.New() +} + +func (s *Spec) UnsetProvidedSpec() { + s.lock.Lock() + defer s.lock.Unlock() + + s.ProvidedSpec = nil + s.ProvidedPathTrie = pathtrie.New() +} + +func (s *Spec) LearnTelemetry(telemetry *Telemetry) error { + s.lock.Lock() + defer s.lock.Unlock() + + method := telemetry.Request.Method + // remove query params if exists + path, _ := GetPathAndQuery(telemetry.Request.Path) + telemetryOp, err := s.telemetryToOperation(telemetry, s.LearningSpec.SecuritySchemes) + if err != nil { + return fmt.Errorf("failed to convert telemetry to operation. %v", err) + } + var existingOp *openapi3.Operation + + // Get existing path item or create a new one + pathItem := s.LearningSpec.GetPathItem(path) + if pathItem == nil { + pathItem = &openapi3.PathItem{} + } + + // Get existing operation of path item, and if exists, merge it with the operation learned from this interaction + existingOp = GetOperationFromPathItem(pathItem, method) + if existingOp != nil { + telemetryOp, _ = mergeOperation(existingOp, telemetryOp) + } + + // save Operation on the path item + AddOperationToPathItem(pathItem, method, telemetryOp) + + // add/update this path item in the spec + s.LearningSpec.AddPathItem(path, pathItem) + + return nil +} + +func (s *Spec) GetPathID(path string, specSource SpecSource) (string, error) { + s.lock.Lock() + defer s.lock.Unlock() + var specID string + + switch specSource { + case SpecSourceProvided: + if !s.HasProvidedSpec() { + log.Infof("No provided spec, path id will be empty") + return "", nil + } + basePath := s.ProvidedSpec.GetBasePath() + + pathNoBase := trimBasePathIfNeeded(basePath, path) + + _, value, found := s.ProvidedPathTrie.GetPathAndValue(pathNoBase) + if found { + if pathID, ok := value.(string); !ok { + log.Warnf("value is not a string. %v", value) + } else { + specID = pathID + } + } + case SpecSourceReconstructed: + if !s.HasApprovedSpec() { + log.Infof("No approved spec. path id will be empty") + return "", nil + } + _, value, found := s.ApprovedPathTrie.GetPathAndValue(path) + if found { + if pathID, ok := value.(string); !ok { + log.Warnf("value is not a string. %v", value) + } else { + specID = pathID + } + } + default: + return "", fmt.Errorf("spec source: %v is not valid", specSource) + } + return specID, nil +} + +func (s *Spec) GenerateOASYaml(version OASVersion) ([]byte, error) { + oasJSON, err := s.GenerateOASJson(version) + if err != nil { + return nil, fmt.Errorf("failed to generate json spec: %w", err) + } + + oasYaml, err := yaml.JSONToYAML(oasJSON) + if err != nil { + return nil, fmt.Errorf("failed to convert json to yaml: %v", err) + } + + return oasYaml, nil +} + +func (s *Spec) GenerateOASJson(version OASVersion) ([]byte, error) { + // yaml.Marshal does not omit empty fields + var schemas openapi3.Schemas + + clonedApprovedSpec, err := s.ApprovedSpec.Clone() + if err != nil { + return nil, fmt.Errorf("failed to clone approved spec. %v", err) + } + + clonedApprovedSpec.PathItems, schemas = reconstructObjectRefs(clonedApprovedSpec.PathItems) + + generatedSpec := &openapi3.T{ + OpenAPI: "3.0.3", + Components: &openapi3.Components{ + Schemas: schemas, + }, + Info: createDefaultSwaggerInfo(), + Paths: getPaths(clonedApprovedSpec.PathItems), + Servers: openapi3.Servers{ + { + // https://swagger.io/docs/specification/api-host-and-base-path/ + URL: "http://" + s.Host + ":" + s.Port, + }, + }, + } + + var ret []byte + if version == OASv2 { + log.Debugf("Generating OASv2 spec") + generatedSpecV2, err := openapi2conv.FromV3(generatedSpec) + if err != nil { + return nil, fmt.Errorf("failed to convert spec from v3: %v", err) + } + + ret, err = json.Marshal(generatedSpecV2) + if err != nil { + return nil, fmt.Errorf("failed to marshal the spec. %v", err) + } + } else { + log.Debugf("Generating OASv3 spec") + ret, err = json.Marshal(generatedSpec) + if err != nil { + return nil, fmt.Errorf("failed to marshal the spec. %v", err) + } + } + + if _, _, err = LoadAndValidateRawJSONSpec(ret); err != nil { + log.Errorf("Failed to validate the spec. %v\n\nspec: %s", err, ret) + return nil, fmt.Errorf("failed to validate the spec. %w", err) + } + + return ret, nil +} + +func getPaths(items map[string]*openapi3.PathItem) *openapi3.Paths { + paths := &openapi3.Paths{} + for path, item := range items { + paths.Set(path, item) + } + return paths +} + +func (s *Spec) SpecInfoClone() (*Spec, error) { + var clonedSpecInfo SpecInfo + + specB, err := json.Marshal(s.SpecInfo) + if err != nil { + return nil, fmt.Errorf("failed to marshal spec info: %w", err) + } + + if err := json.Unmarshal(specB, &clonedSpecInfo); err != nil { + return nil, fmt.Errorf("failed to unmarshal spec info: %w", err) + } + + return &Spec{ + SpecInfo: clonedSpecInfo, + lock: sync.Mutex{}, + }, nil +} + +func LoadAndValidateRawJSONSpecV3(spec []byte) (*openapi3.T, error) { + loader := openapi3.NewLoader() + loader.Context = context.TODO() + + doc, err := loader.LoadFromData(spec) + if err != nil { + return nil, fmt.Errorf("failed to load data: %s. %w", spec, err) + } + + err = doc.Validate(loader.Context) + if err != nil { + return nil, fmt.Errorf("spec validation failed. %v. %w", err, errors.ErrSpecValidation) + } + + return doc, nil +} + +func LoadAndValidateRawJSONSpecV3FromV2(spec []byte) (*openapi3.T, error) { + loader := openapi3.NewLoader() + loader.Context = context.TODO() + + var doc openapi2.T + if err := json.Unmarshal(spec, &doc); err != nil { + return nil, fmt.Errorf("provided spec is not valid. %w", err) + } + + v3, err := openapi2conv.ToV3(&doc) + if err != nil { + return nil, fmt.Errorf("conversion to V3 failed. %w", err) + } + + err = v3.Validate(loader.Context) + if err != nil { + return nil, fmt.Errorf("spec validation failed. %v. %w", err, errors.ErrSpecValidation) + } + + return v3, nil +} + +func reconstructObjectRefs(pathItems map[string]*openapi3.PathItem) (retPathItems map[string]*openapi3.PathItem, schemas openapi3.Schemas) { + for _, item := range pathItems { + schemas, item.Get = updateSchemas(schemas, item.Get) + schemas, item.Put = updateSchemas(schemas, item.Put) + schemas, item.Post = updateSchemas(schemas, item.Post) + schemas, item.Delete = updateSchemas(schemas, item.Delete) + schemas, item.Options = updateSchemas(schemas, item.Options) + schemas, item.Head = updateSchemas(schemas, item.Head) + schemas, item.Patch = updateSchemas(schemas, item.Patch) + } + + return pathItems, schemas +} diff --git a/speculator/pkg/apispec/spec_test.go b/speculator/pkg/apispec/spec_test.go new file mode 100644 index 0000000..bc3089a --- /dev/null +++ b/speculator/pkg/apispec/spec_test.go @@ -0,0 +1,281 @@ +package apispec + +import ( + "bytes" + "encoding/json" + "net/http" + "testing" + + "github.com/getkin/kin-openapi/openapi3" + "github.com/gofrs/uuid" + + "github.com/5gsec/sentryflow/speculator/pkg/pathtrie" +) + +func TestSpec_LearnTelemetry(t *testing.T) { + type fields struct{} + type args struct { + telemetries []*Telemetry + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "one", + fields: fields{}, + args: args{ + telemetries: []*Telemetry{ + { + RequestID: "req-id", + Scheme: "http", + Request: &Request{ + Method: "GET", + Path: "/some/path", + Host: "www.example.com", + Common: &Common{ + Version: "1", + Headers: []*Header{ + { + Key: contentTypeHeaderName, + Value: mediaTypeApplicationJSON, + }, + }, + Body: []byte(req1), + TruncatedBody: false, + }, + }, + Response: &Response{ + StatusCode: "200", + Common: &Common{ + Version: "1", + Headers: []*Header{ + { + Key: contentTypeHeaderName, + Value: mediaTypeApplicationJSON, + }, + }, + Body: []byte(res1), + TruncatedBody: false, + }, + }, + }, + }, + }, + wantErr: false, + }, + { + name: "two", + fields: fields{}, + args: args{ + telemetries: []*Telemetry{ + { + RequestID: "req-id", + Scheme: "http", + Request: &Request{ + Method: "GET", + Path: "/some/path", + Host: "www.example.com", + Common: &Common{ + Version: "1", + Body: []byte(req1), + Headers: []*Header{ + { + Key: contentTypeHeaderName, + Value: mediaTypeApplicationJSON, + }, + { + Key: "X-Test-Req-1", + Value: "req1", + }, + }, + TruncatedBody: false, + }, + }, + Response: &Response{ + StatusCode: "200", + Common: &Common{ + Version: "1", + Body: []byte(res1), + Headers: []*Header{ + { + Key: contentTypeHeaderName, + Value: mediaTypeApplicationJSON, + }, + { + Key: "X-Test-Res-1", + Value: "res1", + }, + }, + TruncatedBody: false, + }, + }, + }, + { + RequestID: "req-id", + Scheme: "http", + Request: &Request{ + Method: "GET", + Path: "/some/path", + Host: "www.example.com", + Common: &Common{ + Version: "1", + Body: []byte(req2), + Headers: []*Header{ + { + Key: contentTypeHeaderName, + Value: mediaTypeApplicationJSON, + }, + { + Key: "X-Test-Req-2", + Value: "req2", + }, + }, + TruncatedBody: false, + }, + }, + Response: &Response{ + StatusCode: "200", + Common: &Common{ + Version: "1", + Body: []byte(res2), + Headers: []*Header{ + { + Key: contentTypeHeaderName, + Value: mediaTypeApplicationJSON, + }, + { + Key: "X-Test-Res-2", + Value: "res2", + }, + }, + TruncatedBody: false, + }, + }, + }, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := CreateDefaultSpec("host", "80", testOperationGeneratorConfig) + for _, telemetry := range tt.args.telemetries { + // file, _ := json.MarshalIndent(telemetry, "", " ") + + //_ = ioutil.WriteFile(fmt.Sprintf("test%v.json", i), file, 0644) + if err := s.LearnTelemetry(telemetry); (err != nil) != tt.wantErr { + t.Errorf("LearnTelemetry() error = %v, wantErr %v", err, tt.wantErr) + } + } + }) + } +} + +func TestSpec_SpecInfoClone(t *testing.T) { + uuidVar, _ := uuid.NewV4() + pathTrie := pathtrie.New() + pathTrie.Insert("/api", 1) + + type fields struct { + Host string + Port string + ID uuid.UUID + ProvidedSpec *ProvidedSpec + ApprovedSpec *ApprovedSpec + LearningSpec *LearningSpec + ApprovedPathTrie pathtrie.PathTrie + ProvidedPathTrie pathtrie.PathTrie + } + tests := []struct { + name string + fields fields + want *Spec + wantErr bool + }{ + { + name: "clone spec", + fields: fields{ + Host: "host", + Port: "80", + ID: uuidVar, + ProvidedSpec: &ProvidedSpec{ + Doc: &openapi3.T{ + Info: createDefaultSwaggerInfo(), + Paths: createPath("/api", &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem), + }, + }, + ApprovedSpec: &ApprovedSpec{ + PathItems: map[string]*openapi3.PathItem{}, + }, + LearningSpec: &LearningSpec{ + PathItems: map[string]*openapi3.PathItem{ + "/api/1": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + "/api/2": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data2).Op).PathItem, + }, + }, + ApprovedPathTrie: pathTrie, + ProvidedPathTrie: pathTrie, + }, + want: &Spec{ + SpecInfo: SpecInfo{ + Host: "host", + Port: "80", + ID: uuidVar, + ProvidedSpec: &ProvidedSpec{ + Doc: &openapi3.T{ + Info: createDefaultSwaggerInfo(), + Paths: createPath("/api", &NewTestPathItem().WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem), + }, + }, + ApprovedSpec: &ApprovedSpec{ + PathItems: map[string]*openapi3.PathItem{}, + }, + LearningSpec: &LearningSpec{ + PathItems: map[string]*openapi3.PathItem{ + "/api/1": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data).Op).PathItem, + "/api/2": &NewTestPathItem(). + WithOperation(http.MethodGet, NewOperation(t, Data2).Op).PathItem, + }, + }, + ApprovedPathTrie: pathTrie, + ProvidedPathTrie: pathTrie, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &Spec{ + SpecInfo: SpecInfo{ + Host: tt.fields.Host, + Port: tt.fields.Port, + ID: tt.fields.ID, + ProvidedSpec: tt.fields.ProvidedSpec, + ApprovedSpec: tt.fields.ApprovedSpec, + LearningSpec: tt.fields.LearningSpec, + ApprovedPathTrie: tt.fields.ApprovedPathTrie, + ProvidedPathTrie: tt.fields.ProvidedPathTrie, + }, + } + got, err := s.SpecInfoClone() + if (err != nil) != tt.wantErr { + t.Errorf("SpecInfoClone() error = %v, wantErr %v", err, tt.wantErr) + return + } + gotB, _ := json.Marshal(got) + wantB, _ := json.Marshal(tt.want) + + if !bytes.Equal(gotB, wantB) { + t.Errorf("SpecInfoClone() got = %s, want %s", gotB, wantB) + } + }) + } +} diff --git a/speculator/pkg/apispec/spec_version.go b/speculator/pkg/apispec/spec_version.go new file mode 100644 index 0000000..1fb4af0 --- /dev/null +++ b/speculator/pkg/apispec/spec_version.go @@ -0,0 +1,57 @@ +package apispec + +import ( + "encoding/json" + "fmt" +) + +type OASVersion int64 + +const ( + Unknown OASVersion = iota + OASv2 + OASv3 +) + +func (o OASVersion) String() string { + switch o { + case Unknown: + return "Unknown" + case OASv2: + return "OASv2" + case OASv3: + return "OASv3" + } + return "Unknown" +} + +type oasV3header struct { + OpenAPI *string `json:"openapi" yaml:"openapi"` // Required +} + +type oasV2header struct { + Swagger *string `json:"swagger" yaml:"swagger"` +} + +func GetJSONSpecVersion(jsonSpec []byte) (OASVersion, error) { + var v3header oasV3header + if err := json.Unmarshal(jsonSpec, &v3header); err != nil { + return Unknown, fmt.Errorf("failed to unmarshel to v3header. %w", err) + } + + var v2header oasV2header + if err := json.Unmarshal(jsonSpec, &v2header); err != nil { + return Unknown, fmt.Errorf("failed to unmarshel to v2header. %w", err) + } + + // openapi field is required in the OpenAPI Specification + if v3header.OpenAPI != nil && *v3header.OpenAPI != "" { + return OASv3, nil + } + + if v2header.Swagger != nil { + return OASv2, nil + } + + return Unknown, fmt.Errorf("provided spec missing spec header") +} diff --git a/speculator/pkg/apispec/spec_version_test.go b/speculator/pkg/apispec/spec_version_test.go new file mode 100644 index 0000000..22d4788 --- /dev/null +++ b/speculator/pkg/apispec/spec_version_test.go @@ -0,0 +1,67 @@ +package apispec + +import ( + "testing" +) + +func TestGetJSONSpecVersion(t *testing.T) { + jsonSpecV2 := "{\n \"swagger\": \"2.0\",\n \"info\": {\n \"version\": \"1.0.0\",\n \"title\": \"APIClarity APIs\"\n },\n \"basePath\": \"/api\",\n \"schemes\": [\n \"http\"\n ],\n \"consumes\": [\n \"application/json\"\n ],\n \"produces\": [\n \"application/json\"\n ],\n \"paths\": {\n \"/dashboard/apiUsage/mostUsed\": {\n \"get\": {\n \"summary\": \"Get most used APIs\",\n \"responses\": {\n \"200\": {\n \"description\": \"Success\",\n \"schema\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n }\n },\n \"default\": {\n \"$ref\": \"#/responses/UnknownError\"\n }\n }\n }\n }\n },\n \"schemas\": {\n \"ApiResponse\": {\n \"description\": \"An object that is return in all cases of failures.\",\n \"type\": \"object\",\n \"properties\": {\n \"message\": {\n \"type\": \"string\"\n }\n }\n }\n },\n \"responses\": {\n \"UnknownError\": {\n \"description\": \"unknown error\",\n \"schema\": {\n \"$ref\": \"\"\n }\n }\n }\n}" + jsonSpecV2Invalid := "{\n \"info\": {\n \"version\": \"1.0.0\",\n \"title\": \"APIClarity APIs\"\n },\n \"basePath\": \"/api\",\n \"schemes\": [\n \"http\"\n ],\n \"consumes\": [\n \"application/json\"\n ],\n \"produces\": [\n \"application/json\"\n ],\n \"paths\": {\n \"/dashboard/apiUsage/mostUsed\": {\n \"get\": {\n \"summary\": \"Get most used APIs\",\n \"responses\": {\n \"200\": {\n \"description\": \"Success\",\n \"schema\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n }\n }\n },\n \"default\": {\n \"$ref\": \"#/responses/UnknownError\"\n }\n }\n }\n }\n },\n \"schemas\": {\n \"ApiResponse\": {\n \"description\": \"An object that is return in all cases of failures.\",\n \"type\": \"object\",\n \"properties\": {\n \"message\": {\n \"type\": \"string\"\n }\n }\n }\n },\n \"responses\": {\n \"UnknownError\": {\n \"description\": \"unknown error\",\n \"schema\": {\n \"$ref\": \"#/schemas/ApiResponse\"\n }\n }\n }\n}" + jsonSpecV3 := "{\n \"openapi\": \"3.0.3\",\n \"info\": {\n \"version\": \"1.0.0\",\n \"title\": \"Simple API\",\n \"description\": \"A simple API to illustrate OpenAPI concepts\"\n },\n \"servers\": [\n {\n \"url\": \"https://example.io/v1\"\n }\n ],\n \"security\": [\n {\n \"BasicAuth\": []\n }\n ],\n \"paths\": {\n \"/artists\": {\n \"get\": {\n \"description\": \"Returns a list of artists\",\n \"parameters\": [\n {\n \"name\": \"limit\",\n \"in\": \"query\",\n \"description\": \"Limits the number of items on a page\",\n \"schema\": {\n \"type\": \"integer\"\n }\n },\n {\n \"name\": \"offset\",\n \"in\": \"query\",\n \"description\": \"Specifies the page number of the artists to be displayed\",\n \"schema\": {\n \"type\": \"integer\"\n }\n }\n ],\n \"responses\": {\n \"200\": {\n \"description\": \"Successfully returned a list of artists\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"required\": [\n \"username\"\n ],\n \"properties\": {\n \"artist_name\": {\n \"type\": \"string\"\n },\n \"artist_genre\": {\n \"type\": \"string\"\n },\n \"albums_recorded\": {\n \"type\": \"integer\"\n },\n \"username\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n },\n \"400\": {\n \"description\": \"Invalid request\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"message\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n }\n },\n \"post\": {\n \"description\": \"Lets a user post a new artist\",\n \"requestBody\": {\n \"required\": true,\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"required\": [\n \"username\"\n ],\n \"properties\": {\n \"artist_name\": {\n \"type\": \"string\"\n },\n \"artist_genre\": {\n \"type\": \"string\"\n },\n \"albums_recorded\": {\n \"type\": \"integer\"\n },\n \"username\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n },\n \"responses\": {\n \"200\": {\n \"description\": \"Successfully created a new artist\"\n },\n \"400\": {\n \"description\": \"Invalid request\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"message\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n }\n }\n },\n \"/artists/{username}\": {\n \"get\": {\n \"description\": \"Obtain information about an artist from his or her unique username\",\n \"parameters\": [\n {\n \"name\": \"username\",\n \"in\": \"path\",\n \"required\": true,\n \"schema\": {\n \"type\": \"string\"\n }\n }\n ],\n \"responses\": {\n \"200\": {\n \"description\": \"Successfully returned an artist\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"artist_name\": {\n \"type\": \"string\"\n },\n \"artist_genre\": {\n \"type\": \"string\"\n },\n \"albums_recorded\": {\n \"type\": \"integer\"\n }\n }\n }\n }\n }\n },\n \"400\": {\n \"description\": \"Invalid request\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"message\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n }\n }\n }\n },\n \"components\": {\n \"securitySchemes\": {\n \"BasicAuth\": {\n \"type\": \"http\",\n \"scheme\": \"basic\"\n }\n }\n }\n}" + jsonSpecV3Invalid := "{\n \"openapi\": \"\",\n \"info\": {\n \"version\": \"1.0.0\",\n \"title\": \"Simple API\",\n \"description\": \"A simple API to illustrate OpenAPI concepts\"\n },\n \"servers\": [\n {\n \"url\": \"https://example.io/v1\"\n }\n ],\n \"security\": [\n {\n \"BasicAuth\": []\n }\n ],\n \"paths\": {\n \"/artists\": {\n \"get\": {\n \"description\": \"Returns a list of artists\",\n \"parameters\": [\n {\n \"name\": \"limit\",\n \"in\": \"query\",\n \"description\": \"Limits the number of items on a page\",\n \"schema\": {\n \"type\": \"integer\"\n }\n },\n {\n \"name\": \"offset\",\n \"in\": \"query\",\n \"description\": \"Specifies the page number of the artists to be displayed\",\n \"schema\": {\n \"type\": \"integer\"\n }\n }\n ],\n \"responses\": {\n \"200\": {\n \"description\": \"Successfully returned a list of artists\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"required\": [\n \"username\"\n ],\n \"properties\": {\n \"artist_name\": {\n \"type\": \"string\"\n },\n \"artist_genre\": {\n \"type\": \"string\"\n },\n \"albums_recorded\": {\n \"type\": \"integer\"\n },\n \"username\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n },\n \"400\": {\n \"description\": \"Invalid request\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"message\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n }\n },\n \"post\": {\n \"description\": \"Lets a user post a new artist\",\n \"requestBody\": {\n \"required\": true,\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"required\": [\n \"username\"\n ],\n \"properties\": {\n \"artist_name\": {\n \"type\": \"string\"\n },\n \"artist_genre\": {\n \"type\": \"string\"\n },\n \"albums_recorded\": {\n \"type\": \"integer\"\n },\n \"username\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n },\n \"responses\": {\n \"200\": {\n \"description\": \"Successfully created a new artist\"\n },\n \"400\": {\n \"description\": \"Invalid request\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"message\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n }\n }\n },\n \"/artists/{username}\": {\n \"get\": {\n \"description\": \"Obtain information about an artist from his or her unique username\",\n \"parameters\": [\n {\n \"name\": \"username\",\n \"in\": \"path\",\n \"required\": true,\n \"schema\": {\n \"type\": \"string\"\n }\n }\n ],\n \"responses\": {\n \"200\": {\n \"description\": \"Successfully returned an artist\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"artist_name\": {\n \"type\": \"string\"\n },\n \"artist_genre\": {\n \"type\": \"string\"\n },\n \"albums_recorded\": {\n \"type\": \"integer\"\n }\n }\n }\n }\n }\n },\n \"400\": {\n \"description\": \"Invalid request\",\n \"content\": {\n \"application/json\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"message\": {\n \"type\": \"string\"\n }\n }\n }\n }\n }\n }\n }\n }\n }\n },\n \"components\": {\n \"securitySchemes\": {\n \"BasicAuth\": {\n \"type\": \"http\",\n \"scheme\": \"basic\"\n }\n }\n }\n}" + + type args struct { + jsonSpec []byte + } + tests := []struct { + name string + args args + want OASVersion + wantErr bool + }{ + { + name: "valid v2 spec", + args: args{ + jsonSpec: []byte(jsonSpecV2), + }, + want: OASv2, + wantErr: false, + }, + { + name: "invalid v2 spec", + args: args{ + jsonSpec: []byte(jsonSpecV2Invalid), + }, + want: Unknown, + wantErr: true, + }, + { + name: "valid v3 spec", + args: args{ + jsonSpec: []byte(jsonSpecV3), + }, + want: OASv3, + wantErr: false, + }, + { + name: "invalid v3 spec", + args: args{ + jsonSpec: []byte(jsonSpecV3Invalid), + }, + want: Unknown, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GetJSONSpecVersion(tt.args.jsonSpec) + if (err != nil) != tt.wantErr { + t.Errorf("GetJSONSpecVersion() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("GetJSONSpecVersion() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/speculator/pkg/apispec/testing.go b/speculator/pkg/apispec/testing.go new file mode 100644 index 0000000..8fde762 --- /dev/null +++ b/speculator/pkg/apispec/testing.go @@ -0,0 +1,205 @@ +package apispec + +import ( + "bytes" + "encoding/json" + "net/http" + "testing" + + "github.com/getkin/kin-openapi/openapi3" +) + +var req1 = `{"active":true, +"certificateVersion":"86eb5278-676a-3b7c-b29d-4a57007dc7be", +"controllerInstanceInfo":{"replicaId":"portshift-agent-66fc77c848-tmmk8"}, +"policyAndAppVersion":1621477900361, +"version":"1.147.1"}` + +var res1 = `{"cvss":[{"score":7.8,"vector":"AV:L/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H"}]}` + +var req2 = `{"active":true,"statusCodes":["NO_METRICS_SERVER"],"version":"1.147.1"}` + +var res2 = `{"cvss":[{"version":"3"}]}` + +var combinedReq = `{"active":true,"statusCodes":["NO_METRICS_SERVER"], +"certificateVersion":"86eb5278-676a-3b7c-b29d-4a57007dc7be", +"controllerInstanceInfo":{"replicaId":"portshift-agent-66fc77c848-tmmk8"}, +"policyAndAppVersion":1621477900361, +"version":"1.147.1"}` + +var combinedRes = `{"cvss":[{"score":7.8,"vector":"AV:L/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H","version":"3"}]}` + +type TestSpec struct { + Doc *openapi3.T +} + +func (t *TestSpec) WithPathItem(path string, pathItem *openapi3.PathItem) *TestSpec { + t.Doc.Paths.Set(path, pathItem) + return t +} + +type TestPathItem struct { + PathItem openapi3.PathItem +} + +func NewTestPathItem() *TestPathItem { + return &TestPathItem{ + PathItem: openapi3.PathItem{}, + } +} + +func (t *TestPathItem) WithPathParams(name string, schema *openapi3.Schema) *TestPathItem { + pathParam := createPathParam(name, schema) + t.PathItem.Parameters = append(t.PathItem.Parameters, &openapi3.ParameterRef{Value: pathParam.Parameter}) + return t +} + +func (t *TestPathItem) WithOperation(method string, op *openapi3.Operation) *TestPathItem { + switch method { + case http.MethodGet: + t.PathItem.Get = op + case http.MethodDelete: + t.PathItem.Delete = op + case http.MethodOptions: + t.PathItem.Options = op + case http.MethodPatch: + t.PathItem.Patch = op + case http.MethodHead: + t.PathItem.Head = op + case http.MethodPost: + t.PathItem.Post = op + case http.MethodPut: + t.PathItem.Put = op + } + return t +} + +type TestOperation struct { + Op *openapi3.Operation +} + +func NewOperation(t *testing.T, data *HTTPInteractionData) *TestOperation { + t.Helper() + securitySchemes := openapi3.SecuritySchemes{} + operation, err := CreateTestNewOperationGenerator().GenerateSpecOperation(data, securitySchemes) + if err != nil { + t.Fatal(err) + } + return &TestOperation{ + Op: operation, + } +} + +func CreateTestNewOperationGenerator() *OperationGenerator { + return NewOperationGenerator(testOperationGeneratorConfig) +} + +var testOperationGeneratorConfig = OperationGeneratorConfig{ + ResponseHeadersToIgnore: []string{contentTypeHeaderName}, + RequestHeadersToIgnore: []string{acceptTypeHeaderName, authorizationTypeHeaderName, contentTypeHeaderName}, +} + +func (op *TestOperation) Deprecated() *TestOperation { + op.Op.Deprecated = true + return op +} + +func (op *TestOperation) WithResponse(status int, response *openapi3.Response) *TestOperation { + op.Op.AddResponse(status, response) + if status != 0 { + // we don't need it to create default response in tests unless we explicitly asked for (status == 0) + delete(op.Op.Responses.Map(), "default") + } + return op +} + +func (op *TestOperation) WithParameter(param *openapi3.Parameter) *TestOperation { + op.Op.AddParameter(param) + return op +} + +func (op *TestOperation) WithRequestBody(requestBody *openapi3.RequestBody) *TestOperation { + operationSetRequestBody(op.Op, requestBody) + return op +} + +func (op *TestOperation) WithSecurityRequirement(securityRequirement openapi3.SecurityRequirement) *TestOperation { + if op.Op.Security == nil { + op.Op.Security = openapi3.NewSecurityRequirements() + } + op.Op.Security.With(securityRequirement) + return op +} + +func createTestOperation() *TestOperation { + return &TestOperation{Op: openapi3.NewOperation()} +} + +type TestResponse struct { + *openapi3.Response +} + +func createTestResponse() *TestResponse { + return &TestResponse{ + Response: openapi3.NewResponse(), + } +} + +func (r *TestResponse) WithHeader(name string, schema *openapi3.Schema) *TestResponse { + if r.Response.Headers == nil { + r.Response.Headers = make(openapi3.Headers) + } + r.Response.Headers[name] = &openapi3.HeaderRef{ + Value: &openapi3.Header{ + Parameter: openapi3.Parameter{ + Schema: &openapi3.SchemaRef{ + Value: schema, + }, + }, + }, + } + return r +} + +func (r *TestResponse) WithJSONSchema(schema *openapi3.Schema) *TestResponse { + r.Response.WithJSONSchema(schema) + return r +} + +type TestResponses struct { + openapi3.Responses +} + +func createTestResponses() *TestResponses { + return &TestResponses{ + Responses: *openapi3.NewResponses(), + } +} + +func (r *TestResponses) WithResponse(code string, response *openapi3.Response) *TestResponses { + r.Responses.Set(code, &openapi3.ResponseRef{ + Value: response, + }) + + return r +} + +func assertEqual(t *testing.T, got any, want any) { + t.Helper() + + gotBytes, err := json.Marshal(got) + if err != nil { + t.Logf("failed to marshal got: %v", err) + t.FailNow() + } + + wantBytes, err := json.Marshal(want) + if err != nil { + t.Logf("failed to marshal want: %v", err) + t.FailNow() + } + + if !bytes.Equal(gotBytes, wantBytes) { + t.Errorf("%v()\nGOT = %v\nWANT %v", t.Name(), string(gotBytes), string(wantBytes)) + } +} diff --git a/speculator/pkg/apispec/utils.go b/speculator/pkg/apispec/utils.go new file mode 100644 index 0000000..b7b6c31 --- /dev/null +++ b/speculator/pkg/apispec/utils.go @@ -0,0 +1,57 @@ +package apispec + +import ( + "fmt" + "strconv" + "strings" + + "github.com/getkin/kin-openapi/openapi3" +) + +// Note: securityDefinitions might be updated. +func (s *Spec) telemetryToOperation(telemetry *Telemetry, securitySchemes openapi3.SecuritySchemes) (*openapi3.Operation, error) { + statusCode, err := strconv.Atoi(telemetry.Response.StatusCode) + if err != nil { + return nil, fmt.Errorf("failed to convert status code: %v. %v", statusCode, err) + } + + queryParams, err := extractQueryParams(telemetry.Request.Path) + if err != nil { + return nil, fmt.Errorf("failed to convert query params: %v", err) + } + + if s.OpGenerator == nil { + return nil, fmt.Errorf("operation generator was not set") + } + + // Generate operation from telemetry + telemetryOp, err := s.OpGenerator.GenerateSpecOperation(&HTTPInteractionData{ + ReqBody: string(telemetry.Request.Common.Body), + RespBody: string(telemetry.Response.Common.Body), + ReqHeaders: ConvertHeadersToMap(telemetry.Request.Common.Headers), + RespHeaders: ConvertHeadersToMap(telemetry.Response.Common.Headers), + QueryParams: queryParams, + statusCode: statusCode, + }, securitySchemes) + if err != nil { + return nil, fmt.Errorf("failed to generate spec operation. %v", err) + } + return telemetryOp, nil +} + +// example: for "/example-path?param=value" returns "/example-path", "param=value" +func GetPathAndQuery(fullPath string) (path, query string) { + index := strings.IndexByte(fullPath, '?') + if index == -1 { + return fullPath, "" + } + + // /path? + if index == (len(fullPath) - 1) { + return fullPath, "" + } + + path = fullPath[:index] + query = fullPath[index+1:] + return +} diff --git a/speculator/pkg/apispec/utils_test.go b/speculator/pkg/apispec/utils_test.go new file mode 100644 index 0000000..193cfa3 --- /dev/null +++ b/speculator/pkg/apispec/utils_test.go @@ -0,0 +1,61 @@ +package apispec + +import ( + "testing" +) + +func TestGetPathAndQuery(t *testing.T) { + type args struct { + fullPath string + } + tests := []struct { + name string + args args + wantPath string + wantQuery string + }{ + { + name: "no query params", + args: args{ + fullPath: "/path", + }, + wantPath: "/path", + wantQuery: "", + }, + { + name: "path with ? in last index", + args: args{ + fullPath: "/path?", + }, + wantPath: "/path?", + wantQuery: "", + }, + { + name: "path with query", + args: args{ + fullPath: "/path?query=param", + }, + wantPath: "/path", + wantQuery: "query=param", + }, + { + name: "path with query and several ?", + args: args{ + fullPath: "/path?query=param?stam=foo", + }, + wantPath: "/path", + wantQuery: "query=param?stam=foo", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotPath, gotQuery := GetPathAndQuery(tt.args.fullPath) + if gotPath != tt.wantPath { + t.Errorf("GetPathAndQuery() gotPath = %v, want %v", gotPath, tt.wantPath) + } + if gotQuery != tt.wantQuery { + t.Errorf("GetPathAndQuery() gotQuery = %v, want %v", gotQuery, tt.wantQuery) + } + }) + } +} diff --git a/speculator/pkg/config/config.go b/speculator/pkg/config/config.go new file mode 100644 index 0000000..41335c3 --- /dev/null +++ b/speculator/pkg/config/config.go @@ -0,0 +1,121 @@ +package config + +import ( + "encoding/json" + "fmt" + + "github.com/spf13/viper" + "go.uber.org/zap" + + "github.com/5gsec/sentryflow/speculator/pkg/util" +) + +type Exchange struct { + Name string `json:"name"` + Type string `json:"type"` + Durable bool `json:"durable,omitempty"` + AutoDelete bool `json:"autoDelete,omitempty"` +} + +type RabbitMQ struct { + Host string `json:"host"` + Port string `json:"port"` + User string `json:"user,omitempty"` + Password string `json:"password,omitempty"` + Exchange *Exchange `json:"exchange"` + QueueName string `json:"queueName"` +} + +type Database struct { + LogLevel string `json:"logLevel,omitempty"` + Uri string `json:"uri,omitempty"` + User string `json:"user"` + Password string `json:"password"` + Name string `json:"name,omitempty"` +} + +type Configuration struct { + RabbitMQ *RabbitMQ `json:"rabbitmq"` + Database *Database `json:"database"` +} + +func (c *Configuration) validate() error { + if c.RabbitMQ == nil { + return fmt.Errorf("configuration does not contain a valid RabbitMQ configuration") + } + if c.RabbitMQ.Host == "" { + return fmt.Errorf("configuration does not contain a valid RabbitMQ host") + } + if c.RabbitMQ.Port == "" || len(c.RabbitMQ.Port) > 5 { + return fmt.Errorf("configuration does not contain a valid RabbitMQ port") + } + if c.RabbitMQ.Exchange == nil { + return fmt.Errorf("configuration does not contain a valid RabbitMQ exchange") + } + if c.RabbitMQ.Exchange.Name == "" { + return fmt.Errorf("configuration does not contain a valid RabbitMQ exchange name") + } + if c.RabbitMQ.QueueName == "" { + return fmt.Errorf("configuration does not contain a valid RabbitMQ queue name") + } + + if c.Database == nil { + return fmt.Errorf("configuration does not contain a valid database configuration") + } + if c.Database.Uri == "" { + return fmt.Errorf("configuration does not contain a valid database URI") + } + if c.Database.User == "" { + return fmt.Errorf("configuration does not contain a valid database user") + } + if c.Database.Password == "" { + return fmt.Errorf("configuration does not contain a valid database password") + } + + return nil +} + +const DefaultConfigFilePath = "config/default.yaml" + +func New(configFilePath string, logger *zap.SugaredLogger) (*Configuration, error) { + if configFilePath == "" { + configFilePath = DefaultConfigFilePath + logger.Warnf("using default configfile path: %s", configFilePath) + } + + viper.SetConfigFile(configFilePath) + if err := viper.ReadInConfig(); err != nil { + return nil, fmt.Errorf("failed to read config file: %w", err) + } + + config := &Configuration{} + if err := viper.Unmarshal(config); err != nil { + return nil, fmt.Errorf("failed to unmarshal config file: %w", err) + } + + if err := config.validate(); err != nil { + return nil, err + } + + if config.Database.LogLevel == "" { + config.Database.LogLevel = util.LevelInfo + logger.Warnf("using default `INFO` database log level: %s", config.Database.LogLevel) + } + + dbUser := config.Database.User + dbPassword := config.Database.Password + + config.Database.User = "" + config.Database.Password = "" + + bytes, err := json.Marshal(config) + if err != nil { + logger.Errorf("failed to marshal config file: %v", err) + } + logger.Debugf("configuration: %s", string(bytes)) + + config.Database.User = dbUser + config.Database.Password = dbPassword + + return config, nil +} diff --git a/speculator/pkg/core/core.go b/speculator/pkg/core/core.go new file mode 100644 index 0000000..d96a712 --- /dev/null +++ b/speculator/pkg/core/core.go @@ -0,0 +1,45 @@ +package core + +import ( + "context" + + "go.uber.org/zap" + + "github.com/5gsec/sentryflow/speculator/pkg/config" + "github.com/5gsec/sentryflow/speculator/pkg/database" + "github.com/5gsec/sentryflow/speculator/pkg/util" +) + +type Manager struct { + Ctx context.Context + Logger *zap.SugaredLogger + DBHandler *database.Handler +} + +func Run(ctx context.Context, configFilePath string) { + mgr := &Manager{ + Ctx: ctx, + Logger: util.LoggerFromCtx(ctx), + } + + mgr.Logger.Info("starting speculator") + + _, err := config.New(configFilePath, mgr.Logger) + if err != nil { + mgr.Logger.Error(err) + return + } + + //dbHandler, err := database.New(mgr.Ctx, cfg.Database) + //if err != nil { + // mgr.Logger.Error(err) + // return + //} + //mgr.DBHandler = dbHandler + //defer func() { + // if err := mgr.DBHandler.Disconnect(); err != nil { + // mgr.Logger.Errorf("failed to disconnect to database: %v", err) + // } + //}() + +} diff --git a/speculator/pkg/database/database.go b/speculator/pkg/database/database.go new file mode 100644 index 0000000..fc1af42 --- /dev/null +++ b/speculator/pkg/database/database.go @@ -0,0 +1,67 @@ +package database + +import ( + "context" + "fmt" + + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/mongo/readpref" + + "github.com/5gsec/sentryflow/speculator/pkg/config" + "github.com/5gsec/sentryflow/speculator/pkg/util" +) + +type Database interface { + //GetApiSpec() (libopenapi.Document, error) + //PutApiSpec(document libopenapi.Document) error + //DeleteProvidedApiSpec() error + //DeleteApprovedAPISpec() error +} + +type Handler struct { + Database *mongo.Database + Disconnect func() error +} + +func New(ctx context.Context, dbConfig *config.Database) (*Handler, error) { + logger := util.LoggerFromCtx(ctx) + + client, err := mongo.Connect(ctx, clientOptions(dbConfig)) + if err != nil { + return nil, fmt.Errorf("failed to connect to database: %w", err) + } + + logger.Infof("connecting to %s database", dbConfig.Name) + if err := client.Ping(ctx, readpref.Primary()); err != nil { + return nil, fmt.Errorf("failed to ping database: %w", err) + } + logger.Info("connected to database") + + return &Handler{ + Database: client.Database(dbConfig.Name), + Disconnect: func() error { + return client.Disconnect(ctx) + }, + }, nil +} + +func clientOptions(dbConfig *config.Database) *options.ClientOptions { + dbLoggerOptions := &options.LoggerOptions{ + ComponentLevels: map[options.LogComponent]options.LogLevel{}, + } + switch dbConfig.LogLevel { + case util.LevelInfo: + dbLoggerOptions.SetComponentLevel(options.LogComponentCommand, options.LogLevelInfo) + case util.LevelDebug: + dbLoggerOptions.SetComponentLevel(options.LogComponentCommand, options.LogLevelDebug) + } + + return options.Client(). + ApplyURI(dbConfig.Uri). + SetAuth(options.Credential{ + Username: dbConfig.User, + Password: dbConfig.Password, + }). + SetLoggerOptions(dbLoggerOptions) +} diff --git a/speculator/pkg/pathtrie/path_trie.go b/speculator/pkg/pathtrie/path_trie.go new file mode 100644 index 0000000..4e1a00c --- /dev/null +++ b/speculator/pkg/pathtrie/path_trie.go @@ -0,0 +1,219 @@ +package pathtrie + +import ( + "strings" + + "github.com/5gsec/sentryflow/speculator/pkg/util" +) + +type PathToTrieNode map[string]*TrieNode + +type TrieNode struct { + Children PathToTrieNode + + // Name of the path segment corresponding to this node. + // E.g. if this node represents /v1/foo/bar, the Name would be "bar" and the + // FullPath would be "/v1/foo/bar". + Name string + + // FullPath includes the node's name and uniquely identifies the node in the + // tree. + FullPath string + + // PathParamCounter counts the number of path params in the FullPath. + PathParamCounter int + + // Value of the full path. + Value any +} + +type PathTrie struct { + Trie PathToTrieNode + PathSeparator string +} + +type ValueMergeFunc func(existing, newV *any) + +func (pt *PathTrie) createPathTrieNode(segments []string, idx int, isLastSegment bool, val any) *TrieNode { + fullPathSegments := segments[:idx+1] + node := &TrieNode{ + Children: make(PathToTrieNode), + Name: segments[idx], + FullPath: strings.Join(fullPathSegments, pt.PathSeparator), + } + node.PathParamCounter = countPathParam(fullPathSegments) + if isLastSegment { + node.Value = val + } + + return node +} + +func countPathParam(segments []string) int { + count := 0 + + for _, segment := range segments { + if util.IsPathParam(segment) { + count += 1 + } + } + + return count +} + +// InsertMerge takes a merge function which is responsible for updating the +// existing value with the new value. +func (pt *PathTrie) InsertMerge(path string, val any, merge ValueMergeFunc) (isNewPath bool) { + trie := pt.Trie + isNewPath = true + // TODO: what about path that ends with pt.PathSeparator is it different ? + segments := strings.Split(path, pt.PathSeparator) + + // Traverse the Trie along path, inserting nodes where necessary. + for idx, segment := range segments { + isLastSegment := idx == len(segments)-1 + if node, ok := trie[segment]; ok { + if isLastSegment { + // If this is the last path segment, then this is the node to update. + // If node value is not empty it means that an existing path is overwritten. + isNewPath = util.IsNil(node.Value) + merge(&node.Value, &val) + } else { + // Otherwise, continue descending. + trie = node.Children + } + } else { + newNode := pt.createPathTrieNode(segments, idx, isLastSegment, val) + trie[segment] = newNode + trie = newNode.Children + } + } + + return isNewPath +} + +// Insert inserts val at path, with path segments separated by PathSeparator. +// Returns true if a new path was created, false if an existing path was +// overwritten. +func (pt *PathTrie) Insert(path string, val any) bool { + return pt.InsertMerge(path, val, func(existing, newV *any) { + *existing = *newV + }) +} + +// GetValue returns the given node path value, nil if node is not found. +func (pt *PathTrie) GetValue(path string) any { + node := pt.getNode(path) + if node == nil { + return nil + } + + return node.Value +} + +// GetPathAndValue returns the given node full path and value, nil if node is not found. +func (pt *PathTrie) GetPathAndValue(path string) (string, any, bool) { + node := pt.getNode(path) + if node == nil { + return "", nil, false + } + + return node.FullPath, node.Value, true +} + +func (pt *PathTrie) getNode(path string) *TrieNode { + segments := strings.Split(path, pt.PathSeparator) + + nodes := pt.Trie.getMatchNodes(segments, 0) + + if len(nodes) == 0 { + return nil + } + + if len(nodes) == 1 { + return nodes[0] + } + + // if multiple nodes found, return the node with less path params segments + return getMostAccurateNode(nodes, path, len(segments)) +} + +// getMostAccurateNode returns the node with less path params segments. +func getMostAccurateNode(nodes []*TrieNode, path string, segmentsLen int) *TrieNode { + var retNode *TrieNode + minPathParamSegmentsCount := segmentsLen + 1 + + for _, node := range nodes { + if node.isFullPathMatch(path) { + // return exact match + return node + } + + // TODO: if node.PathParamCounter == minPathParamSegmentsCount + if node.PathParamCounter < minPathParamSegmentsCount { + // found more accurate node + minPathParamSegmentsCount = node.PathParamCounter + retNode = node + } + } + + return retNode +} + +func (trie PathToTrieNode) getMatchNodes(segments []string, idx int) []*TrieNode { + var nodes []*TrieNode + + isLastSegment := idx == len(segments)-1 + + for _, node := range trie { + // Check for node segment match + if !node.isNameMatch(segments[idx]) { + continue + } + + // If this is the last path segment, then return node if it holds a value. + if isLastSegment { + if node.Value != nil { + nodes = append(nodes, node) + } + continue + } + + // Otherwise, continue descending. + newNodes := node.Children.getMatchNodes(segments, idx+1) + if len(newNodes) > 0 { + nodes = append(nodes, newNodes...) + } + } + + return nodes +} + +func (node *TrieNode) isNameMatch(segment string) bool { + if util.IsPathParam(node.Name) { + return true + } + + if node.Name == segment { + return true + } + + return false +} + +func (node *TrieNode) isFullPathMatch(path string) bool { + return node.FullPath == path +} + +// NewWithPathSeparator creates a PathTrie with a user-supplied path separator. +func NewWithPathSeparator(pathSeparator string) PathTrie { + return PathTrie{ + Trie: make(PathToTrieNode), + PathSeparator: pathSeparator, + } +} + +// New creates a PathTrie with "/" as the path separator. +func New() PathTrie { + return NewWithPathSeparator("/") +} diff --git a/speculator/pkg/pathtrie/path_trie_test.go b/speculator/pkg/pathtrie/path_trie_test.go new file mode 100644 index 0000000..2d8fab3 --- /dev/null +++ b/speculator/pkg/pathtrie/path_trie_test.go @@ -0,0 +1,998 @@ +package pathtrie + +import ( + "encoding/json" + "fmt" + "reflect" + "sort" + "testing" +) + +type pathAndValue struct { + path string + value any +} + +func TestPathTrie_getNode(t *testing.T) { + pt := New() + if err := populateDummyPathsAndValues(pt, + pathAndValue{path: "/api/{param1}/items", value: 1}, + pathAndValue{path: "/api/items", value: 2}, + pathAndValue{path: "/api/{param1}/{param2}", value: 3}, + pathAndValue{path: "/api/{param1}/cat", value: 4}, + pathAndValue{path: "/api/items/cat", value: 5}, + ); err != nil { + t.Error(err) + } + + type args struct { + path string + } + tests := []struct { + name string + args args + want *TrieNode + }{ + { + name: "most accurate match - will match both `/api/{param1}/items` and `/api/{param1}/{param2}`", + args: args{ + path: "/api/1/items", + }, + want: &TrieNode{ + Children: make(PathToTrieNode), + Name: "items", + FullPath: "/api/{param1}/items", + PathParamCounter: 1, + Value: 1, + }, + }, + { + name: "exact match with path param", + args: args{ + path: "/api/{param1}/items", + }, + want: &TrieNode{ + Children: make(PathToTrieNode), + Name: "items", + FullPath: "/api/{param1}/items", + PathParamCounter: 1, + Value: 1, + }, + }, + { + name: "short match - not continue to `/api/items/cat`", + args: args{ + path: "/api/items", + }, + want: &TrieNode{ + Children: map[string]*TrieNode{ + "cat": { + Children: make(PathToTrieNode), + Name: "cat", + FullPath: "/api/items/cat", + PathParamCounter: 0, + Value: 5, + }, + }, + Name: "items", + FullPath: "/api/items", + PathParamCounter: 0, + Value: 2, + }, + }, + { + name: "simple path param match", + args: args{ + path: "/api/1/2", + }, + want: &TrieNode{ + Children: make(PathToTrieNode), + Name: "{param2}", + FullPath: "/api/{param1}/{param2}", + PathParamCounter: 2, + Value: 3, + }, + }, + { + name: "most accurate match - will match both `/api/{param1}/cat` and `/api/{param1}/{param2}`", + args: args{ + path: "/api/1/cat", + }, + want: &TrieNode{ + Children: make(PathToTrieNode), + Name: "cat", + FullPath: "/api/{param1}/cat", + PathParamCounter: 1, + Value: 4, + }, + }, + { + name: "exact match with no path param", + args: args{ + path: "/api/items/cat", + }, + want: &TrieNode{ + Children: make(PathToTrieNode), + Name: "cat", + FullPath: "/api/items/cat", + PathParamCounter: 0, + Value: 5, + }, + }, + { + name: "no match", + args: args{ + path: "api/items/cat", + }, + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := pt.getNode(tt.args.path); !reflect.DeepEqual(got, tt.want) { + t.Errorf("getNode() = %v, want %v", got, tt.want) + } + }) + } +} + +func populateDummyPathsAndValues(trie PathTrie, pathAndValues ...pathAndValue) error { + for _, pathAndVal := range pathAndValues { + if !trie.Insert(pathAndVal.path, pathAndVal.value) { + return fmt.Errorf("insertion failed for path %s and value: %d", pathAndVal.path, pathAndVal.value) + } + } + return nil +} + +func TestPathTrie_GetValue(t *testing.T) { + pt := New() + if err := populateDummyPathsAndValues(pt, + pathAndValue{path: "/api/{param1}/items", value: 1}, + ); err != nil { + t.Error(err) + } + + type args struct { + path string + } + tests := []struct { + name string + args args + want interface{} + }{ + { + name: "match", + args: args{ + path: "/api/1/items", + }, + want: 1, + }, + { + name: "no match", + args: args{ + path: "api/items/cat", + }, + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := pt.GetValue(tt.args.path); !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetValue() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPathTrie_GetPathAndValue(t *testing.T) { + pt := New() + if err := populateDummyPathsAndValues(pt, + pathAndValue{path: "/api/{param1}/items", value: 1}, + ); err != nil { + t.Error(err) + } + type args struct { + path string + } + tests := []struct { + name string + args args + wantPath string + wantValue interface{} + wantFound bool + }{ + { + name: "match", + args: args{ + path: "/api/1/items", + }, + wantPath: "/api/{param1}/items", + wantValue: 1, + wantFound: true, + }, + { + name: "no match", + args: args{ + path: "api/items/cat", + }, + wantPath: "", + wantValue: nil, + wantFound: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotPath, gotValue, gotFound := pt.GetPathAndValue(tt.args.path) + if gotPath != tt.wantPath { + t.Errorf("GetPathAndValue() gotPath = %v, wantPath %v", gotPath, tt.wantPath) + } + if !reflect.DeepEqual(gotValue, tt.wantValue) { + t.Errorf("GetPathAndValue() gotValue = %v, wantValue %v", gotValue, tt.wantValue) + } + if gotFound != tt.wantFound { + t.Errorf("GetPathAndValue() gotFound = %v, wantFound %v", gotFound, tt.wantFound) + } + }) + } +} + +func TestPathTrieMap_getMatchNodes(t *testing.T) { + type args struct { + segments []string + idx int + } + tests := []struct { + name string + trie PathToTrieNode + args args + want []*TrieNode + }{ + { + name: "return 2 matches nodes", + trie: map[string]*TrieNode{ + "api": { + Children: map[string]*TrieNode{ + "{param1}": { + Children: map[string]*TrieNode{ + "test": { + Children: make(PathToTrieNode), + Name: "test", + FullPath: "/api/{param1}/test", + PathParamCounter: 1, + Value: 1, + }, + "{param2}": { + Children: make(PathToTrieNode), + Name: "{param2}", + FullPath: "/api/{param1}/{param2}", + PathParamCounter: 2, + Value: 2, + }, + }, + Name: "{param1}", + FullPath: "/api/{param1}", + PathParamCounter: 1, + }, + }, + Name: "api", + FullPath: "/api", + }, + }, + args: args{ + segments: []string{"api", "123", "test"}, + idx: 0, + }, + want: []*TrieNode{ + { + Children: make(PathToTrieNode), + Name: "test", + FullPath: "/api/{param1}/test", + PathParamCounter: 1, + Value: 1, + }, + { + Children: make(PathToTrieNode), + Name: "{param2}", + FullPath: "/api/{param1}/{param2}", + PathParamCounter: 2, + Value: 2, + }, + }, + }, + { + name: "last path segment has nil value - return only 1 matches nodes (/api/{param1}/{param2})", + trie: map[string]*TrieNode{ + "api": { + Children: map[string]*TrieNode{ + "{param1}": { + Children: map[string]*TrieNode{ + "test": { + Children: map[string]*TrieNode{ + "cats": { + Children: make(PathToTrieNode), + Name: "cats", + FullPath: "/api/{param1}/test/cats", + PathParamCounter: 1, + Value: 1, + }, + }, + Name: "test", + FullPath: "/api/{param1}/test", + PathParamCounter: 1, + Value: nil, + }, + "{param2}": { + Children: make(PathToTrieNode), + Name: "{param2}", + FullPath: "/api/{param1}/{param2}", + PathParamCounter: 2, + Value: 2, + }, + }, + Name: "{param1}", + FullPath: "/api/{param1}", + PathParamCounter: 1, + }, + }, + Name: "api", + FullPath: "/api", + }, + }, + args: args{ + segments: []string{"api", "123", "test"}, + idx: 0, + }, + want: []*TrieNode{ + { + Children: make(PathToTrieNode), + Name: "{param2}", + FullPath: "/api/{param1}/{param2}", + PathParamCounter: 2, + Value: 2, + }, + }, + }, + { + name: "0 nodes match", + trie: map[string]*TrieNode{ + "api": { + Children: map[string]*TrieNode{ + "{param1}": { + Children: map[string]*TrieNode{ + "test": { + Children: make(PathToTrieNode), + Name: "test", + FullPath: "/api/{param1}/test", + PathParamCounter: 1, + Value: 1, + }, + "{param2}": { + Children: make(PathToTrieNode), + Name: "{param2}", + FullPath: "/api/{param1}/{param2}", + PathParamCounter: 2, + Value: 2, + }, + }, + Name: "{param1}", + FullPath: "/api/{param1}", + PathParamCounter: 1, + Value: nil, + }, + }, + Name: "api", + FullPath: "/api", + }, + }, + args: args{ + segments: []string{"api", "cats", "dogs", "test"}, + idx: 0, + }, + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.trie.getMatchNodes(tt.args.segments, tt.args.idx) + sort.Slice(got, func(i, j int) bool { + return got[i].FullPath < got[j].FullPath + }) + sort.Slice(tt.want, func(i, j int) bool { + return tt.want[i].FullPath < tt.want[j].FullPath + }) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("getMatchNodes() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getMostAccurateNode(t *testing.T) { + pt := New() + type args struct { + nodes []*TrieNode + path string + segmentsLen int + } + tests := []struct { + name string + args args + want *TrieNode + }{ + { + name: "exact prefix match", + args: args{ + nodes: []*TrieNode{ + pt.createPathTrieNode([]string{"", "api", "{param1}", "test"}, 3, true, 1), + pt.createPathTrieNode([]string{"", "api", "{param1}", "{param2}"}, 3, true, 2), + }, + path: "/api/{param1}/test", + segmentsLen: 4, + }, + want: pt.createPathTrieNode([]string{"", "api", "{param1}", "test"}, 3, true, 1), + }, + { + name: "less path params match", + args: args{ + nodes: []*TrieNode{ + pt.createPathTrieNode([]string{"", "api", "{param1}", "test", "{param2}"}, 4, true, 1), + pt.createPathTrieNode([]string{"", "api", "{param1}", "{param2}", "{param3}"}, 4, true, 2), + }, + path: "/api/cats/test/dogs", + segmentsLen: 5, + }, + want: pt.createPathTrieNode([]string{"", "api", "{param1}", "test", "{param2}"}, 4, true, 1), + }, + { + name: "single match", + args: args{ + nodes: []*TrieNode{ + pt.createPathTrieNode([]string{"", "api", "{param1}", "test"}, 3, true, 1), + }, + path: "/api/cats/test", + segmentsLen: 4, + }, + want: pt.createPathTrieNode([]string{"", "api", "{param1}", "test"}, 3, true, 1), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getMostAccurateNode(tt.args.nodes, tt.args.path, tt.args.segmentsLen); !reflect.DeepEqual(got, tt.want) { + t.Errorf("getMostAccurateNode() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPathTrieNode_isNameMatch(t *testing.T) { + type fields struct { + Name string + } + type args struct { + segment string + } + tests := []struct { + name string + fields fields + args args + want bool + }{ + { + name: "path param match", + fields: fields{ + Name: "{param}", + }, + args: args{ + segment: "match", + }, + want: true, + }, + { + name: "segment name match", + fields: fields{ + Name: "match", + }, + args: args{ + segment: "match", + }, + want: true, + }, + { + name: "segment name not match", + fields: fields{ + Name: "match", + }, + args: args{ + segment: "not-match", + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := &TrieNode{ + Name: tt.fields.Name, + } + if got := node.isNameMatch(tt.args.segment); got != tt.want { + t.Errorf("isNameMatch() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPathTrieNode_isFullPathMatch(t *testing.T) { + type fields struct { + Prefix string + } + type args struct { + path string + } + tests := []struct { + name string + fields fields + args args + want bool + }{ + { + name: "match", + fields: fields{ + Prefix: "/api/{param1}/test", + }, + args: args{ + path: "/api/{param1}/test", + }, + want: true, + }, + { + name: "no match", + fields: fields{ + Prefix: "/api/{param1}/test", + }, + args: args{ + path: "/api/{param1}/{param2}", + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := &TrieNode{ + FullPath: tt.fields.Prefix, + } + if got := node.isFullPathMatch(tt.args.path); got != tt.want { + t.Errorf("isFullPathMatch() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_countPathParam(t *testing.T) { + type args struct { + segments []string + } + tests := []struct { + name string + args args + want int + }{ + { + name: "no path param", + args: args{ + segments: []string{"", "api", "cat", "test"}, + }, + want: 0, + }, + { + name: "single path param", + args: args{ + segments: []string{"", "api", "{param1}", "test"}, + }, + want: 1, + }, + { + name: "multiple path params", + args: args{ + segments: []string{"", "api", "{param1}", "test", "{param2}"}, + }, + want: 2, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := countPathParam(tt.args.segments); got != tt.want { + t.Errorf("countPathParam() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPathTrie_createPathTrieNode(t *testing.T) { + type args struct { + segments []string + idx int + isLastSegment bool + val interface{} + } + tests := []struct { + name string + args args + want *TrieNode + }{ + { + name: "last segment with 1 path param", + args: args{ + segments: []string{"", "api", "{param}"}, + idx: 2, + isLastSegment: true, + val: 1, + }, + want: &TrieNode{ + Children: make(PathToTrieNode), + Name: "{param}", + FullPath: "/api/{param}", + PathParamCounter: 1, + Value: 1, + }, + }, + { + name: "not last segment with 3 path param", + args: args{ + segments: []string{"", "api", "{param1}", "{param2}", "{param3}", "test"}, + idx: 4, + isLastSegment: false, + val: 1, + }, + want: &TrieNode{ + Children: make(PathToTrieNode), + Name: "{param3}", + FullPath: "/api/{param1}/{param2}/{param3}", + PathParamCounter: 3, + Value: nil, + }, + }, + { + name: "not last segment with no path param", + args: args{ + segments: []string{"", "api", "{param1}", "{param2}", "{param3}", "test"}, + idx: 1, + isLastSegment: false, + val: 1, + }, + want: &TrieNode{ + Children: make(PathToTrieNode), + Name: "api", + FullPath: "/api", + PathParamCounter: 0, + Value: nil, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pt := New() + if got := pt.createPathTrieNode(tt.args.segments, tt.args.idx, tt.args.isLastSegment, tt.args.val); !reflect.DeepEqual(got, tt.want) { + t.Errorf("createPathTrieNode() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPathTrie_InsertMerge(t *testing.T) { + swapMerge := func(existing, newV *interface{}) { + *existing = *newV + } + shouldNotBeCalledMergeFunc := func(existing, newV *interface{}) { + panic(fmt.Sprintf("merge should not be called. existing=%+v, newV=%+v", *existing, *newV)) + } + type fields struct { + Trie PathToTrieNode + PathSeparator string + } + type args struct { + path string + val interface{} + merge ValueMergeFunc + } + tests := []struct { + name string + fields fields + args args + wantIsNewPath bool + expectedTrie PathToTrieNode + }{ + { + name: "new path", + fields: fields{ + Trie: PathToTrieNode{}, + PathSeparator: "/", + }, + args: args{ + path: "/api", + val: 1, + merge: shouldNotBeCalledMergeFunc, + }, + wantIsNewPath: true, + expectedTrie: PathToTrieNode{ + "": &TrieNode{ + Children: map[string]*TrieNode{ + "api": { + Children: make(PathToTrieNode), + Name: "api", + FullPath: "/api", + PathParamCounter: 0, + Value: 1, + }, + }, + Name: "", + FullPath: "", + PathParamCounter: 0, + Value: nil, + }, + }, + }, + { + name: "existing path", + fields: fields{ + Trie: PathToTrieNode{ + "": &TrieNode{ + Children: map[string]*TrieNode{ + "api": { + Children: make(PathToTrieNode), + Name: "api", + FullPath: "/api", + PathParamCounter: 0, + Value: 1, + }, + }, + Name: "", + FullPath: "", + PathParamCounter: 0, + Value: nil, + }, + }, + PathSeparator: "/", + }, + args: args{ + path: "/api", + val: 2, + merge: swapMerge, + }, + wantIsNewPath: false, + expectedTrie: PathToTrieNode{ + "": &TrieNode{ + Children: map[string]*TrieNode{ + "api": { + Children: make(PathToTrieNode), + Name: "api", + FullPath: "/api", + PathParamCounter: 0, + Value: 2, + }, + }, + Name: "", + FullPath: "", + PathParamCounter: 0, + Value: nil, + }, + }, + }, + { + name: "path with separator at the end - expected new path", + fields: fields{ + Trie: PathToTrieNode{ + "": &TrieNode{ + Children: map[string]*TrieNode{ + "api": { + Children: make(PathToTrieNode), + Name: "api", + FullPath: "/api", + PathParamCounter: 0, + Value: 1, + }, + }, + Name: "", + FullPath: "", + PathParamCounter: 0, + Value: nil, + }, + }, + PathSeparator: "/", + }, + args: args{ + path: "/api/", + val: 2, + merge: shouldNotBeCalledMergeFunc, + }, + wantIsNewPath: true, + expectedTrie: PathToTrieNode{ + "": &TrieNode{ + Children: map[string]*TrieNode{ + "api": { + Children: map[string]*TrieNode{ + "": { + Children: make(PathToTrieNode), + Name: "", + FullPath: "/api/", + PathParamCounter: 0, + Value: 2, + }, + }, + Name: "api", + FullPath: "/api", + PathParamCounter: 0, + Value: 1, + }, + }, + Name: "", + FullPath: "", + PathParamCounter: 0, + Value: nil, + }, + }, + }, + { + name: "path param addition", + fields: fields{ + Trie: PathToTrieNode{ + "": &TrieNode{ + Children: map[string]*TrieNode{ + "api": { + Children: make(PathToTrieNode), + Name: "api", + FullPath: "/api", + PathParamCounter: 0, + Value: 1, + }, + }, + Name: "", + FullPath: "", + PathParamCounter: 0, + Value: nil, + }, + }, + PathSeparator: "/", + }, + args: args{ + path: "/api/{param}", + val: 2, + merge: swapMerge, + }, + wantIsNewPath: true, + expectedTrie: PathToTrieNode{ + "": &TrieNode{ + Children: map[string]*TrieNode{ + "api": { + Children: map[string]*TrieNode{ + "{param}": { + Children: make(PathToTrieNode), + Name: "{param}", + FullPath: "/api/{param}", + PathParamCounter: 1, + Value: 2, + }, + }, + Name: "api", + FullPath: "/api", + PathParamCounter: 0, + Value: 1, + }, + }, + Name: "", + FullPath: "", + PathParamCounter: 0, + Value: nil, + }, + }, + }, + { + name: "new path for existing node", + fields: fields{ + // /carts/{customerID}/items/{itemID} + Trie: PathToTrieNode{ + "": &TrieNode{ + Children: map[string]*TrieNode{ + "carts": { + Children: map[string]*TrieNode{ + "{customerID}": { + Children: map[string]*TrieNode{ + "items": { + Children: map[string]*TrieNode{ + "{itemID}": { + Children: make(PathToTrieNode), + Name: "{itemID}", + FullPath: "/carts/{customerID}/items/{itemID}", + PathParamCounter: 1, + Value: "1", + }, + }, + Name: "items", + FullPath: "/carts/{customerID}/items", + PathParamCounter: 1, + }, + }, + Name: "{customerID}", + FullPath: "/carts/{customerID}", + PathParamCounter: 0, + }, + }, + Name: "carts", + FullPath: "/carts", + PathParamCounter: 0, + }, + }, + Name: "", + FullPath: "", + PathParamCounter: 0, + Value: nil, + }, + }, + PathSeparator: "/", + }, + args: args{ + path: "/carts/{customerID}/items", + val: "2", + merge: swapMerge, + }, + wantIsNewPath: true, + expectedTrie: PathToTrieNode{ + "": &TrieNode{ + Children: map[string]*TrieNode{ + "carts": { + Children: map[string]*TrieNode{ + "{customerID}": { + Children: map[string]*TrieNode{ + "items": { + Children: map[string]*TrieNode{ + "{itemID}": { + Children: map[string]*TrieNode{}, + Name: "{itemID}", + FullPath: "/carts/{customerID}/items/{itemID}", + PathParamCounter: 1, + Value: "1", + }, + }, + Name: "items", + FullPath: "/carts/{customerID}/items", + PathParamCounter: 1, + Value: "2", + }, + }, + Name: "{customerID}", + FullPath: "/carts/{customerID}", + PathParamCounter: 0, + }, + }, + Name: "carts", + FullPath: "/carts", + PathParamCounter: 0, + }, + }, + Name: "", + FullPath: "", + PathParamCounter: 0, + Value: nil, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pt := &PathTrie{ + Trie: tt.fields.Trie, + PathSeparator: tt.fields.PathSeparator, + } + if gotIsNewPath := pt.InsertMerge(tt.args.path, tt.args.val, tt.args.merge); gotIsNewPath != tt.wantIsNewPath { + t.Errorf("InsertMerge() = %v, want %v", gotIsNewPath, tt.wantIsNewPath) + } + if !reflect.DeepEqual(pt.Trie, tt.expectedTrie) { + t.Errorf("InsertMerge() Trie = %+v, want %+v", marshal(pt.Trie), marshal(tt.expectedTrie)) + } + }) + } +} + +func marshal(trie PathToTrieNode) any { + trieBytes, _ := json.Marshal(trie) + return string(trieBytes) +} diff --git a/speculator/pkg/speculator/map_repository.go b/speculator/pkg/speculator/map_repository.go new file mode 100644 index 0000000..403f12d --- /dev/null +++ b/speculator/pkg/speculator/map_repository.go @@ -0,0 +1,85 @@ +package speculator + +import ( + "encoding/gob" + "fmt" + "os" + "sync" + + log "github.com/sirupsen/logrus" +) + +type Repository struct { + Speculators map[uint]*Speculator + speculatorConfig Config + + lock *sync.RWMutex +} + +func NewMapRepository(config Config) *Repository { + return &Repository{ + Speculators: map[uint]*Speculator{}, + speculatorConfig: config, + lock: &sync.RWMutex{}, + } +} + +func DecodeState(filePath string, config Config) (*Repository, error) { + r := Repository{} + + const perm = 400 + file, err := os.OpenFile(filePath, os.O_RDONLY, os.FileMode(perm)) + if err != nil { + return nil, fmt.Errorf("failed to open file (%v): %v", filePath, err) + } + defer closeFile(file) + + decoder := gob.NewDecoder(file) + err = decoder.Decode(&r) + if err != nil { + return nil, fmt.Errorf("failed to decode state: %v", err) + } + + r.speculatorConfig = config + r.lock = &sync.RWMutex{} + log.Info("Speculator state was decoded") + + return &r, nil +} + +func (r *Repository) Get(speculatorID uint) *Speculator { + r.lock.RLock() + defer r.lock.RUnlock() + + speculator, ok := r.Speculators[speculatorID] + if !ok { + r.Speculators[speculatorID] = CreateSpeculator(r.speculatorConfig) + + return r.Speculators[speculatorID] + } + + return speculator +} + +func (r *Repository) EncodeState(filePath string) error { + const perm = 400 + file, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE, os.FileMode(perm)) + if err != nil { + return fmt.Errorf("failed to open state file: %v", err) + } + defer closeFile(file) + + encoder := gob.NewEncoder(file) + err = encoder.Encode(r) + if err != nil { + return fmt.Errorf("failed to encode state: %v", err) + } + + return nil +} + +func closeFile(file *os.File) { + if err := file.Close(); err != nil { + log.Errorf("Failed to close file: %v", err) + } +} diff --git a/speculator/pkg/speculator/speculator.go b/speculator/pkg/speculator/speculator.go new file mode 100644 index 0000000..e9ecd88 --- /dev/null +++ b/speculator/pkg/speculator/speculator.go @@ -0,0 +1,286 @@ +package speculator + +import ( + "encoding/gob" + "encoding/json" + "fmt" + "os" + "strings" + + log "github.com/sirupsen/logrus" + + "github.com/5gsec/sentryflow/speculator/pkg/apispec" +) + +type SpecKey string + +type Config struct { + OperationGeneratorConfig apispec.OperationGeneratorConfig +} + +type Speculator struct { + Specs map[SpecKey]*apispec.Spec `json:"specs,omitempty"` + + // config is not exported and is not encoded part of the state + config Config +} + +// nolint:gochecknoinits +func init() { + gob.Register(json.RawMessage{}) +} + +func CreateSpeculator(config Config) *Speculator { + log.Info("Creating Speculator") + log.Debugf("Speculator Config %+v", config) + return &Speculator{ + Specs: make(map[SpecKey]*apispec.Spec), + config: config, + } +} + +func GetSpecKey(host, port string) SpecKey { + return SpecKey(host + ":" + port) +} + +func GetHostAndPortFromSpecKey(key SpecKey) (host, port string, err error) { + const hostAndPortLen = 2 + hostAndPort := strings.Split(string(key), ":") + if len(hostAndPort) != hostAndPortLen { + return "", "", fmt.Errorf("invalid key: %v", key) + } + host = hostAndPort[0] + if len(host) == 0 { + return "", "", fmt.Errorf("no host for key: %v", key) + } + port = hostAndPort[1] + if len(port) == 0 { + return "", "", fmt.Errorf("no port for key: %v", key) + } + return host, port, nil +} + +func (s *Speculator) SuggestedReview(specKey SpecKey) (*apispec.SuggestedSpecReview, error) { + spec, ok := s.Specs[specKey] + if !ok { + return nil, fmt.Errorf("spec doesn't exist for key %v", specKey) + } + + return spec.CreateSuggestedReview(), nil +} + +type AddressInfo struct { + IP string + Port string +} + +func GetAddressInfoFromAddress(address string) (*AddressInfo, error) { + const addrLen = 2 + addr := strings.Split(address, ":") + if len(addr) != addrLen { + return nil, fmt.Errorf("invalid address: %v", addr) + } + + return &AddressInfo{ + IP: addr[0], + Port: addr[1], + }, nil +} + +func (s *Speculator) InitSpec(host, port string) error { + specKey := GetSpecKey(host, port) + if _, ok := s.Specs[specKey]; ok { + return fmt.Errorf("spec was already initialized using host and port: %s:%s", host, port) + } + s.Specs[specKey] = apispec.CreateDefaultSpec(host, port, s.config.OperationGeneratorConfig) + return nil +} + +func (s *Speculator) LearnTelemetry(telemetry *apispec.Telemetry) error { + destInfo, err := GetAddressInfoFromAddress(telemetry.DestinationAddress) + if err != nil { + return fmt.Errorf("failed get destination info: %v", err) + } + specKey := GetSpecKey(telemetry.Request.Host, destInfo.Port) + if _, ok := s.Specs[specKey]; !ok { + s.Specs[specKey] = apispec.CreateDefaultSpec(telemetry.Request.Host, destInfo.Port, s.config.OperationGeneratorConfig) + } + spec := s.Specs[specKey] + if err := spec.LearnTelemetry(telemetry); err != nil { + return fmt.Errorf("failed to insert telemetry: %v. %v", telemetry, err) + } + + return nil +} + +func (s *Speculator) GetPathID(specKey SpecKey, path string, specSource apispec.SpecSource) (string, error) { + spec, ok := s.Specs[specKey] + if !ok { + return "", fmt.Errorf("no spec for key %v", specKey) + } + + pathID, err := spec.GetPathID(path, specSource) + if err != nil { + return "", fmt.Errorf("failed to get path id. specKey=%v, specSource=%v: %v", specKey, specSource, err) + } + + return pathID, nil +} + +func (s *Speculator) DiffTelemetry(telemetry *apispec.Telemetry, diffSource apispec.SpecSource) (*apispec.APIDiff, error) { + destInfo, err := GetAddressInfoFromAddress(telemetry.DestinationAddress) + if err != nil { + return nil, fmt.Errorf("failed get destination info: %v", err) + } + specKey := GetSpecKey(telemetry.Request.Host, destInfo.Port) + spec, ok := s.Specs[specKey] + if !ok { + return nil, fmt.Errorf("no spec for key %v", specKey) + } + + apiDiff, err := spec.DiffTelemetry(telemetry, diffSource) + if err != nil { + return nil, fmt.Errorf("failed to run DiffTelemetry: %v", err) + } + + return apiDiff, nil +} + +func (s *Speculator) HasApprovedSpec(key SpecKey) bool { + spec, ok := s.Specs[key] + if !ok { + return false + } + + return spec.HasApprovedSpec() +} + +func (s *Speculator) GetApprovedSpecVersion(key SpecKey) apispec.OASVersion { + spec, ok := s.Specs[key] + if !ok { + return apispec.Unknown + } + + return spec.ApprovedSpec.GetSpecVersion() +} + +func (s *Speculator) LoadProvidedSpec(key SpecKey, providedSpec []byte, pathToPathID map[string]string) error { + spec, ok := s.Specs[key] + if !ok { + return fmt.Errorf("no spec found with key: %v", key) + } + + if err := spec.LoadProvidedSpec(providedSpec, pathToPathID); err != nil { + return fmt.Errorf("failed to load provided spec: %w", err) + } + + return nil +} + +func (s *Speculator) UnsetProvidedSpec(key SpecKey) error { + spec, ok := s.Specs[key] + if !ok { + return fmt.Errorf("no spec found with key: %v", key) + } + spec.UnsetProvidedSpec() + return nil +} + +func (s *Speculator) UnsetApprovedSpec(key SpecKey) error { + spec, ok := s.Specs[key] + if !ok { + return fmt.Errorf("no spec found with key: %v", key) + } + spec.UnsetApprovedSpec() + return nil +} + +func (s *Speculator) HasProvidedSpec(key SpecKey) bool { + spec, ok := s.Specs[key] + if !ok { + return false + } + + return spec.HasProvidedSpec() +} + +func (s *Speculator) GetProvidedSpecVersion(key SpecKey) apispec.OASVersion { + spec, ok := s.Specs[key] + if !ok { + return apispec.Unknown + } + + return spec.ProvidedSpec.GetSpecVersion() +} + +func (s *Speculator) DumpSpecs() { + log.Infof("Generating Open API Specs...\n") + for specKey, spec := range s.Specs { + approvedYaml, err := spec.GenerateOASYaml(apispec.OASv3) + if err != nil { + log.Errorf("failed to generate OAS yaml for %v.: %v", specKey, err) + continue + } + log.Infof("Spec for %s:\n%s\n\n", specKey, approvedYaml) + } +} + +func (s *Speculator) ApplyApprovedReview(specKey SpecKey, approvedReview *apispec.ApprovedSpecReview, version apispec.OASVersion) error { + if err := s.Specs[specKey].ApplyApprovedReview(approvedReview, version); err != nil { + return fmt.Errorf("failed to apply approved review for spec: %v. %w", specKey, err) + } + return nil +} + +func (s *Speculator) EncodeState(filePath string) error { + file, err := openFile(filePath) + if err != nil { + return fmt.Errorf("failed to open state file: %v", err) + } + encoder := gob.NewEncoder(file) + err = encoder.Encode(s) + if err != nil { + return fmt.Errorf("failed to encode state: %v", err) + } + closeSpeculatorStateFile(file) + + return nil +} + +func DecodeSpeculatorState(filePath string, config Config) (*Speculator, error) { + r := &Speculator{} + file, err := openFile(filePath) + if err != nil { + return nil, fmt.Errorf("failed to open file (%v): %v", filePath, err) + } + defer closeSpeculatorStateFile(file) + + decoder := gob.NewDecoder(file) + err = decoder.Decode(r) + if err != nil { + return nil, fmt.Errorf("failed to decode state: %v", err) + } + + r.config = config + + log.Info("Speculator state was decoded") + log.Debugf("Speculator Config %+v", config) + + return r, nil +} + +func openFile(filePath string) (*os.File, error) { + const perm = 400 + file, err := os.OpenFile(filePath, os.O_RDWR|os.O_CREATE, os.FileMode(perm)) + if err != nil { + return nil, fmt.Errorf("failed to open file (%v) for writing: %v", filePath, err) + } + + return file, nil +} + +func closeSpeculatorStateFile(f *os.File) { + if err := f.Close(); err != nil { + log.Errorf("Failed to close file: %v", err) + } +} diff --git a/speculator/pkg/speculator/speculator_accesssor.go b/speculator/pkg/speculator/speculator_accesssor.go new file mode 100644 index 0000000..33384d4 --- /dev/null +++ b/speculator/pkg/speculator/speculator_accesssor.go @@ -0,0 +1,42 @@ +package speculator + +import ( + "github.com/5gsec/sentryflow/speculator/pkg/apispec" +) + +type SpeculatorsAccessor interface { + DiffTelemetry(speculatorID uint, telemetry *apispec.Telemetry, diffSource apispec.SpecSource) (*apispec.APIDiff, error) + HasApprovedSpec(speculatorID uint, specKey SpecKey) bool + HasProvidedSpec(speculatorID uint, specKey SpecKey) bool + GetProvidedSpecVersion(speculatorID uint, specKey SpecKey) apispec.OASVersion + GetApprovedSpecVersion(speculatorID uint, specKey SpecKey) apispec.OASVersion +} + +func NewSpeculatorAccessor(speculators *Repository) SpeculatorsAccessor { + return &Impl{speculators: speculators} +} + +type Impl struct { + speculators *Repository +} + +func (s *Impl) DiffTelemetry(speculatorID uint, telemetry *apispec.Telemetry, diffSource apispec.SpecSource) (*apispec.APIDiff, error) { + //nolint: wrapcheck + return s.speculators.Get(speculatorID).DiffTelemetry(telemetry, diffSource) +} + +func (s *Impl) HasApprovedSpec(speculatorID uint, specKey SpecKey) bool { + return s.speculators.Get(speculatorID).HasApprovedSpec(specKey) +} + +func (s *Impl) HasProvidedSpec(speculatorID uint, specKey SpecKey) bool { + return s.speculators.Get(speculatorID).HasProvidedSpec(specKey) +} + +func (s *Impl) GetProvidedSpecVersion(speculatorID uint, specKey SpecKey) apispec.OASVersion { + return s.speculators.Get(speculatorID).GetProvidedSpecVersion(specKey) +} + +func (s *Impl) GetApprovedSpecVersion(speculatorID uint, specKey SpecKey) apispec.OASVersion { + return s.speculators.Get(speculatorID).GetApprovedSpecVersion(specKey) +} diff --git a/speculator/pkg/speculator/speculator_test.go b/speculator/pkg/speculator/speculator_test.go new file mode 100644 index 0000000..272eab3 --- /dev/null +++ b/speculator/pkg/speculator/speculator_test.go @@ -0,0 +1,125 @@ +package speculator + +import ( + "os" + "testing" + + "github.com/gofrs/uuid" + + "github.com/5gsec/sentryflow/speculator/pkg/apispec" +) + +func TestGetHostAndPortFromSpecKey(t *testing.T) { + type args struct { + key SpecKey + } + tests := []struct { + name string + args args + wantHost string + wantPort string + wantErr bool + }{ + { + name: "invalid key", + args: args{ + key: "invalid", + }, + wantHost: "", + wantPort: "", + wantErr: true, + }, + { + name: "invalid:key:invalid", + args: args{ + key: "invalid", + }, + wantHost: "", + wantPort: "", + wantErr: true, + }, + { + name: "invalid key - no host", + args: args{ + key: ":8080", + }, + wantHost: "", + wantPort: "", + wantErr: true, + }, + { + name: "invalid key - no port", + args: args{ + key: "host:", + }, + wantHost: "", + wantPort: "", + wantErr: true, + }, + { + name: "valid key", + args: args{ + key: "host:8080", + }, + wantHost: "host", + wantPort: "8080", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotHost, gotPort, err := GetHostAndPortFromSpecKey(tt.args.key) + if (err != nil) != tt.wantErr { + t.Errorf("GetHostAndPortFromSpecKey() error = %v, wantErr %v", err, tt.wantErr) + return + } + if gotHost != tt.wantHost { + t.Errorf("GetHostAndPortFromSpecKey() gotHost = %v, want %v", gotHost, tt.wantHost) + } + if gotPort != tt.wantPort { + t.Errorf("GetHostAndPortFromSpecKey() gotPort = %v, want %v", gotPort, tt.wantPort) + } + }) + } +} + +func TestDecodeState(t *testing.T) { + testSpec := GetSpecKey("host", "port") + uid, _ := uuid.NewV4() + uidStr := uid.String() + testStatePath := "/tmp/" + uidStr + "state.gob" + defer func() { + _ = os.Remove(testStatePath) + }() + + speculatorConfig := Config{ + OperationGeneratorConfig: apispec.OperationGeneratorConfig{ + ResponseHeadersToIgnore: []string{"before"}, + }, + } + speculator := CreateSpeculator(speculatorConfig) + speculator.Specs[testSpec] = apispec.CreateDefaultSpec("host", "port", speculator.config.OperationGeneratorConfig) + + if err := speculator.EncodeState(testStatePath); err != nil { + t.Errorf("EncodeSpeculatorState() error = %v", err) + return + } + + newSpeculatorConfig := Config{ + OperationGeneratorConfig: apispec.OperationGeneratorConfig{ + ResponseHeadersToIgnore: []string{"after"}, + }, + } + got, err := DecodeSpeculatorState(testStatePath, newSpeculatorConfig) + if err != nil { + t.Errorf("DecodeSpeculatorState() error = %v", err) + return + } + + // OpGenerator on the decoded state should hold the previous OperationGeneratorConfig + responseHeadersToIgnore := got.Specs[testSpec].OpGenerator.ResponseHeadersToIgnore + if _, ok := responseHeadersToIgnore["before"]; !ok { + t.Errorf("ResponseHeadersToIgnore not as expected = %+v", responseHeadersToIgnore) + return + } +} diff --git a/speculator/pkg/util/errors/errors.go b/speculator/pkg/util/errors/errors.go new file mode 100644 index 0000000..21045a9 --- /dev/null +++ b/speculator/pkg/util/errors/errors.go @@ -0,0 +1,5 @@ +package errors + +import "errors" + +var ErrSpecValidation = errors.New("apispec validation failed") diff --git a/speculator/pkg/util/logger.go b/speculator/pkg/util/logger.go new file mode 100644 index 0000000..d6023d8 --- /dev/null +++ b/speculator/pkg/util/logger.go @@ -0,0 +1,24 @@ +package util + +import ( + "context" + "reflect" + + "go.uber.org/zap" +) + +const ( + LevelInfo = "info" + LevelDebug = "debug" +) + +type LoggerContextKey struct{} + +func LoggerFromCtx(ctx context.Context) *zap.SugaredLogger { + logger, _ := ctx.Value(LoggerContextKey{}).(*zap.SugaredLogger) + return logger +} + +func IsNil(a any) bool { + return a == nil || (reflect.ValueOf(a).Kind() == reflect.Ptr && reflect.ValueOf(a).IsNil()) +} diff --git a/speculator/pkg/util/mime.go b/speculator/pkg/util/mime.go new file mode 100644 index 0000000..ffa9b48 --- /dev/null +++ b/speculator/pkg/util/mime.go @@ -0,0 +1,11 @@ +package util + +import ( + "strings" +) + +// IsApplicationJSONMediaType will return true if mediaType is in the format of application/*json (application/json, application/hal+json...) +func IsApplicationJSONMediaType(mediaType string) bool { + return strings.HasPrefix(mediaType, "application/") && + strings.HasSuffix(mediaType, "json") +} diff --git a/speculator/pkg/util/mime_test.go b/speculator/pkg/util/mime_test.go new file mode 100644 index 0000000..e92745a --- /dev/null +++ b/speculator/pkg/util/mime_test.go @@ -0,0 +1,52 @@ +package util + +import ( + "testing" +) + +func TestIsApplicationJsonMediaType(t *testing.T) { + type args struct { + mediaType string + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "application/json", + args: args{ + mediaType: "application/json", + }, + want: true, + }, + { + name: "application/hal+json", + args: args{ + mediaType: "application/hal+json", + }, + want: true, + }, + { + name: "not application json mime", + args: args{ + mediaType: "test/html", + }, + want: false, + }, + { + name: "empty mediaType", + args: args{ + mediaType: "", + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsApplicationJSONMediaType(tt.args.mediaType); got != tt.want { + t.Errorf("IsApplicationJSONMediaType() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/speculator/pkg/util/path_param.go b/speculator/pkg/util/path_param.go new file mode 100644 index 0000000..76e14a1 --- /dev/null +++ b/speculator/pkg/util/path_param.go @@ -0,0 +1,15 @@ +package util + +import ( + "strings" +) + +const ( + ParamPrefix = "{" + ParamSuffix = "}" +) + +func IsPathParam(segment string) bool { + return strings.HasPrefix(segment, ParamPrefix) && + strings.HasSuffix(segment, ParamSuffix) +}