691 - CLI: Allow external schema refs (#695)

* process json access logs from openapi-mock container

* support external references to schemas in kusk CLI

api/v1alpha1/api_webhook.go

@@ -32,6 +32,7 @@ import (
 	"reflect"
 	"strings"
 
+	"github.com/getkin/kin-openapi/openapi3"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
@@ -192,9 +193,9 @@ func (a *APIValidator) PathAlreadyDeployed(ctx context.Context, fleet *EnvoyFlee
 }
 
 func (r *API) validate() error {
-	parser := spec.NewParser(nil)
-
-	apiSpec, err := parser.ParseFromReader(strings.NewReader(r.Spec.Spec))
+	apiSpec, err := spec.
+		NewParser(&openapi3.Loader{IsExternalRefsAllowed: true}).
+		ParseFromReader(strings.NewReader(r.Spec.Spec))
 	if err != nil {
 		return fmt.Errorf("spec: should be a valid OpenAPI spec: %w", err)
 	}

cmd/kusk/cmd/generate.go

@@ -127,7 +127,7 @@ var generateCmd = &cobra.Command{
 			}
 		}
 
-		parsedApiSpec, err := spec.NewParser(openapi3.NewLoader()).Parse(apiSpecPath)
+		parsedApiSpec, err := spec.NewParser(&openapi3.Loader{IsExternalRefsAllowed: true}).Parse(apiSpecPath)
 		if err != nil {
 			fmt.Fprintln(os.Stderr, err)
 			reportError(err)

cmd/kusk/cmd/mock.go

@@ -141,7 +141,7 @@ $ kusk mock -i https://url.to.api.com
 			ui.Fail(err)
 		}
 
-		spec, err := spec.NewParser(openapi3.NewLoader()).Parse(apiSpecPath)
+		spec, err := spec.NewParser(&openapi3.Loader{IsExternalRefsAllowed: true}).Parse(apiSpecPath)
 		if err != nil {
 			err := fmt.Errorf("error when parsing openapi spec: %w", err)
 			reportError(err)

cmd/kusk/internal/mocking/mocking.go

@@ -7,7 +7,7 @@ import (
 const openApiMockingConfigStr = `application:
   debug: false
   log_format: json
-  log_level: warn
+  log_level: info
 
 generation:
   suppress_errors: false

cmd/kusk/internal/mocking/server/server.go

@@ -3,11 +3,12 @@ package server
 import (
 	"bufio"
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"net/url"
-	"strings"
+	"strconv"
 	"time"
 
 	"github.com/docker/docker/api/types"
@@ -38,7 +39,7 @@ type AccessLogEntry struct {
 }
 
 func New(ctx context.Context, client *client.Client, configFile, apiToMock string, port uint32) (MockServer, error) {
-	const openApiMockImage = "muonsoft/openapi-mock:v0.3.1"
+	const openApiMockImage = "muonsoft/openapi-mock:v0.3.3"
 
 	reader, err := client.ImagePull(ctx, openApiMockImage, types.ImagePullOptions{})
 	if err != nil {
@@ -152,30 +153,50 @@ func (m MockServer) StreamLogs(ctx context.Context, containerId string) {
 
 	scanner := bufio.NewScanner(reader)
 	for scanner.Scan() {
-		if le, err := newAccessLogEntry(scanner.Text()); err != nil {
+		le, err := newAccessLogEntry(scanner.Text())
+		if err != nil {
 			m.ErrCh <- err
-		} else {
-			m.LogCh <- le
+		} else if le != nil {
+			m.LogCh <- *le
 		}
 	}
 }
 
-func newAccessLogEntry(rawLog string) (AccessLogEntry, error) {
-	if strings.Contains(rawLog, "warning") || strings.Contains(rawLog, "error") {
-		return AccessLogEntry{}, errors.New(rawLog)
+func newAccessLogEntry(rawLog string) (*AccessLogEntry, error) {
+	type log struct {
+		Host       string    `json:"host"`
+		Level      string    `json:"level"`
+		Method     string    `json:"method"`
+		Msg        string    `json:"msg"`
+		Proto      string    `json:"proto"`
+		Referrer   string    `json:"referrer"`
+		RequestID  string    `json:"requestID"`
+		Size       int       `json:"size"`
+		StatusCode int       `json:"statusCode"`
+		Time       time.Time `json:"time"`
+		URI        string    `json:"uri"`
+		UserAgent  string    `json:"userAgent"`
 	}
 
-	logLine := strings.Split(rawLog, " ")
+	var l log
+	if err := json.Unmarshal([]byte(rawLog), &l); err != nil {
+		return nil, err
+	}
+
+	// not an access log entry
+	if l.URI == "" {
+		return nil, nil
+	}
 
-	timeStamp := strings.TrimPrefix(logLine[3], "[")
-	method := strings.TrimPrefix(logLine[5], "\"")
-	path := logLine[6]
-	statusCode := logLine[8]
+	// if warning or error, return error with raw log contents
+	if l.Level != "info" {
+		return nil, errors.New(rawLog)
+	}
 
-	return AccessLogEntry{
-		TimeStamp:  timeStamp,
-		Method:     method,
-		Path:       path,
-		StatusCode: statusCode,
+	return &AccessLogEntry{
+		TimeStamp:  l.Time.String(),
+		Method:     l.Method,
+		Path:       l.URI,
+		StatusCode: strconv.Itoa(l.StatusCode),
 	}, nil
 }

cmd/manager/main.go

@@ -35,6 +35,7 @@ import (
 
 	// +kubebuilder:scaffold:imports
 
+	"github.com/getkin/kin-openapi/openapi3"
 	"github.com/go-logr/logr"
 	"github.com/go-logr/zapr"
 	"github.com/kelseyhightower/envconfig"
@@ -65,6 +66,7 @@ import (
 	"github.com/kubeshop/kusk-gateway/internal/validation"
 	"github.com/kubeshop/kusk-gateway/internal/webhooks"
 	"github.com/kubeshop/kusk-gateway/pkg/analytics"
+	"github.com/kubeshop/kusk-gateway/pkg/spec"
 )
 
 var (
@@ -272,7 +274,9 @@ func main() {
 		Validator:          proxy,
 		SecretToEnvoyFleet: map[string]gateway.EnvoyFleetID{},
 		WatchedSecretsChan: secretsChan,
+		OpenApiParser:      spec.NewParser(&openapi3.Loader{IsExternalRefsAllowed: true}),
 	}
+
 	analytics.SendAnonymousInfo(ctx, controllerConfigManager.Client, "kusk", "kusk-gateway manager bootstrapping")
 	heartBeat(ctx, controllerConfigManager.Client)
 

internal/controllers/config_manager.go

@@ -61,6 +61,8 @@ type KubeEnvoyConfigManager struct {
 
 	WatchedSecretsChan chan *v1.Secret
 	SecretToEnvoyFleet map[string]gateway.EnvoyFleetID
+
+	OpenApiParser spec.Parser
 }
 
 var (
@@ -96,10 +98,9 @@ func (c *KubeEnvoyConfigManager) UpdateConfiguration(ctx context.Context, fleetI
 	}
 
 	clBuilder := cloudentity.NewBuilder()
-	parser := spec.NewParser(nil)
 	for _, api := range apis {
 		l.Info("Processing API configuration", "fleet", fleetIDstr, "api", api.Name)
-		apiSpec, err := parser.ParseFromReader(strings.NewReader(api.Spec.Spec))
+		apiSpec, err := c.OpenApiParser.ParseFromReader(strings.NewReader(api.Spec.Spec))
 		if err != nil {
 			return fmt.Errorf("failed to parse OpenAPI spec: %w", err)
 		}

pkg/spec/spec.go

@@ -128,7 +128,7 @@ func parseSwagger(spec []byte) (*openapi3.T, error) {
 }
 
 func parseOpenAPI3(spec []byte) (*openapi3.T, error) {
-	return openapi3.NewLoader().LoadFromData(spec)
+	return (&openapi3.Loader{IsExternalRefsAllowed: true}).LoadFromData(spec)
 }
 
 // GetExampleResponse returns a single example response from the given operation