Change Access logging to JSON and Update kusk-gateway-dashboard ima…

…ge (#1033) Issue and Discussion ==================== See: #784. Resolves #784. Example Output ============== ```json {"response_code":200,"bytes_received":0,"bytes_sent":47,"upstream_local_address":"172.17.0.5:50434","response_flags":"-","start_time":"2022-12-14T14:51:21.281Z","protocol":"HTTP/1.1","upstream_transport_failure_reason":null,"upstream_host":"10.96.207.174:80","requested_server_name":null,"duration":0,"downstream_local_address":"172.17.0.5:8080","downstream_remote_address":"172.17.0.1:62724","response_code_details":"via_upstream","x_forwarded_for":null,"method":"GET","upstream_cluster":"traffic-splitting-httpbin-1.default.svc.cluster.local.-80","authority":"traffic-splitting-httpbin-1.default.svc.cluster.local.","route_name":"/uuid-GET","path":"/uuid","connection_termination_details":null,"request_id":"1c15c266-7e62-4dee-b5fa-cec1c83237f4","user_agent":"curl/7.85.0","upstream_service_time":"0"} ``` Changes ======= `config/samples/gateway_v1_envoyfleet.yaml` ------------------------------------------- Default to `json` format for `envoy.access_loggers.stdout`. `internal/envoy/config/logging.go` ---------------------------------- Slight refactoring. `cmd/kusk/manifests/dashboard.yaml` ----------------------------------- Update image to `kubeshop/kusk-gateway-dashboard:v1.2.6`. `cmd/kusk/manifests/fleets.yaml`, `cmd/kusk/manifests/devportal.yaml` and `cmd/kusk/manifests/dashboard_envoyfleet.yaml` ------------------------------------------------------------------------------------------------------------------------ Change ``` apiVersion: gateway.kusk.io/v1alpha1 kind: EnvoyFleet ``` to use JSON as the logging format by using: ```yaml accesslog: format: json ``` Same applies to: * `smoketests/basic/envoyfleet.yaml`. * `smoketests/mocking/envoyfleet.yaml`. * `smoketests/mocking/envoyfleet.yaml`. References ========== 1. https://www.envoyproxy.io/docs/envoy/v1.23.1/configuration/observability/access_log/usage#config-access-log. 2. https://istio.io/latest/docs/tasks/observability/logs/access-log/#default-access-log-format. --- Signed-off-by: Mohamed Bana <[email protected]>
kubeshop · Dec 15, 2022 · 37c8dc8 · 37c8dc8
1 parent 11166ef
commit 37c8dc8
Show file tree

Hide file tree

Showing 8 changed files with 49 additions and 85 deletions.
diff --git a/cmd/kusk/manifests/dashboard.yaml b/cmd/kusk/manifests/dashboard.yaml
@@ -28,7 +28,7 @@ metadata:
   labels:
     app.kubernetes.io/instance: kusk-gateway-dashboard
     app.kubernetes.io/name: kusk-gateway-dashboard
-    app.kubernetes.io/version: v1.1.0-stable
+    app.kubernetes.io/version: v1.2.6
 spec:
   selector:
     matchLabels:
@@ -46,7 +46,7 @@ spec:
         app.kubernetes.io/name: kusk-gateway-dashboard
     spec:
       containers:
-        - image: kubeshop/kusk-gateway-dashboard:v1.2.5
+        - image: kubeshop/kusk-gateway-dashboard:v1.2.6
           imagePullPolicy: Always
           livenessProbe:
             httpGet:

diff --git a/cmd/kusk/manifests/dashboard_envoyfleet.yaml b/cmd/kusk/manifests/dashboard_envoyfleet.yaml
@@ -5,7 +5,7 @@ metadata:
   namespace: kusk-system
 spec:
   accesslog:
-    format: text
+    format: json
   service:
     ports:
       - name: http

diff --git a/cmd/kusk/manifests/devportal.yaml b/cmd/kusk/manifests/devportal.yaml
@@ -35,7 +35,7 @@ spec:
             limits:
               memory: 512Mi
             requests:
-                memory: 256Mi
+              memory: 256Mi
 ---
 apiVersion: v1
 kind: Service
@@ -61,7 +61,7 @@ metadata:
   namespace: kusk-system
 spec:
   accesslog:
-    format: text
+    format: json
   service:
     ports:
       - name: http

diff --git a/cmd/kusk/manifests/fleets.yaml b/cmd/kusk/manifests/fleets.yaml
@@ -6,7 +6,7 @@ metadata:
 spec:
   default: true
   accesslog:
-    format: text
+    format: json
   service:
     ports:
       - name: http

diff --git a/config/samples/gateway_v1_envoyfleet.yaml b/config/samples/gateway_v1_envoyfleet.yaml
@@ -72,23 +72,7 @@ spec:
   # If the entry is missing no access logging will be done
   accesslog:
     # json|text
-    format: text
-    # Depending on format we can specify our own log template or if template is not specified - default Kusk Gateway will be used
-    # Below are specified the examples of similar and minimalistic formats for both text and json format types.
-    # Text format fields order is preserved.
-    # The output example:
-    # "[2021-12-15T16:50:50.217Z]" "GET" "/" "200" "1"
-    text_template: |
-      "[%START_TIME%]" "%REQ(:METHOD)%" "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" "%RESPONSE_CODE%" "%DURATION%"
-    # Json format fields order isn't preserved
-    # The output example:
-    # {"start_time":"2021-12-15T16:46:52.135Z","path":"/","response_code":200,"method":"GET","duration":1}
-    json_template:
-      start_time: "%START_TIME%"
-      method: "%REQ(:METHOD)%"
-      path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%"
-      response_code: "%RESPONSE_CODE%"
-      duration: "%DURATION%"
+    format: json
 
   # OPTIONAL - specify TLS options for HTTPS traffic
   # If TLS specified, only tlsSecrets is a mandatory field

diff --git a/internal/envoy/config/logging.go b/internal/envoy/config/logging.go
@@ -36,49 +36,6 @@ import (
 const (
 	AccessLogFormatJson string = "json"
 	AccessLogFormatText string = "text"
-
-	// The name of the Envoy extention that handles stdout access log
-	AccessLogStdoutName string = "envoy.access_loggers.stdout"
-
-	defaultTextLogTemplate = "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% " +
-		"%PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% " +
-		"%RESPONSE_CODE_DETAILS% %CONNECTION_TERMINATION_DETAILS% " +
-		"\"%UPSTREAM_TRANSPORT_FAILURE_REASON%\" %BYTES_RECEIVED% %BYTES_SENT% " +
-		"%DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(X-FORWARDED-FOR)%\" " +
-		"\"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\" " +
-		"%UPSTREAM_CLUSTER% %UPSTREAM_LOCAL_ADDRESS% %DOWNSTREAM_LOCAL_ADDRESS% " +
-		"%DOWNSTREAM_REMOTE_ADDRESS% %REQUESTED_SERVER_NAME% %ROUTE_NAME%\n"
-)
-
-var (
-	defaultJsonLogTemplate = &structpb.Struct{
-		Fields: map[string]*structpb.Value{
-			"start_time":                        {Kind: &structpb.Value_StringValue{StringValue: "%START_TIME%"}},
-			"route_name":                        {Kind: &structpb.Value_StringValue{StringValue: "%ROUTE_NAME%"}},
-			"method":                            {Kind: &structpb.Value_StringValue{StringValue: "%REQ(:METHOD)%"}},
-			"path":                              {Kind: &structpb.Value_StringValue{StringValue: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%"}},
-			"protocol":                          {Kind: &structpb.Value_StringValue{StringValue: "%PROTOCOL%"}},
-			"response_code":                     {Kind: &structpb.Value_StringValue{StringValue: "%RESPONSE_CODE%"}},
-			"response_flags":                    {Kind: &structpb.Value_StringValue{StringValue: "%RESPONSE_FLAGS%"}},
-			"response_code_details":             {Kind: &structpb.Value_StringValue{StringValue: "%RESPONSE_CODE_DETAILS%"}},
-			"connection_termination_details":    {Kind: &structpb.Value_StringValue{StringValue: "%CONNECTION_TERMINATION_DETAILS%"}},
-			"bytes_received":                    {Kind: &structpb.Value_StringValue{StringValue: "%BYTES_RECEIVED%"}},
-			"bytes_sent":                        {Kind: &structpb.Value_StringValue{StringValue: "%BYTES_SENT%"}},
-			"duration":                          {Kind: &structpb.Value_StringValue{StringValue: "%DURATION%"}},
-			"upstream_service_time":             {Kind: &structpb.Value_StringValue{StringValue: "%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%"}},
-			"x_forwarded_for":                   {Kind: &structpb.Value_StringValue{StringValue: "%REQ(X-FORWARDED-FOR)%"}},
-			"user_agent":                        {Kind: &structpb.Value_StringValue{StringValue: "%REQ(USER-AGENT)%"}},
-			"request_id":                        {Kind: &structpb.Value_StringValue{StringValue: "%REQ(X-REQUEST-ID)%"}},
-			"authority":                         {Kind: &structpb.Value_StringValue{StringValue: "%REQ(:AUTHORITY)%"}},
-			"upstream_host":                     {Kind: &structpb.Value_StringValue{StringValue: "%UPSTREAM_HOST%"}},
-			"upstream_cluster":                  {Kind: &structpb.Value_StringValue{StringValue: "%UPSTREAM_CLUSTER%"}},
-			"upstream_local_address":            {Kind: &structpb.Value_StringValue{StringValue: "%UPSTREAM_LOCAL_ADDRESS%"}},
-			"downstream_local_address":          {Kind: &structpb.Value_StringValue{StringValue: "%DOWNSTREAM_LOCAL_ADDRESS%"}},
-			"downstream_remote_address":         {Kind: &structpb.Value_StringValue{StringValue: "%DOWNSTREAM_REMOTE_ADDRESS%"}},
-			"requested_server_name":             {Kind: &structpb.Value_StringValue{StringValue: "%REQUESTED_SERVER_NAME%"}},
-			"upstream_transport_failure_reason": {Kind: &structpb.Value_StringValue{StringValue: "%UPSTREAM_TRANSPORT_FAILURE_REASON%"}},
-		},
-	}
 )
 
 type AccessLogBuilder struct {
@@ -94,6 +51,38 @@ func (a *AccessLogBuilder) GetAccessLog() *accesslog.AccessLog {
 }
 
 func NewJSONAccessLog(template map[string]string) (*AccessLogBuilder, error) {
+	var (
+		// See https://istio.io/latest/docs/tasks/observability/logs/access-log/#default-access-log-format.
+		defaultJsonLogTemplate = &structpb.Struct{
+			Fields: map[string]*structpb.Value{
+				"start_time":                        {Kind: &structpb.Value_StringValue{StringValue: "%START_TIME%"}},
+				"method":                            {Kind: &structpb.Value_StringValue{StringValue: "%REQ(:METHOD)%"}},
+				"path":                              {Kind: &structpb.Value_StringValue{StringValue: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%"}},
+				"protocol":                          {Kind: &structpb.Value_StringValue{StringValue: "%PROTOCOL%"}},
+				"response_code":                     {Kind: &structpb.Value_StringValue{StringValue: "%RESPONSE_CODE%"}},
+				"response_flags":                    {Kind: &structpb.Value_StringValue{StringValue: "%RESPONSE_FLAGS%"}},
+				"response_code_details":             {Kind: &structpb.Value_StringValue{StringValue: "%RESPONSE_CODE_DETAILS%"}},
+				"connection_termination_details":    {Kind: &structpb.Value_StringValue{StringValue: "%CONNECTION_TERMINATION_DETAILS%"}},
+				"upstream_transport_failure_reason": {Kind: &structpb.Value_StringValue{StringValue: "%UPSTREAM_TRANSPORT_FAILURE_REASON%"}},
+				"bytes_received":                    {Kind: &structpb.Value_StringValue{StringValue: "%BYTES_RECEIVED%"}},
+				"bytes_sent":                        {Kind: &structpb.Value_StringValue{StringValue: "%BYTES_SENT%"}},
+				"duration":                          {Kind: &structpb.Value_StringValue{StringValue: "%DURATION%"}},
+				"upstream_service_time":             {Kind: &structpb.Value_StringValue{StringValue: "%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%"}},
+				"x_forwarded_for":                   {Kind: &structpb.Value_StringValue{StringValue: "%REQ(X-FORWARDED-FOR)%"}},
+				"user_agent":                        {Kind: &structpb.Value_StringValue{StringValue: "%REQ(USER-AGENT)%"}},
+				"request_id":                        {Kind: &structpb.Value_StringValue{StringValue: "%REQ(X-REQUEST-ID)%"}},
+				"authority":                         {Kind: &structpb.Value_StringValue{StringValue: "%REQ(:AUTHORITY)%"}},
+				"upstream_host":                     {Kind: &structpb.Value_StringValue{StringValue: "%UPSTREAM_HOST%"}},
+				"upstream_cluster":                  {Kind: &structpb.Value_StringValue{StringValue: "%UPSTREAM_CLUSTER%"}},
+				"upstream_local_address":            {Kind: &structpb.Value_StringValue{StringValue: "%UPSTREAM_LOCAL_ADDRESS%"}},
+				"downstream_local_address":          {Kind: &structpb.Value_StringValue{StringValue: "%DOWNSTREAM_LOCAL_ADDRESS%"}},
+				"downstream_remote_address":         {Kind: &structpb.Value_StringValue{StringValue: "%DOWNSTREAM_REMOTE_ADDRESS%"}},
+				"requested_server_name":             {Kind: &structpb.Value_StringValue{StringValue: "%REQUESTED_SERVER_NAME%"}},
+				"route_name":                        {Kind: &structpb.Value_StringValue{StringValue: "%ROUTE_NAME%"}},
+			},
+		}
+	)
+
 	// Default template on the start
 	var formatTemplate *structpb.Struct = defaultJsonLogTemplate
 
@@ -120,6 +109,13 @@ func NewJSONAccessLog(template map[string]string) (*AccessLogBuilder, error) {
 }
 
 func NewTextAccessLog(template string) (*AccessLogBuilder, error) {
+	const (
+		// See https://istio.io/latest/docs/tasks/observability/logs/access-log/#default-access-log-format
+		defaultTextLogTemplate = `[%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %RESPONSE_CODE_DETAILS% %CONNECTION_TERMINATION_DETAILS%
+"%UPSTREAM_TRANSPORT_FAILURE_REASON%" %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%"
+"%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" %UPSTREAM_CLUSTER% %UPSTREAM_LOCAL_ADDRESS% %DOWNSTREAM_LOCAL_ADDRESS% %DOWNSTREAM_REMOTE_ADDRESS% %REQUESTED_SERVER_NAME% %ROUTE_NAME%\n`
+	)
+
 	var formatTemplate string = defaultTextLogTemplate
 	if template != "" {
 		formatTemplate = template
@@ -152,7 +148,7 @@ func accessLogFinalize(accessLogFormatString *core.SubstitutionFormatString) (*A
 	}
 
 	accessLog := &accesslog.AccessLog{
-		Name: AccessLogStdoutName,
+		Name: "envoy.access_loggers.stdout",
 		ConfigType: &accesslog.AccessLog_TypedConfig{
 			TypedConfig: anyAccessLog,
 		},

diff --git a/smoketests/basic/envoyfleet.yaml b/smoketests/basic/envoyfleet.yaml
@@ -15,12 +15,4 @@ spec:
         targetPort: http
   size: 1
   accesslog:
-    format: text
-    text_template: |
-      "[%START_TIME%]" "%REQ(:METHOD)%" "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" "%RESPONSE_CODE%" "%DURATION%"
-    json_template:
-      start_time: "%START_TIME%"
-      method: "%REQ(:METHOD)%"
-      path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%"
-      response_code: "%RESPONSE_CODE%"
-      duration: "%DURATION%"
+    format: json
diff --git a/smoketests/mocking/envoyfleet.yaml b/smoketests/mocking/envoyfleet.yaml
@@ -15,12 +15,4 @@ spec:
         targetPort: http
   size: 1
   accesslog:
-    format: text
-    text_template: |
-      "[%START_TIME%]" "%REQ(:METHOD)%" "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" "%RESPONSE_CODE%" "%DURATION%"
-    json_template:
-      start_time: "%START_TIME%"
-      method: "%REQ(:METHOD)%"
-      path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%"
-      response_code: "%RESPONSE_CODE%"
-      duration: "%DURATION%"
+    format: json