feat(logparser): add log4j, python

Update: #243

internal/logparser/_golden/genericjson_log4j_01.json

@@ -0,0 +1,21 @@
+{
+  "severity": {
+    "str": "Debug",
+    "number": 5,
+    "text": "DEBUG"
+  },
+  "body": "Debug message",
+  "attrs": {
+    "instant": {
+      "epochSecond": 1696419692,
+      "nanoOfSecond": 479118362
+    },
+    "thread": "main",
+    "loggerName": "CONSOLE_JSON_APPENDER",
+    "endOfBatch": false,
+    "loggerFqcn": "org.apache.logging.log4j.spi.AbstractLogger",
+    "threadId": 1,
+    "threadPriority": 5,
+    "myCustomField": "myCustomValue"
+  }
+}

internal/logparser/_golden/genericjson_logrus_01.json

@@ -1,9 +1,13 @@
 {
-  "severity_number_str": "Info",
-  "severity_number": 9,
-  "severity_text": "info",
+  "severity": {
+    "str": "Info",
+    "number": 9,
+    "text": "info"
+  },
   "body": "A group of walrus emerges from the ocean",
   "timestamp": "2023-12-04T10:54:23Z",
-  "animal": "walrus",
-  "size": 10
+  "attrs": {
+    "animal": "walrus",
+    "size": 10
+  }
 }

internal/logparser/_golden/genericjson_python_01.json

@@ -0,0 +1,29 @@
+{
+  "severity": {
+    "str": "Info",
+    "number": 9,
+    "text": "INFO"
+  },
+  "body": "testing logging format",
+  "attrs": {
+    "threadName": "MainThread",
+    "name": "root",
+    "thread": 140735202359648,
+    "created": 1336281068.506248,
+    "process": 41937,
+    "processName": "MainProcess",
+    "relativeCreated": 9.100914001464844,
+    "module": "tests",
+    "funcName": "testFormatKeys",
+    "levelno": 20,
+    "msecs": 506.24799728393555,
+    "pathname": "tests/tests.py",
+    "lineno": 60,
+    "asctime": [
+      "12-05-05 22:11:08,506248"
+    ],
+    "filename": "tests.py",
+    "special": "value",
+    "run": 12
+  }
+}

internal/logparser/_golden/genericjson_slog_01.json

@@ -1,8 +1,12 @@
 {
-  "severity_number_str": "Info",
-  "severity_number": 9,
-  "severity_text": "INFO",
+  "severity": {
+    "str": "Info",
+    "number": 9,
+    "text": "INFO"
+  },
   "body": "hello world",
   "timestamp": "2023-12-04T10:54:23.012381337Z",
-  "answer": 42
+  "attrs": {
+    "answer": 42
+  }
 }

internal/logparser/_golden/genericjson_slog_02.json

@@ -1,6 +1,8 @@
 {
-  "severity_number_str": "Info",
-  "severity_number": 9,
-  "severity_text": "INFO",
+  "severity": {
+    "str": "Info",
+    "number": 9,
+    "text": "INFO"
+  },
   "body": "No time"
 }

internal/logparser/_golden/genericjson_zap_01.json

@@ -1,22 +1,26 @@
 {
-  "severity_number_str": "Info",
-  "severity_number": 9,
-  "severity_text": "info",
+  "severity": {
+    "str": "Info",
+    "number": 9,
+    "text": "info"
+  },
   "body": "Events",
   "timestamp": "2023-12-04T09:11:59.1613184Z",
-  "logger": "poll",
-  "caller": "gh-archived/main.go:197",
-  "duration": 0.437301455,
-  "integers": [
-    1,
-    2,
-    3
-  ],
-  "new_count": 100,
-  "remaining": 1040,
-  "used": 3960,
-  "reset": 660.838704106,
-  "reset_human": "in 12 minutes",
-  "sleep": 0.198120375,
-  "target_rate": 0.63542183
+  "attrs": {
+    "logger": "poll",
+    "caller": "gh-archived/main.go:197",
+    "duration": 0.437301455,
+    "integers": [
+      1,
+      2,
+      3
+    ],
+    "new_count": 100,
+    "remaining": 1040,
+    "used": 3960,
+    "reset": 660.838704106,
+    "reset_human": "in 12 minutes",
+    "sleep": 0.198120375,
+    "target_rate": 0.63542183
+  }
 }

internal/logparser/_golden/genericjson_zerolog_01.json

@@ -1,9 +1,13 @@
 {
-  "severity_number_str": "Debug",
-  "severity_number": 5,
-  "severity_text": "debug",
+  "severity": {
+    "str": "Debug",
+    "number": 5,
+    "text": "debug"
+  },
   "body": "Fibonacci is everywhere",
   "timestamp": "2023-12-04T10:59:00Z",
-  "Scale": "833 cents",
-  "Interval": 833.09
+  "attrs": {
+    "Scale": "833 cents",
+    "Interval": 833.09
+  }
 }

internal/logparser/_golden/genericjson_zerolog_02.json

@@ -1,7 +1,11 @@
 {
-  "severity_number_str": "Debug",
-  "severity_number": 5,
-  "severity_text": "debug",
+  "severity": {
+    "str": "Debug",
+    "number": 5,
+    "text": "debug"
+  },
   "timestamp": "2023-12-04T10:59:00Z",
-  "Name": "Tom"
+  "attrs": {
+    "Name": "Tom"
+  }
 }

internal/logparser/_testdata/genericjson/log4j.jsonl

@@ -0,0 +1 @@
+{"instant":{"epochSecond":1696419692,"nanoOfSecond":479118362},"thread":"main","level":"DEBUG","loggerName":"CONSOLE_JSON_APPENDER","message":"Debug message","endOfBatch":false,"loggerFqcn":"org.apache.logging.log4j.spi.AbstractLogger","threadId":1,"threadPriority":5,"myCustomField":"myCustomValue"}

internal/logparser/_testdata/genericjson/python.jsonl

@@ -0,0 +1 @@
+{"threadName":"MainThread","name":"root","thread":140735202359648,"created":1336281068.506248,"process":41937,"processName":"MainProcess","relativeCreated":9.100914001464844,"module":"tests","funcName":"testFormatKeys","levelno":20,"msecs":506.24799728393555,"pathname":"tests/tests.py","lineno":60,"asctime":["12-05-05 22:11:08,506248"],"message":"testing logging format","filename":"tests.py","levelname":"INFO","special":"value","run":12}

internal/logparser/json.go

@@ -238,7 +238,7 @@ func (GenericJSONParser) Parse(data []byte) (*Line, error) {
 			var spanID otelstorage.SpanID
 			copy(spanID[:], raw)
 			line.SpanID = spanID
-		case "level", "lvl", "levelStr", "severity_text", "severity":
+		case "level", "lvl", "levelStr", "severity_text", "severity", "levelname":
 			if d.Next() != jx.String {
 				return addJSONMapKey(attrs, string(k), d)
 			}

internal/logparser/line.go

@@ -32,17 +32,24 @@ func (l Line) String() string {
 // Encode line as json.
 func (l Line) Encode(e *jx.Encoder) {
 	e.Obj(func(e *jx.Encoder) {
-		if l.SeverityNumber != 0 {
-			e.Field("severity_number_str", func(e *jx.Encoder) {
-				e.Str(l.SeverityNumber.String())
-			})
-			e.Field("severity_number", func(e *jx.Encoder) {
-				e.Int64(int64(l.SeverityNumber))
-			})
-		}
-		if l.SeverityText != "" {
-			e.Field("severity_text", func(e *jx.Encoder) {
-				e.Str(l.SeverityText)
+		hasSeverity := l.SeverityNumber != 0 || l.SeverityText != ""
+		if hasSeverity {
+			e.Field("severity", func(e *jx.Encoder) {
+				e.Obj(func(e *jx.Encoder) {
+					if l.SeverityNumber != 0 {
+						e.Field("str", func(e *jx.Encoder) {
+							e.Str(l.SeverityNumber.String())
+						})
+						e.Field("number", func(e *jx.Encoder) {
+							e.Int64(int64(l.SeverityNumber))
+						})
+					}
+					if l.SeverityText != "" {
+						e.Field("text", func(e *jx.Encoder) {
+							e.Str(l.SeverityText)
+						})
+					}
+				})
 			})
 		}
 		if l.Body != "" {
@@ -55,13 +62,6 @@ func (l Line) Encode(e *jx.Encoder) {
 				e.Str(l.Timestamp.AsTime().Format(time.RFC3339Nano))
 			})
 		}
-		if !l.Attrs.IsZero() {
-			l.Attrs.AsMap().Range(func(k string, v pcommon.Value) bool {
-				return !e.Field(k, func(e *jx.Encoder) {
-					encodeValue(v, e)
-				})
-			})
-		}
 		if !l.TraceID.IsEmpty() {
 			e.Field("trace_id", func(e *jx.Encoder) {
 				e.Str(hex.EncodeToString(l.TraceID[:]))
@@ -72,5 +72,16 @@ func (l Line) Encode(e *jx.Encoder) {
 				e.Str(hex.EncodeToString(l.SpanID[:]))
 			})
 		}
+		if !l.Attrs.IsZero() {
+			e.Field("attrs", func(e *jx.Encoder) {
+				e.Obj(func(e *jx.Encoder) {
+					l.Attrs.AsMap().Range(func(k string, v pcommon.Value) bool {
+						return !e.Field(k, func(e *jx.Encoder) {
+							encodeValue(v, e)
+						})
+					})
+				})
+			})
+		}
 	})
 }