From aef49d09fb1697afe14e4f7e2557b67a0bc87b16 Mon Sep 17 00:00:00 2001 From: Aleksandr Razumov Date: Mon, 4 Dec 2023 13:58:02 +0300 Subject: [PATCH] feat(logparser): add slog, logrus Updates: #243 --- .../_golden/genericjson_logrus_01.json | 9 ++++ .../_golden/genericjson_slog_01.json | 8 ++++ .../_golden/genericjson_slog_02.json | 6 +++ ...icjson_01.json => genericjson_zap_01.json} | 1 + .../_testdata/genericjson/logrus.jsonl | 1 + .../_testdata/genericjson/slog.jsonl | 2 + internal/logparser/json.go | 3 +- internal/logparser/json_test.go | 41 ++++++++++++++----- internal/logparser/line.go | 5 ++- 9 files changed, 63 insertions(+), 13 deletions(-) create mode 100644 internal/logparser/_golden/genericjson_logrus_01.json create mode 100644 internal/logparser/_golden/genericjson_slog_01.json create mode 100644 internal/logparser/_golden/genericjson_slog_02.json rename internal/logparser/_golden/{genericjson_01.json => genericjson_zap_01.json} (94%) create mode 100644 internal/logparser/_testdata/genericjson/logrus.jsonl create mode 100644 internal/logparser/_testdata/genericjson/slog.jsonl diff --git a/internal/logparser/_golden/genericjson_logrus_01.json b/internal/logparser/_golden/genericjson_logrus_01.json new file mode 100644 index 00000000..53b22e79 --- /dev/null +++ b/internal/logparser/_golden/genericjson_logrus_01.json @@ -0,0 +1,9 @@ +{ + "severity_number_str": "Info", + "severity_number": 9, + "severity_text": "info", + "body": "A group of walrus emerges from the ocean", + "timestamp": "2023-12-04T10:54:23Z", + "animal": "walrus", + "size": 10 +} \ No newline at end of file diff --git a/internal/logparser/_golden/genericjson_slog_01.json b/internal/logparser/_golden/genericjson_slog_01.json new file mode 100644 index 00000000..2146f097 --- /dev/null +++ b/internal/logparser/_golden/genericjson_slog_01.json @@ -0,0 +1,8 @@ +{ + "severity_number_str": "Info", + "severity_number": 9, + "severity_text": "INFO", + "body": "hello world", + "timestamp": "2023-12-04T10:54:23.012381337Z", + "answer": 42 +} \ No newline at end of file diff --git a/internal/logparser/_golden/genericjson_slog_02.json b/internal/logparser/_golden/genericjson_slog_02.json new file mode 100644 index 00000000..094d371d --- /dev/null +++ b/internal/logparser/_golden/genericjson_slog_02.json @@ -0,0 +1,6 @@ +{ + "severity_number_str": "Info", + "severity_number": 9, + "severity_text": "INFO", + "body": "No time" +} \ No newline at end of file diff --git a/internal/logparser/_golden/genericjson_01.json b/internal/logparser/_golden/genericjson_zap_01.json similarity index 94% rename from internal/logparser/_golden/genericjson_01.json rename to internal/logparser/_golden/genericjson_zap_01.json index f24e2adf..0ee64dde 100644 --- a/internal/logparser/_golden/genericjson_01.json +++ b/internal/logparser/_golden/genericjson_zap_01.json @@ -1,5 +1,6 @@ { "severity_number_str": "Info", + "severity_number": 9, "severity_text": "info", "body": "Events", "timestamp": "2023-12-04T09:11:59.1613184Z", diff --git a/internal/logparser/_testdata/genericjson/logrus.jsonl b/internal/logparser/_testdata/genericjson/logrus.jsonl new file mode 100644 index 00000000..20083a22 --- /dev/null +++ b/internal/logparser/_testdata/genericjson/logrus.jsonl @@ -0,0 +1 @@ +{"animal":"walrus","level":"info","msg":"A group of walrus emerges from the ocean","size":10,"time":"2023-12-04T13:54:23+03:00"} \ No newline at end of file diff --git a/internal/logparser/_testdata/genericjson/slog.jsonl b/internal/logparser/_testdata/genericjson/slog.jsonl new file mode 100644 index 00000000..a92b4cec --- /dev/null +++ b/internal/logparser/_testdata/genericjson/slog.jsonl @@ -0,0 +1,2 @@ +{"time":"2023-12-04T13:54:23.012381337+03:00","level":"INFO","msg":"hello world","answer":42} +{"level":"INFO","msg":"No time"} \ No newline at end of file diff --git a/internal/logparser/json.go b/internal/logparser/json.go index 01fea525..a0a35511 100644 --- a/internal/logparser/json.go +++ b/internal/logparser/json.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" "time" + "unicode" "github.com/go-faster/errors" "github.com/go-faster/jx" @@ -250,7 +251,7 @@ func (GenericJSONParser) Parse(data []byte) (*Line, error) { return nil } line.SeverityText = v - line.SeverityNumber = _severityMap[rune(v[0])] + line.SeverityNumber = _severityMap[unicode.ToLower(rune(v[0]))] case msgField: if d.Next() != jx.String { return addJSONMapKey(attrs, string(k), d) diff --git a/internal/logparser/json_test.go b/internal/logparser/json_test.go index b87f2038..a52d5fb5 100644 --- a/internal/logparser/json_test.go +++ b/internal/logparser/json_test.go @@ -6,28 +6,47 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" + "github.com/go-faster/jx" "github.com/go-faster/sdk/gold" "github.com/stretchr/testify/require" ) func TestGenericJSONParser_Parse(t *testing.T) { - data, err := os.ReadFile(filepath.Join("_testdata", "genericjson", "zap.jsonl")) + files, err := os.ReadDir(filepath.Join("_testdata", "genericjson")) require.NoError(t, err, "read testdata") - var parser GenericJSONParser - scanner := bufio.NewScanner(bytes.NewReader(data)) + for _, file := range files { + t.Run(file.Name(), func(t *testing.T) { + data, err := os.ReadFile(filepath.Join("_testdata", "genericjson", file.Name())) + require.NoError(t, err, "read testdata") - var i int - for scanner.Scan() { - i++ - t.Run(fmt.Sprintf("Line%02d", i), func(t *testing.T) { - line, err := parser.Parse(scanner.Bytes()) - require.NoError(t, err, "parse") + var parser GenericJSONParser + + scanner := bufio.NewScanner(bytes.NewReader(data)) - name := fmt.Sprintf("genericjson_%02d.json", i) - gold.Str(t, line.String(), name) + var i int + for scanner.Scan() { + s := strings.TrimSpace(scanner.Text()) + if s == "" { + continue + } + i++ + t.Run(fmt.Sprintf("Line%02d", i), func(t *testing.T) { + t.Logf("%s", s) + if !jx.Valid(scanner.Bytes()) { + t.Fatal("invalid") + } + line, err := parser.Parse([]byte(s)) + require.NoError(t, err, "parse") + name := fmt.Sprintf("genericjson_%s_%02d.json", + strings.TrimSuffix(file.Name(), filepath.Ext(file.Name())), i, + ) + gold.Str(t, line.String(), name) + }) + } }) } } diff --git a/internal/logparser/line.go b/internal/logparser/line.go index 5dc261ed..ba9300b0 100644 --- a/internal/logparser/line.go +++ b/internal/logparser/line.go @@ -36,6 +36,9 @@ func (l Line) Encode(e *jx.Encoder) { e.Field("severity_number_str", func(e *jx.Encoder) { e.Str(l.SeverityNumber.String()) }) + e.Field("severity_number", func(e *jx.Encoder) { + e.Int64(int64(l.SeverityNumber)) + }) } if l.SeverityText != "" { e.Field("severity_text", func(e *jx.Encoder) { @@ -47,7 +50,7 @@ func (l Line) Encode(e *jx.Encoder) { e.Str(l.Body) }) } - if !l.Timestamp.AsTime().IsZero() { + if !l.Timestamp.AsTime().IsZero() && l.Timestamp != 0 { e.Field("timestamp", func(e *jx.Encoder) { e.Str(l.Timestamp.AsTime().Format(time.RFC3339Nano)) })