fix: join multiparts on file beginning (#3784)

* fix: join multiparts on file beginning Signed-off-by: Dominik Rosiek <[email protected]> * chore: changelog Signed-off-by: Dominik Rosiek <[email protected]> * Apply suggestions from code review * Apply suggestions from code review --------- Signed-off-by: Dominik Rosiek <[email protected]> (cherry picked from commit 58f19f8)
SumoLogic · Jun 28, 2024 · 67f2e76 · 67f2e76
1 parent 1d42a8b
commit 67f2e76
Show file tree

Hide file tree

Showing 9 changed files with 19 additions and 15 deletions.
diff --git a/.changelog/3784.fixed.txt b/.changelog/3784.fixed.txt
@@ -0,0 +1 @@
+fix: join multiparts on file beginning
diff --git a/deploy/helm/sumologic/conf/logs/collector/common/filelog_receiver.yaml b/deploy/helm/sumologic/conf/logs/collector/common/filelog_receiver.yaml
@@ -99,7 +99,8 @@ filelog/containers:
       output: strip-trailing-newline
       source_identifier: attributes["log.file.path"]
       type: recombine
-      max_unmatched_batch_size: 1
+      ## Ensure we combine everything up to `is_last_entry` even on the file beginning
+      max_unmatched_batch_size: 0
 
     ## merge-cri-lines stitches back together log lines split by CRI logging drivers.
     ## Input Body (JSON): { "log": "2001-02-03 04:05:06 very long li", "logtag": "P" }
@@ -113,7 +114,8 @@ filelog/containers:
       overwrite_with: newest
       source_identifier: attributes["log.file.path"]
       type: recombine
-      max_unmatched_batch_size: 1
+      ## Ensure we combine everything up to `is_last_entry` even on the file beginning
+      max_unmatched_batch_size: 0
 
     ## strip-trailing-newline removes the trailing "\n" from the `log` key. This is required for logs coming from Docker container runtime.
     ## Input Body (JSON): { "log": "2001-02-03 04:05:06 very long line that was split by the logging driver\n", "stream": "stdout" }

diff --git a/deploy/helm/sumologic/conf/logs/collector/otelcloudwatch/config.yaml b/deploy/helm/sumologic/conf/logs/collector/otelcloudwatch/config.yaml
@@ -61,7 +61,8 @@ processors:
         overwrite_with: newest
         source_identifier: resource["cloudwatch.log.stream"]
         type: recombine
-        max_unmatched_batch_size: 1
+        ## Ensure we combine everything up to `is_last_entry` even on the file beginning
+        max_unmatched_batch_size: 0
       - id: merge-multiline-logs
         combine_field: attributes.log
         combine_with: "\n"

diff --git a/tests/helm/testdata/goldenfile/logs_otc/basic.output.yaml b/tests/helm/testdata/goldenfile/logs_otc/basic.output.yaml
@@ -99,15 +99,15 @@ data:
           combine_with: ""
           id: merge-docker-lines
           is_last_entry: body.log matches "\n$"
-          max_unmatched_batch_size: 1
+          max_unmatched_batch_size: 0
           output: strip-trailing-newline
           source_identifier: attributes["log.file.path"]
           type: recombine
         - combine_field: body.log
           combine_with: ""
           id: merge-cri-lines
           is_last_entry: body.logtag == "F"
-          max_unmatched_batch_size: 1
+          max_unmatched_batch_size: 0
           output: extract-metadata-from-filepath
           overwrite_with: newest
           source_identifier: attributes["log.file.path"]

diff --git a/tests/helm/testdata/goldenfile/logs_otc/debug.output.yaml b/tests/helm/testdata/goldenfile/logs_otc/debug.output.yaml
@@ -109,15 +109,15 @@ data:
           combine_with: ""
           id: merge-docker-lines
           is_last_entry: body.log matches "\n$"
-          max_unmatched_batch_size: 1
+          max_unmatched_batch_size: 0
           output: strip-trailing-newline
           source_identifier: attributes["log.file.path"]
           type: recombine
         - combine_field: body.log
           combine_with: ""
           id: merge-cri-lines
           is_last_entry: body.logtag == "F"
-          max_unmatched_batch_size: 1
+          max_unmatched_batch_size: 0
           output: extract-metadata-from-filepath
           overwrite_with: newest
           source_identifier: attributes["log.file.path"]

diff --git a/tests/helm/testdata/goldenfile/logs_otc_daemonset/multiple_multiline.output.yaml b/tests/helm/testdata/goldenfile/logs_otc_daemonset/multiple_multiline.output.yaml
@@ -99,15 +99,15 @@ data:
           combine_with: ""
           id: merge-docker-lines
           is_last_entry: body.log matches "\n$"
-          max_unmatched_batch_size: 1
+          max_unmatched_batch_size: 0
           output: strip-trailing-newline
           source_identifier: attributes["log.file.path"]
           type: recombine
         - combine_field: body.log
           combine_with: ""
           id: merge-cri-lines
           is_last_entry: body.logtag == "F"
-          max_unmatched_batch_size: 1
+          max_unmatched_batch_size: 0
           output: extract-metadata-from-filepath
           overwrite_with: newest
           source_identifier: attributes["log.file.path"]

diff --git a/tests/helm/testdata/goldenfile/logs_otc_windows/configmap/basic.output.yaml b/tests/helm/testdata/goldenfile/logs_otc_windows/configmap/basic.output.yaml
@@ -78,15 +78,15 @@ data:
           combine_with: ""
           id: merge-docker-lines
           is_last_entry: body.log matches "\n$"
-          max_unmatched_batch_size: 1
+          max_unmatched_batch_size: 0
           output: strip-trailing-newline
           source_identifier: attributes["log.file.path"]
           type: recombine
         - combine_field: body.log
           combine_with: ""
           id: merge-cri-lines
           is_last_entry: body.logtag == "F"
-          max_unmatched_batch_size: 1
+          max_unmatched_batch_size: 0
           output: extract-metadata-from-filepath
           overwrite_with: newest
           source_identifier: attributes["log.file.path"]

diff --git a/tests/helm/testdata/goldenfile/logs_otc_windows/configmap/debug.output.yaml b/tests/helm/testdata/goldenfile/logs_otc_windows/configmap/debug.output.yaml
@@ -89,15 +89,15 @@ data:
           combine_with: ""
           id: merge-docker-lines
           is_last_entry: body.log matches "\n$"
-          max_unmatched_batch_size: 1
+          max_unmatched_batch_size: 0
           output: strip-trailing-newline
           source_identifier: attributes["log.file.path"]
           type: recombine
         - combine_field: body.log
           combine_with: ""
           id: merge-cri-lines
           is_last_entry: body.logtag == "F"
-          max_unmatched_batch_size: 1
+          max_unmatched_batch_size: 0
           output: extract-metadata-from-filepath
           overwrite_with: newest
           source_identifier: attributes["log.file.path"]

diff --git a/tests/helm/testdata/goldenfile/logs_otc_windows/daemonset/multiple_multiline.output.yaml b/tests/helm/testdata/goldenfile/logs_otc_windows/daemonset/multiple_multiline.output.yaml
@@ -78,15 +78,15 @@ data:
           combine_with: ""
           id: merge-docker-lines
           is_last_entry: body.log matches "\n$"
-          max_unmatched_batch_size: 1
+          max_unmatched_batch_size: 0
           output: strip-trailing-newline
           source_identifier: attributes["log.file.path"]
           type: recombine
         - combine_field: body.log
           combine_with: ""
           id: merge-cri-lines
           is_last_entry: body.logtag == "F"
-          max_unmatched_batch_size: 1
+          max_unmatched_batch_size: 0
           output: extract-metadata-from-filepath
           overwrite_with: newest
           source_identifier: attributes["log.file.path"]