Merge pull request #1840 from rahulsadanandan/xray-examples

Added fluentbit logging examples for xray

examples/xray/logging/README.md

@@ -0,0 +1,22 @@
+# Setting up a logging sidecar
+Due to the nature of Xray running with many services, each writing multiple logs to the file system, it's hard to collect them all in a Kubernetes based deployment.<br>
+The example in this directory has an example of using a [fluent-bit](https://fluentbit.io/) sidecar that collects all logs from Xray's `log/` directory and writes them to STDOUT in a nice json formatted way
+
+See the [values-logging-fluent-bit.yaml](values-logging-fluent-bit.yaml) for the configuration example
+
+## Deploy
+Install Xray with the following command
+```shell
+helm upgrade --install xray jfrog/xray -f values-logging-fluent-bit.yaml
+```
+
+## Fluent-bit STDOUT
+Once running, the `fluent-bit` sidecar tails the logs in the configured directories and outputs them to the container's STDOUT in a json format.<br>
+Each line had a `"file"` key that lists the source file, which later can be used to separate the sources.<br>
+The actual log line is in the `"log"` key.
+```json
+{"date":1700135001.943258,"file":"/var/opt/jfrog/xray/log/xray-server-service.log","log":"2023-11-16T11:43:21.942Z \u001b[33m[jfxr ]\u001b[0m \u001b[34m[INFO ]\u001b[0m [0e860ce4f1fd6552] [migrate:36                    ] [MainServer                      ] Data migration that should run always: starting for name V1_Update_Features_table"}
+```
+
+## Cluster log collector
+Once this is setup, you need to configure your cluster log collector (probably running as a DaemonSet) to collect logs from this container only.

examples/xray/logging/values-logging-fluent-bit.yaml

@@ -0,0 +1,68 @@
+common:
+
+  # Create extra config maps with the required fluent-bit configuration
+  configMaps: |
+    fluent-bit-input.conf: |-
+      [INPUT]
+          Name              tail
+          Path              /var/opt/jfrog/xray/log/*.log
+          DB                /var/opt/jfrog/fluentdb/fluentdb.db
+          Exclude_Path      *router-request.log,*console.log,*-metrics.log
+          Path_Key          file
+          Refresh_Interval  20
+          Multiline         On
+          Parser_Firstline  multiline_pattern
+          Skip_Long_Lines   On
+
+    fluent-bit-output.conf: |-
+      [OUTPUT]
+          Name   stdout
+          Match  *
+          Format json_lines
+
+    fluent-bit-service.conf: |-
+      [SERVICE]
+          Flush        1
+          Daemon       Off
+          Log_Level    info
+          Parsers_File parsers.conf
+
+    fluent-bit.conf: |-
+      @INCLUDE fluent-bit-service.conf
+      @INCLUDE fluent-bit-input.conf
+      @INCLUDE fluent-bit-output.conf
+
+    parsers.conf: |-
+      [PARSER]
+          Name      multiline_pattern
+          Format    regex
+          Regex     ^(?<log>\d{2,4}\-\d{2,4}\-\d{2,4}T\d{2,4}\:\d{2,4}\:\d{2,4}\.\d{1,6}?.*)
+
+  # Create the extra sidecar container
+  customSidecarContainers: |
+    - name: xray-fluent-bit
+      image: "fluent/fluent-bit:2.1.2"
+      volumeMounts:
+      - mountPath: /var/opt/jfrog/xray
+        name: data-volume
+      - mountPath: /var/opt/jfrog/fluentdb
+        name: fluentdb
+      - mountPath: /fluent-bit/etc/fluent-bit.conf
+        name: xray-configmaps
+        subPath: fluent-bit.conf
+      - mountPath: /fluent-bit/etc/fluent-bit-service.conf
+        name: xray-configmaps
+        subPath: fluent-bit-service.conf
+      - mountPath: /fluent-bit/etc/fluent-bit-input.conf
+        name: xray-configmaps
+        subPath: fluent-bit-input.conf
+      - mountPath: /fluent-bit/etc/fluent-bit-output.conf
+        name: xray-configmaps
+        subPath: fluent-bit-output.conf
+      - mountPath: /fluent-bit/etc/parsers.conf
+        name: xray-configmaps
+        subPath: parsers.conf
+
+  customVolumes: |
+    - name: fluentdb
+      emptyDir: {}