exporter/debug: json encoding for verbose payloads

[ringerc]

Is your feature request related to a problem? Please describe.

It's presently difficult and inconvenient to obtain a list of metrics and attributes being generated from the collector in an easily summarised form. The debug exporter with verbosity: detailed provides it, but in a custom text format that is not easily machine parsed and processed.

Describe the solution you'd like

It would be brilliant if the debug exporter supported an encoding: json option like service.telemetry.logs does. When set, the detailed output format would be output as json payloads.

These could easily be extracted from the collector's log stream and post-processed to produce aggregates of resources, metrics, attributes etc.

Useful for automated testing and validation, diagnosis, reporting, etc.

Describe alternatives you've considered

I can (and do) parse this information out of the current debug exporter output but it's difficult and inconvenient to parse the current representation.

The file exporter has a format: json option that can be used to generate json representations of exporter payloads. But it's inconvenient to access the resulting file when running the collector in a containerised environment especially in stacks like k8s. It may also require changes to the deployment manifests to provide a writeable location to send the logs to, temporarily lifting security policies limiting writeable container mount points, etc. If path is set to /dev/stdout it should be possible to get the metrics stream from stdout and the logs from stderr, but some stacks (like k8s) don't make it easy to capture the two streams separately, so they get merged into an unusable mess. The file exporter doesn't support routing its output to the otel collector's logger where it'd be merged into the stderr stream for output.

This means there's no easy way to get the metrics in json form via kubectl logs or kubectl cp. You can't access the resulting files produced by the file exporter easily with the os-less container because kubectl cp needs a tar binary and kubectl exec needs a shell or something else to run. Nothing like that exists (or should exist) in the os-less otel collector container.

It's possible to use the file exporter to write to a file then read that file back in with the file receiver and send that to the debug exporter's verbose mode for dumping into the collector log stream but that's unspeakably ugly.

Additional context

Maybe loosely related to #7806

[ian-howell]

I'm very glad to see that someone else has the same issue. There's already an existing JSON Marshaler for Metrics, but it doesn't appear to be used outside of unit tests for some reason. The JSON Marshaler implements the plog.Marshaler. And the debug exporter uses an instance of that interface here. I don't think it should be terribly difficult to plumb some configuration into the constructor to get this working.

I'll take a look over the contributing guide and give this a shot over the weekend.

[andrewdinunzio]

Yeah I'm trying to parse out the text and this would definitely be useful

[andrewdinunzio]

Well here's something I hacked up quickly to parse the log output. May have bugs (assumes it will always be the year 2024 :D) and may need to be adjusted as the output format changes:

package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"regexp"
	"strings"

	"encoding/json"
)

type Scanner struct {
	*bufio.Scanner
	peeked     bool
	peekedLine string
}

func NewScanner(scanner *bufio.Scanner) *Scanner {
	return &Scanner{Scanner: scanner}
}

// Peek returns true if there is a line to read and stores it if so. It can be read with Text().
// It does not advance the scanner.
func (s *Scanner) Peek() bool {
	if s.peeked {
		return true
	}
	if s.Scan() {
		s.peekedLine = s.Text()
		s.peeked = true
		return true
	}
	return false
}

// Scan advances the scanner and returns true if there is a line to read.
func (s *Scanner) Scan() bool {
	if s.peeked {
		s.peeked = false
		return true
	}
	return s.Scanner.Scan()
}

func (s *Scanner) Text() string {
	if s.peeked {
		return s.peekedLine
	}
	return s.Scanner.Text()
}

type SpanEvent struct {
	Name                   string            `json:"name"`
	Timestamp              string            `json:"timestamp"`
	DroppedAttributesCount string            `json:"droppedAttributesCount"`
	Attributes             map[string]string `json:"attributes"`
}

type Span struct {
	TraceID       string            `json:"traceID"`
	ParentID      string            `json:"parentID"`
	ID            string            `json:"id"`
	Name          string            `json:"name"`
	Kind          string            `json:"kind"`
	TraceState    string            `json:"traceState"`
	StartTime     string            `json:"startTime"`
	EndTime       string            `json:"endTime"`
	StatusCode    string            `json:"statusCode"`
	StatusMessage string            `json:"statusMessage"`
	Attributes    map[string]string `json:"attributes"`
	Events        []SpanEvent       `json:"events"`
}

type ScopeSpan struct {
	SchemaURL            string `json:"schemaURL"`
	InstrumentationScope string `json:"instrumentationScope"`
	Spans                []Span `json:"spans"`
}

type ResourceSpan struct {
	SchemaURL  string            `json:"schemaURL"`
	Attributes map[string]string `json:"attributes"`
	ScopeSpans []ScopeSpan       `json:"scopeSpans"`
}

type LogEntry struct {
	Timestamp     string         `json:"timestamp"`
	Level         string         `json:"level"`
	ResourceSpans []ResourceSpan `json:"resourceSpans"`
}

func parseSpan(scanner *Scanner) Span {
	var span Span
	span.Attributes = make(map[string]string)

	for scanner.Peek() {
		line := scanner.Text()
		line = strings.TrimSpace(line)

		if strings.HasPrefix(line, "Trace ID") {
			span.TraceID = strings.TrimSpace(line[strings.Index(line, ":")+1:])
			scanner.Scan()
		} else if strings.HasPrefix(line, "Parent ID") {
			span.ParentID = strings.TrimSpace(line[strings.Index(line, ":")+1:])
			scanner.Scan()
		} else if strings.HasPrefix(line, "ID") {
			span.ID = strings.TrimSpace(line[strings.Index(line, ":")+1:])
			scanner.Scan()
		} else if strings.HasPrefix(line, "Name") {
			span.Name = strings.TrimSpace(line[strings.Index(line, ":")+1:])
			scanner.Scan()
		} else if strings.HasPrefix(line, "Kind") {
			span.Kind = strings.TrimSpace(line[strings.Index(line, ":")+1:])
			scanner.Scan()
		} else if strings.HasPrefix(line, "TraceState") {
			span.TraceState = strings.TrimSpace(line[strings.Index(line, ":")+1:])
			scanner.Scan()
		} else if strings.HasPrefix(line, "Start time") {
			span.StartTime = strings.TrimSpace(line[strings.Index(line, ":")+1:])
			scanner.Scan()
		} else if strings.HasPrefix(line, "End time") {
			span.EndTime = strings.TrimSpace(line[strings.Index(line, ":")+1:])
			scanner.Scan()
		} else if strings.HasPrefix(line, "Status code") {
			span.StatusCode = strings.TrimSpace(line[strings.Index(line, ":")+1:])
			scanner.Scan()
		} else if strings.HasPrefix(line, "Status message") {
			span.StatusMessage = strings.TrimSpace(line[strings.Index(line, ":")+1:])
			scanner.Scan()
		} else if strings.HasPrefix(line, "Attributes:") {
			scanner.Scan()
			span.Attributes = parseAttributes(scanner)
		} else if strings.HasPrefix(line, "Events:") {
			scanner.Scan()
			span.Events = parseEvents(scanner)
		} else {
			break
		}
	}

	return span
}

func parseSpanEvent(scanner *Scanner) SpanEvent {
	var event SpanEvent
	event.Attributes = make(map[string]string)

	for scanner.Peek() {
		line := scanner.Text()
		line = strings.TrimSpace(line)

		if strings.HasPrefix(line, "-> Name:") {
			event.Name = strings.TrimSpace(line[strings.Index(line, ":")+1:])
			scanner.Scan()
		} else if strings.HasPrefix(line, "-> Timestamp:") {
			event.Timestamp = strings.TrimSpace(line[strings.Index(line, ":")+1:])
			scanner.Scan()
		} else if strings.HasPrefix(line, "-> DroppedAttributesCount:") {
			event.DroppedAttributesCount = strings.TrimSpace(line[strings.Index(line, ":")+1:])
			scanner.Scan()
		} else if strings.HasPrefix(line, "-> Attributes:") {
			scanner.Scan()
			event.Attributes = parseAttributes(scanner)
		} else {
			break
		}
	}

	return event
}

func parseEvents(scanner *Scanner) []SpanEvent {
	var events []SpanEvent
	for scanner.Peek() {
		line := scanner.Text()
		if strings.HasPrefix(line, "SpanEvent #") {
			scanner.Scan()
			events = append(events, parseSpanEvent(scanner))
		} else {
			break
		}
	}
	return events
}

func parseScopeSpan(scanner *Scanner) ScopeSpan {
	var scope ScopeSpan
	for scanner.Peek() {
		line := scanner.Text()
		if strings.HasPrefix(line, "ScopeSpans SchemaURL:") {
			scope.SchemaURL = strings.TrimSpace(line[strings.Index(line, ":")+1:])
			scanner.Scan()
		} else if strings.HasPrefix(line, "InstrumentationScope") {
			scope.InstrumentationScope = strings.TrimSpace(strings.Replace(line, "InstrumentationScope", "", 1))
			scanner.Scan()
		} else if strings.HasPrefix(line, "Span #") {
			scanner.Scan()
			scope.Spans = append(scope.Spans, parseSpan(scanner))
		} else {
			break
		}
	}
	return scope
}

func parseResourceSpan(scanner *Scanner) ResourceSpan {
	var res ResourceSpan
	for scanner.Peek() {
		line := scanner.Text()
		if strings.HasPrefix(line, "Resource attributes:") {
			scanner.Scan()
			res.Attributes = parseResourceAttributes(scanner)
		} else if strings.HasPrefix(line, "Resource SchemaURL:") {
			res.SchemaURL = strings.TrimSpace(line[strings.Index(line, ":")+1:])
			scanner.Scan()
		} else if strings.HasPrefix(line, "ScopeSpans #") {
			scanner.Scan()
			res.ScopeSpans = append(res.ScopeSpans, parseScopeSpan(scanner))
		} else {
			break
		}
	}
	return res
}

func parseResourceAttributes(scanner *Scanner) map[string]string {
	attributes := make(map[string]string)
	attrRegex := regexp.MustCompile(`\s*->\s*(\S+):\s*(\S+)\((.+)\)`)

	for scanner.Peek() {
		line := scanner.Text()
		if attrRegex.MatchString(line) {
			matches := attrRegex.FindStringSubmatch(line)
			attributes[matches[1]] = matches[3]
			scanner.Scan()
		} else {
			break
		}
	}

	return attributes
}

func parseAttributes(scanner *Scanner) map[string]string {
	attributes := make(map[string]string)
	attrRegex := regexp.MustCompile(`\s*->\s*(\S+):\s*(\S+)\((.*)\)`)

	for scanner.Peek() {
		line := scanner.Text()
		if attrRegex.MatchString(line) {
			matches := attrRegex.FindStringSubmatch(line)
			attributes[matches[1]] = matches[3]
			scanner.Scan()
		} else {
			break
		}
	}

	return attributes
}

func parseLogEntry(scanner *Scanner) LogEntry {
	entry := LogEntry{}
	entry.ResourceSpans = make([]ResourceSpan, 0)

	resRegex := regexp.MustCompile(`ResourceSpans #\d+$`)

	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, "2024") {
			entry.Timestamp = strings.SplitN(line, "\t", 3)[0]
			entry.Level = strings.SplitN(line, "\t", 3)[1]
		}
		if resRegex.MatchString(line) {
			entry.ResourceSpans = append(entry.ResourceSpans, parseResourceSpan(scanner))
		} else {
			break
		}
	}

	return entry
}

func parseLogEntries(scanner *Scanner) []LogEntry {
	entries := make([]LogEntry, 0)

	resRegex := regexp.MustCompile(`ResourceSpans #\d+$`)

	for scanner.Peek() {
		line := scanner.Text()
		if strings.HasPrefix(line, "2024") && resRegex.MatchString(line) {
			entries = append(entries, parseLogEntry(scanner))
		} else {
			scanner.Scan()
		}
	}

	return entries
}

func main() {
	file, err := os.Open("log.txt")
	if err != nil {
		fmt.Println("Error opening file:", err)
		return
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	entries := parseLogEntries(NewScanner(scanner))

	jsonEntries, err := json.Marshal(entries)
	if err != nil {
		log.Fatal(err)
	}
	err = os.WriteFile("out.json", jsonEntries, 0644)
	if err != nil {
		log.Fatal(err)
	}
}