Refactored how firecore tools handles printing

chain.go

@@ -325,6 +325,13 @@ func (c *Chain[B]) LoggerPackageID(subPackage string) string {
 	return fmt.Sprintf("%s/%s", c.FullyQualifiedModule, subPackage)
 }
 
+// BlockFileDescriptor returns the `protoreflect.FileDescriptor` of the chain's block
+// extracted from the block factory defined on the chain. This would resolve for example
+// to Proto file descriptor `sf/ethereum/type/v2/type.proto` for Ethereum.
+func (c *Chain[B]) BlockFileDescriptor() protoreflect.FileDescriptor {
+	return c.BlockFactory().ProtoReflect().Descriptor().ParentFile()
+}
+
 // VersionString computes the version string that will be display when calling `firexxx --version`
 // and extract build information from Git via Golang `debug.ReadBuildInfo`.
 func (c *Chain[B]) VersionString() string {

cmd/tools/check/blocks.go

@@ -2,7 +2,6 @@ package check
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"io"
 	"os"
@@ -181,6 +180,8 @@ func validateBlockSegment[B firecore.Block](
 		return
 	}
 
+	printer := print2.TextOutputPrinter{}
+
 	seenBlockCount := 0
 	for {
 		block, err := readerFactory.Read()
@@ -231,14 +232,20 @@ func validateBlockSegment[B firecore.Block](
 			seenBlockCount++
 
 			if printDetails == PrintStats {
-				err := print2.PrintBStreamBlock(block, false, os.Stdout)
+				err := printer.PrintTo(block, os.Stdout)
 				if err != nil {
 					fmt.Printf("❌ Unable to print block %s: %s\n", block.AsRef(), err)
 					continue
 				}
 			}
 
 			if printDetails == PrintFull {
+				printer, err := print2.GetOutputPrinter(globalToolsCheckCmd, chain.BlockFileDescriptor())
+				if err != nil {
+					fmt.Printf("❌ Unable to create output printer: %s\n", err)
+					break
+				}
+
 				var b = chain.BlockFactory()
 
 				if _, ok := b.(*pbbstream.Block); ok {
@@ -251,14 +258,11 @@ func validateBlockSegment[B firecore.Block](
 					break
 				}
 
-				out, err := json.MarshalIndent(b, "", "  ")
-
+				err = printer.PrintTo(b, os.Stdout)
 				if err != nil {
 					fmt.Printf("❌ Unable to print full block %s: %s\n", block.AsRef(), err)
 					continue
 				}
-
-				fmt.Println(string(out))
 			}
 
 			continue

cmd/tools/check/check.go

@@ -31,7 +31,12 @@ import (
 	"golang.org/x/exp/maps"
 )
 
-func NewCheckCommand[B firecore.Block](chain *firecore.Chain[B], rootLog *zap.Logger) *cobra.Command {
+// Super hackish way to get the *cobra.command needed for sflags call but where
+// CheckMergedBlocks public method doesn't receive the *cobra.Command
+var globalToolsCheckCmd *cobra.Command
+
+func NewCheckCommand[B firecore.Block](chain *firecore.Chain[B], rootLog *zap.Logger) (out *cobra.Command) {
+	defer func() { globalToolsCheckCmd = out }()
 
 	toolsCheckCmd := &cobra.Command{Use: "check", Short: "Various checks for deployment, data integrity & debugging"}
 

cmd/tools/compare/tools_compare_blocks.go

@@ -73,9 +73,7 @@ func NewToolsCompareBlocksCmd[B firecore.Block](chain *firecore.Chain[B]) *cobra
 
 	flags := cmd.PersistentFlags()
 	flags.Bool("diff", false, "When activated, difference is displayed for each block with a difference")
-	flags.String("bytes-encoding", "hex", "Encoding for bytes fields, either 'hex' or 'base58'")
 	flags.Bool("include-unknown-fields", false, "When activated, the 'unknown fields' in the protobuf message will also be compared. These would not generate any difference when unmarshalled with the current protobuf definition.")
-	flags.StringSlice("proto-paths", []string{""}, "Paths to proto files to use for dynamic decoding of blocks")
 
 	return cmd
 }

cmd/tools/firehose/client.go

@@ -1,6 +1,7 @@
 package firehose
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io"
@@ -24,10 +25,8 @@ func NewToolsFirehoseClientCmd[B firecore.Block](chain *firecore.Chain[B], logge
 
 	addFirehoseStreamClientFlagsToSet(cmd.Flags(), chain)
 
-	cmd.Flags().StringSlice("proto-paths", []string{""}, "Paths to proto files to use for dynamic decoding of blocks")
 	cmd.Flags().Bool("final-blocks-only", false, "Only ask for final blocks")
 	cmd.Flags().Bool("print-cursor-only", false, "Skip block decoding, only print the step cursor (useful for performance testing)")
-	cmd.Flags().String("bytes-encoding", "hex", "Encoding for bytes fields, either 'hex' or 'base58'")
 
 	return cmd
 }
@@ -90,9 +89,9 @@ func getFirehoseClientE[B firecore.Block](chain *firecore.Chain[B], rootLog *zap
 			}()
 		}
 
-		jencoder, err := print.SetupJsonMarshaller(cmd, chain.BlockFactory().ProtoReflect().Descriptor().ParentFile())
+		printer, err := print.GetOutputPrinter(cmd, chain.BlockFileDescriptor())
 		if err != nil {
-			return fmt.Errorf("unable to create json encoder: %w", err)
+			return fmt.Errorf("unable to create output printer: %w", err)
 		}
 
 		for {
@@ -116,12 +115,15 @@ func getFirehoseClientE[B firecore.Block](chain *firecore.Chain[B], rootLog *zap
 
 			// async process the response
 			go func() {
-				line, err := jencoder.MarshalToString(response)
+				buffer := bytes.NewBuffer(nil)
+				err := printer.PrintTo(response, buffer)
 				if err != nil {
 					rootLog.Error("marshalling to string", zap.Error(err))
+					resp.ch <- ""
+					return
 				}
 
-				resp.ch <- line
+				resp.ch <- buffer.String()
 			}()
 		}
 		if printCursorOnly {

cmd/tools/firehose/single_block_client.go

@@ -3,12 +3,14 @@ package firehose
 import (
 	"context"
 	"fmt"
+	"os"
 	"strconv"
 	"strings"
 
 	"github.com/spf13/cobra"
+	"github.com/streamingfast/cli"
 	firecore "github.com/streamingfast/firehose-core"
-	"github.com/streamingfast/jsonpb"
+	"github.com/streamingfast/firehose-core/cmd/tools/print"
 	"github.com/streamingfast/logging"
 	pbfirehose "github.com/streamingfast/pbgo/sf/firehose/v2"
 	"go.uber.org/zap"
@@ -18,9 +20,16 @@ import (
 func NewToolsFirehoseSingleBlockClientCmd[B firecore.Block](chain *firecore.Chain[B], zlog *zap.Logger, tracer logging.Tracer) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "firehose-single-block-client {endpoint} {block_num|block_num:block_id|cursor}",
-		Short: "fetch a single block from firehose and print as JSON",
-		Args:  cobra.ExactArgs(2),
-		RunE:  getFirehoseSingleBlockClientE(chain, zlog, tracer),
+		Short: "Performs a FetchClient#Block call against a Firehose endpoint and print the response",
+		Long: string(cli.Description(`
+			Performs a sf.firehose.v2.Fetch/Block call against a Firehose endpoint and print the full response
+			object.
+
+			By default, the response is printed in JSON format, but you can use the --output flag to
+			choose a different output format (text, json, jsonl, protojson, protojsonl).
+		`)),
+		Args: cobra.ExactArgs(2),
+		RunE: getFirehoseSingleBlockClientE(chain, zlog, tracer),
 		Example: firecore.ExamplePrefixed(chain, "tools ", `
 			firehose-single-block-client --compression=gzip my.firehose.endpoint:443 2344:0x32d8e8d98a798da98d6as9d69899as86s9898d8ss8d87
 		`),
@@ -76,11 +85,11 @@ func getFirehoseSingleBlockClientE[B firecore.Block](chain *firecore.Chain[B], z
 			return err
 		}
 
-		line, err := jsonpb.MarshalToString(resp)
-		if err != nil {
-			return err
-		}
-		fmt.Println(line)
+		printer, err := print.GetOutputPrinter(cmd, chain.BlockFileDescriptor())
+		cli.NoError(err, "Unable to get output printer")
+
+		cli.NoError(printer.PrintTo(resp, os.Stdout), "Unable to print block")
+
 		return nil
 	}
 }

cmd/tools/print/printer.go

@@ -0,0 +1,131 @@
+// Copyright 2021 dfuse Platform Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package print
+
+import (
+	"fmt"
+	"io"
+	"strconv"
+	"unsafe"
+
+	"github.com/spf13/cobra"
+	"github.com/streamingfast/cli/sflags"
+	fcproto "github.com/streamingfast/firehose-core/proto"
+	"google.golang.org/protobuf/reflect/protoreflect"
+)
+
+func GetOutputPrinter(cmd *cobra.Command, chainFileDescriptor protoreflect.FileDescriptor) (OutputPrinter, error) {
+	printer := sflags.MustGetString(cmd, "output")
+	if printer == "" {
+		printer = "jsonl"
+	}
+
+	var protoPaths []string
+	if sflags.FlagDefined(cmd, "proto-paths") {
+		protoPaths = sflags.MustGetStringSlice(cmd, "proto-paths")
+	}
+
+	bytesEncoding := "hex"
+	if sflags.FlagDefined(cmd, "bytes-encoding") {
+		bytesEncoding = sflags.MustGetString(cmd, "bytes-encoding")
+	}
+
+	registry, err := fcproto.NewRegistry(chainFileDescriptor, protoPaths...)
+	if err != nil {
+		return nil, fmt.Errorf("new registry: %w", err)
+	}
+
+	if printer == "json" || printer == "jsonl" {
+		jsonPrinter, err := NewJSONOutputPrinter(bytesEncoding, printer == "jsonl", registry)
+		if err != nil {
+			return nil, fmt.Errorf("unable to create json encoder: %w", err)
+		}
+
+		return jsonPrinter, nil
+	}
+
+	if printer == "protojson" || printer == "protojsonl" {
+		indent := ""
+		if printer == "protojson" {
+			indent = "  "
+		}
+
+		return NewProtoJSONOutputPrinter(indent, registry), nil
+	}
+
+	if printer == "text" {
+		// Supports the `transactions` flag defined on `firecore tools print` sub-command,
+		// we should move it to a proper `text` sub-option like `output-text-details` or something
+		// like that.
+		printTransactions := false
+		if sflags.FlagDefined(cmd, "transactions") {
+			printTransactions = sflags.MustGetBool(cmd, "transactions")
+		}
+
+		return NewTextOutputPrinter(bytesEncoding, registry, printTransactions), nil
+	}
+
+	return nil, fmt.Errorf("unsupported output printer %q", printer)
+}
+
+//go:generate go-enum -f=$GOFILE --marshal --names --nocase
+
+// ENUM(Text, JSON, JSONL, ProtoJSON, ProtoJSONL)
+type PrintOutputMode uint
+
+type OutputPrinter interface {
+	PrintTo(message any, w io.Writer) error
+}
+
+func writeStringToWriter(w io.Writer, str string) error {
+	return writeBytesToWriter(w, unsafe.Slice(unsafe.StringData(str), len(str)))
+}
+
+func writeStringFToWriter(w io.Writer, format string, args ...any) error {
+	return writeStringToWriter(w, fmt.Sprintf(format, args...))
+}
+
+func writeBytesToWriter(w io.Writer, data []byte) error {
+	n, err := w.Write(data)
+	if err != nil {
+		return err
+	}
+
+	if n != len(data) {
+		return io.ErrShortWrite
+	}
+
+	return nil
+}
+
+func ptr[T any](v T) *T {
+	return &v
+}
+
+func deref[T any](v *T, orDefault T) T {
+	if v == nil {
+		return orDefault
+	}
+
+	return *v
+}
+
+func uint64PtrToString(v *uint64, orDefault string) string {
+	if v == nil {
+		return orDefault
+	}
+
+	return strconv.FormatUint(*v, 10)
+}