Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactored how firecore tools handles printing #69

Open
wants to merge 3 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions chain.go
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,13 @@ func (c *Chain[B]) LoggerPackageID(subPackage string) string {
return fmt.Sprintf("%s/%s", c.FullyQualifiedModule, subPackage)
}

// BlockFileDescriptor returns the `protoreflect.FileDescriptor` of the chain's block
// extracted from the block factory defined on the chain. This would resolve for example
// to Proto file descriptor `sf/ethereum/type/v2/type.proto` for Ethereum.
func (c *Chain[B]) BlockFileDescriptor() protoreflect.FileDescriptor {
return c.BlockFactory().ProtoReflect().Descriptor().ParentFile()
}

// VersionString computes the version string that will be display when calling `firexxx --version`
// and extract build information from Git via Golang `debug.ReadBuildInfo`.
func (c *Chain[B]) VersionString() string {
Expand Down
16 changes: 10 additions & 6 deletions cmd/tools/check/blocks.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package check

import (
"context"
"encoding/json"
"fmt"
"io"
"os"
Expand Down Expand Up @@ -181,6 +180,8 @@ func validateBlockSegment[B firecore.Block](
return
}

printer := print2.TextOutputPrinter{}

seenBlockCount := 0
for {
block, err := readerFactory.Read()
Expand Down Expand Up @@ -231,14 +232,20 @@ func validateBlockSegment[B firecore.Block](
seenBlockCount++

if printDetails == PrintStats {
err := print2.PrintBStreamBlock(block, false, os.Stdout)
err := printer.PrintTo(block, os.Stdout)
if err != nil {
fmt.Printf("❌ Unable to print block %s: %s\n", block.AsRef(), err)
continue
}
}

if printDetails == PrintFull {
printer, err := print2.GetOutputPrinter(globalToolsCheckCmd, chain.BlockFileDescriptor())
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I plan to remove all printing support from check.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

agreed.

if err != nil {
fmt.Printf("❌ Unable to create output printer: %s\n", err)
break
}

var b = chain.BlockFactory()

if _, ok := b.(*pbbstream.Block); ok {
Expand All @@ -251,14 +258,11 @@ func validateBlockSegment[B firecore.Block](
break
}

out, err := json.MarshalIndent(b, "", " ")

err = printer.PrintTo(b, os.Stdout)
if err != nil {
fmt.Printf("❌ Unable to print full block %s: %s\n", block.AsRef(), err)
continue
}

fmt.Println(string(out))
}

continue
Expand Down
7 changes: 6 additions & 1 deletion cmd/tools/check/check.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,12 @@ import (
"golang.org/x/exp/maps"
)

func NewCheckCommand[B firecore.Block](chain *firecore.Chain[B], rootLog *zap.Logger) *cobra.Command {
// Super hackish way to get the *cobra.command needed for sflags call but where
// CheckMergedBlocks public method doesn't receive the *cobra.Command
var globalToolsCheckCmd *cobra.Command

func NewCheckCommand[B firecore.Block](chain *firecore.Chain[B], rootLog *zap.Logger) (out *cobra.Command) {
defer func() { globalToolsCheckCmd = out }()

toolsCheckCmd := &cobra.Command{Use: "check", Short: "Various checks for deployment, data integrity & debugging"}

Expand Down
14 changes: 6 additions & 8 deletions cmd/tools/compare/tools_compare_blocks.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import (
"github.com/streamingfast/dstore"
firecore "github.com/streamingfast/firehose-core"
"github.com/streamingfast/firehose-core/cmd/tools/check"
"github.com/streamingfast/firehose-core/json"
fcjson "github.com/streamingfast/firehose-core/json"
fcproto "github.com/streamingfast/firehose-core/proto"
"github.com/streamingfast/firehose-core/types"
"go.uber.org/multierr"
Expand Down Expand Up @@ -73,9 +73,7 @@ func NewToolsCompareBlocksCmd[B firecore.Block](chain *firecore.Chain[B]) *cobra

flags := cmd.PersistentFlags()
flags.Bool("diff", false, "When activated, difference is displayed for each block with a difference")
flags.String("bytes-encoding", "hex", "Encoding for bytes fields, either 'hex' or 'base58'")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here you replaced with the global bytes-encoding that supports more values (like base64), but the implementation in tools_compare_blocks.go does NOT support anything else than hex or base58, watch out there needs to be a few changes later in this file, in func Compare(...)

flags.Bool("include-unknown-fields", false, "When activated, the 'unknown fields' in the protobuf message will also be compared. These would not generate any difference when unmarshalled with the current protobuf definition.")
flags.StringSlice("proto-paths", []string{""}, "Paths to proto files to use for dynamic decoding of blocks")

return cmd
}
Expand Down Expand Up @@ -363,16 +361,16 @@ func Compare(reference proto.Message, current proto.Message, includeUnknownField

//todo: check if there is a equals that do not compare unknown fields
if !proto.Equal(reference, current) {
var opts []json.MarshallerOption
var opts []fcjson.MarshallerOption
if !includeUnknownFields {
opts = append(opts, json.WithoutUnknownFields())
opts = append(opts, fcjson.WithoutUnknownFields())
}

if bytesEncoding == "base58" {
opts = append(opts, json.WithBytesEncoderFunc(json.ToBase58))
if bytesEncoding != "" {
opts = append(opts, fcjson.WithBytesEncoding(bytesEncoding))
}

encoder := json.NewMarshaller(registry, opts...)
encoder := fcjson.NewMarshaller(registry, opts...)

referenceAsJSON, err := encoder.MarshalToString(reference)
cli.NoError(err, "marshal JSON reference")
Expand Down
14 changes: 8 additions & 6 deletions cmd/tools/firehose/client.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package firehose

import (
"bytes"
"context"
"fmt"
"io"
Expand All @@ -24,10 +25,8 @@ func NewToolsFirehoseClientCmd[B firecore.Block](chain *firecore.Chain[B], logge

addFirehoseStreamClientFlagsToSet(cmd.Flags(), chain)

cmd.Flags().StringSlice("proto-paths", []string{""}, "Paths to proto files to use for dynamic decoding of blocks")
cmd.Flags().Bool("final-blocks-only", false, "Only ask for final blocks")
cmd.Flags().Bool("print-cursor-only", false, "Skip block decoding, only print the step cursor (useful for performance testing)")
cmd.Flags().String("bytes-encoding", "hex", "Encoding for bytes fields, either 'hex' or 'base58'")

return cmd
}
Expand Down Expand Up @@ -90,9 +89,9 @@ func getFirehoseClientE[B firecore.Block](chain *firecore.Chain[B], rootLog *zap
}()
}

jencoder, err := print.SetupJsonMarshaller(cmd, chain.BlockFactory().ProtoReflect().Descriptor().ParentFile())
printer, err := print.GetOutputPrinter(cmd, chain.BlockFileDescriptor())
if err != nil {
return fmt.Errorf("unable to create json encoder: %w", err)
return fmt.Errorf("unable to create output printer: %w", err)
}

for {
Expand All @@ -116,12 +115,15 @@ func getFirehoseClientE[B firecore.Block](chain *firecore.Chain[B], rootLog *zap

// async process the response
go func() {
line, err := jencoder.MarshalToString(response)
buffer := bytes.NewBuffer(nil)
err := printer.PrintTo(response, buffer)
if err != nil {
rootLog.Error("marshalling to string", zap.Error(err))
resp.ch <- ""
return
}

resp.ch <- line
resp.ch <- buffer.String()
}()
}
if printCursorOnly {
Expand Down
27 changes: 18 additions & 9 deletions cmd/tools/firehose/single_block_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,14 @@ package firehose
import (
"context"
"fmt"
"os"
"strconv"
"strings"

"github.com/spf13/cobra"
"github.com/streamingfast/cli"
firecore "github.com/streamingfast/firehose-core"
"github.com/streamingfast/jsonpb"
"github.com/streamingfast/firehose-core/cmd/tools/print"
"github.com/streamingfast/logging"
pbfirehose "github.com/streamingfast/pbgo/sf/firehose/v2"
"go.uber.org/zap"
Expand All @@ -18,9 +20,16 @@ import (
func NewToolsFirehoseSingleBlockClientCmd[B firecore.Block](chain *firecore.Chain[B], zlog *zap.Logger, tracer logging.Tracer) *cobra.Command {
cmd := &cobra.Command{
Use: "firehose-single-block-client {endpoint} {block_num|block_num:block_id|cursor}",
Short: "fetch a single block from firehose and print as JSON",
Args: cobra.ExactArgs(2),
RunE: getFirehoseSingleBlockClientE(chain, zlog, tracer),
Short: "Performs a FetchClient#Block call against a Firehose endpoint and print the response",
Long: string(cli.Description(`
Performs a sf.firehose.v2.Fetch/Block call against a Firehose endpoint and print the full response
object.

By default, the response is printed in JSON format, but you can use the --output flag to
choose a different output format (text, json, jsonl, protojson, protojsonl).
`)),
Args: cobra.ExactArgs(2),
RunE: getFirehoseSingleBlockClientE(chain, zlog, tracer),
Example: firecore.ExamplePrefixed(chain, "tools ", `
firehose-single-block-client --compression=gzip my.firehose.endpoint:443 2344:0x32d8e8d98a798da98d6as9d69899as86s9898d8ss8d87
`),
Expand Down Expand Up @@ -76,11 +85,11 @@ func getFirehoseSingleBlockClientE[B firecore.Block](chain *firecore.Chain[B], z
return err
}

line, err := jsonpb.MarshalToString(resp)
if err != nil {
return err
}
fmt.Println(line)
printer, err := print.GetOutputPrinter(cmd, chain.BlockFileDescriptor())
cli.NoError(err, "Unable to get output printer")

cli.NoError(printer.PrintTo(resp, os.Stdout), "Unable to print block")

return nil
}
}
131 changes: 131 additions & 0 deletions cmd/tools/print/printer.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
// Copyright 2021 dfuse Platform Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package print

import (
"fmt"
"io"
"strconv"
"unsafe"

"github.com/spf13/cobra"
"github.com/streamingfast/cli/sflags"
fcproto "github.com/streamingfast/firehose-core/proto"
"google.golang.org/protobuf/reflect/protoreflect"
)

func GetOutputPrinter(cmd *cobra.Command, chainFileDescriptor protoreflect.FileDescriptor) (OutputPrinter, error) {
printer := sflags.MustGetString(cmd, "output")
if printer == "" {
printer = "jsonl"
}

var protoPaths []string
if sflags.FlagDefined(cmd, "proto-paths") {
protoPaths = sflags.MustGetStringSlice(cmd, "proto-paths")
}

bytesEncoding := "hex"
if sflags.FlagDefined(cmd, "bytes-encoding") {
bytesEncoding = sflags.MustGetString(cmd, "bytes-encoding")
}

registry, err := fcproto.NewRegistry(chainFileDescriptor, protoPaths...)
if err != nil {
return nil, fmt.Errorf("new registry: %w", err)
}

if printer == "json" || printer == "jsonl" {
jsonPrinter, err := NewJSONOutputPrinter(bytesEncoding, printer == "jsonl", registry)
if err != nil {
return nil, fmt.Errorf("unable to create json encoder: %w", err)
}

return jsonPrinter, nil
}

if printer == "protojson" || printer == "protojsonl" {
indent := ""
if printer == "protojson" {
indent = " "
}

return NewProtoJSONOutputPrinter(indent, registry), nil
}

if printer == "text" {
// Supports the `transactions` flag defined on `firecore tools print` sub-command,
// we should move it to a proper `text` sub-option like `output-text-details` or something
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe just a different output mode:
short
full

or summary vs fulltext

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah let's discuss that part, I created a generic block printer using Golang protoreflect, we could maybe extend it!

// like that.
printTransactions := false
if sflags.FlagDefined(cmd, "transactions") {
printTransactions = sflags.MustGetBool(cmd, "transactions")
}

return NewTextOutputPrinter(bytesEncoding, registry, printTransactions), nil
}

return nil, fmt.Errorf("unsupported output printer %q", printer)
}

//go:generate go-enum -f=$GOFILE --marshal --names --nocase

// ENUM(Text, JSON, JSONL, ProtoJSON, ProtoJSONL)
type PrintOutputMode uint

type OutputPrinter interface {
PrintTo(message any, w io.Writer) error
}

func writeStringToWriter(w io.Writer, str string) error {
return writeBytesToWriter(w, unsafe.Slice(unsafe.StringData(str), len(str)))
}

func writeStringFToWriter(w io.Writer, format string, args ...any) error {
return writeStringToWriter(w, fmt.Sprintf(format, args...))
}

func writeBytesToWriter(w io.Writer, data []byte) error {
n, err := w.Write(data)
if err != nil {
return err
}

if n != len(data) {
return io.ErrShortWrite
}

return nil
}

func ptr[T any](v T) *T {
return &v
}

func deref[T any](v *T, orDefault T) T {
if v == nil {
return orDefault
}

return *v
}

func uint64PtrToString(v *uint64, orDefault string) string {
if v == nil {
return orDefault
}

return strconv.FormatUint(*v, 10)
}
Loading
Loading