Skip to content

Commit

Permalink
Replace logrus with zerolog for structured logging (#23)
Browse files Browse the repository at this point in the history
### TL;DR

Replaced logrus with zerolog for improved logging and implemented custom logger initialization.

### What changed?

- Replaced logrus with zerolog across the codebase
- Added a new `log` package with `InitLogger` and `NewLogger` functions
- Updated logging calls to use zerolog's structured logging
- Implemented log level configuration via environment variable
- Added option for prettified console output

### How to test?

1. Set the `LOG_LEVEL` environment variable to control log verbosity (e.g., "debug", "info", "warn")
2. Set `LOG_PRETTIFY=true` for human-readable console output
3. Run the application and verify that logs are correctly formatted and filtered based on the set log level

### Why make this change?

- Zerolog offers better performance and more flexible structured logging
- Custom logger initialization allows for consistent logging across the application
- Environment variable configuration enables easy log level adjustment without code changes
- Prettified output option improves readability during development and debugging
  • Loading branch information
iuwqyir authored Sep 20, 2024
1 parent ea44e73 commit 7e11a98
Show file tree
Hide file tree
Showing 12 changed files with 109 additions and 70 deletions.
12 changes: 6 additions & 6 deletions cmd/api/main.go
Original file line number Diff line number Diff line change
@@ -1,28 +1,28 @@
package main

import (
"fmt"
"net/http"

"github.com/go-chi/chi/v5"
"github.com/joho/godotenv"
log "github.com/sirupsen/logrus"
"github.com/rs/zerolog/log"
"github.com/thirdweb-dev/indexer/internal/handlers"
customLogger "github.com/thirdweb-dev/indexer/internal/log"
)

func main() {
err := godotenv.Load()
if err != nil {
log.Errorf("error loading .env file: %w", err)
log.Error().Err(err).Msg("error loading .env file")
}
customLogger.InitLogger()

log.SetReportCaller(true)
var r *chi.Mux = chi.NewRouter()
handlers.Handler(r)

fmt.Println("Starting Server on port 3000")
log.Info().Msg("Starting Server on port 3000")
err = http.ListenAndServe("localhost:3000", r)
if err != nil {
log.Error(err)
log.Error().Err(err).Msg("Error starting server")
}
}
14 changes: 7 additions & 7 deletions cmd/indexer/main.go
Original file line number Diff line number Diff line change
@@ -1,29 +1,29 @@
package main

import (
"log"
"os"

"github.com/joho/godotenv"
"github.com/rs/zerolog/log"
"github.com/thirdweb-dev/indexer/internal/common"
customLogger "github.com/thirdweb-dev/indexer/internal/log"
"github.com/thirdweb-dev/indexer/internal/orchestrator"
)

func main() {
err := godotenv.Load()
if err != nil {
log.Fatalf("error loading .env file: %v", err)
log.Fatal().Err(err).Msg("error loading .env file")
}
customLogger.InitLogger()

log.SetOutput(os.Stdout)
log.Info().Msg("Starting indexer")
rpc, err := common.InitializeRPC()
if err != nil {
log.Fatalf("Failed to initialize RPC: %v", err)
log.Fatal().Err(err).Msg("Failed to initialize RPC")
}

orchestrator, err := orchestrator.NewOrchestrator(*rpc)
if err != nil {
log.Fatalf("Failed to create orchestrator: %v", err)
log.Fatal().Err(err).Msg("Failed to create orchestrator")
}

orchestrator.Start()
Expand Down
8 changes: 5 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,11 @@ require (
github.com/ethereum/go-ethereum v1.14.8
github.com/go-chi/chi v1.5.4
github.com/go-chi/chi/v5 v5.1.0
github.com/google/uuid v1.6.0
github.com/gorilla/schema v1.2.0
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/joho/godotenv v1.5.1
github.com/sirupsen/logrus v1.9.3
github.com/rs/zerolog v1.33.0
)

require (
Expand All @@ -28,10 +29,11 @@ require (
github.com/go-faster/city v1.0.1 // indirect
github.com/go-faster/errors v0.7.1 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
github.com/holiman/uint256 v1.3.1 // indirect
github.com/klauspost/compress v1.17.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mmcloughlin/addchain v0.4.0 // indirect
github.com/paulmach/orb v0.11.1 // indirect
github.com/pierrec/lz4/v4 v4.1.21 // indirect
Expand All @@ -48,7 +50,7 @@ require (
golang.org/x/crypto v0.26.0 // indirect
golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/sys v0.23.0 // indirect
golang.org/x/sys v0.25.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
rsc.io/tmplfunc v0.0.3 // indirect
)
18 changes: 12 additions & 6 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/Yj
github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI=
github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M=
github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c h1:uQYC5Z1mdLRPrZhHjHxufI8+2UG/i25QG92j0Er9p6I=
Expand Down Expand Up @@ -74,6 +75,7 @@ github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7F
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
Expand Down Expand Up @@ -131,6 +133,8 @@ github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
Expand Down Expand Up @@ -170,6 +174,9 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
Expand All @@ -178,13 +185,10 @@ github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKl
github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA=
github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9+mHxBEeo3Hbg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4=
Expand Down Expand Up @@ -243,12 +247,14 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
Expand Down
10 changes: 5 additions & 5 deletions internal/common/rpc.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,13 @@ package common
import (
"context"
"fmt"
"log"
"math/big"
"os"
"strings"

"github.com/ethereum/go-ethereum/ethclient"
"github.com/ethereum/go-ethereum/rpc"
"github.com/rs/zerolog/log"
)

type RPC struct {
Expand Down Expand Up @@ -57,21 +57,21 @@ func (rpc *RPC) checkSupportedMethods() error {
if err != nil {
return fmt.Errorf("eth_getBlockByNumber method not supported: %v", err)
}
fmt.Println("eth_getBlockByNumber method supported")
log.Debug().Msg("eth_getBlockByNumber method supported")

var getLogsResult interface{}
logsErr := rpc.RPCClient.Call(&getLogsResult, "eth_getLogs", map[string]string{"fromBlock": "0x0", "toBlock": "0x0"})
if logsErr != nil {
return fmt.Errorf("eth_getBlockByNumber method not supported: %v", logsErr)
}
fmt.Println("eth_getLogs method supported")
log.Debug().Msg("eth_getLogs method supported")

var traceBlockResult interface{}
if traceBlockErr := rpc.RPCClient.Call(&traceBlockResult, "trace_block", "latest"); traceBlockErr != nil {
log.Printf("Optional method trace_block not supported: %v", traceBlockErr)
log.Warn().Err(traceBlockErr).Msg("Optional method trace_block not supported")
}
rpc.SupportsTraceBlock = traceBlockResult != nil
fmt.Printf("trace_block method supported: %v\n", rpc.SupportsTraceBlock)
log.Debug().Msgf("trace_block method supported: %v", rpc.SupportsTraceBlock)
return nil
}

Expand Down
32 changes: 16 additions & 16 deletions internal/handlers/get_blocks.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
"net/http"

"github.com/gorilla/schema"
log "github.com/sirupsen/logrus"
"github.com/rs/zerolog/log"
"github.com/thirdweb-dev/indexer/api"
"github.com/thirdweb-dev/indexer/internal/storage"
)
Expand All @@ -20,39 +20,39 @@ func GetBlocks(w http.ResponseWriter, r *http.Request) {
err = decoder.Decode(&params, r.URL.Query())

if err != nil {
log.Error(err)
log.Error().Err(err).Msg("Error decoding query params")
api.InternalErrorHandler(w)
return
}

conn, err := storage.ConnectDB()
if err != nil {
log.Error(err)
log.Error().Err(err).Msg("Error connecting to DB")
api.InternalErrorHandler(w)
return
}

row := conn.QueryRow(context.Background(), "SELECT block_number FROM chainsaw.blocks LIMIT 1")
var blockNumber uint64
err = row.Scan(&blockNumber)
if err != nil {
log.Error(err)
api.RequestErrorHandler(w, err)
return
}

var response = api.QueryResponse{
Result: fmt.Sprintf("%d", blockNumber),
Code: http.StatusOK,
}
err = row.Scan(&blockNumber)
if err != nil {
log.Error().Err(err).Msg("Error getting block number")
api.RequestErrorHandler(w, err)
return
}

var response = api.QueryResponse{
Result: fmt.Sprintf("%d", blockNumber),
Code: http.StatusOK,
}

w.Header().Set("Content-Type", "application/json")
err = json.NewEncoder(w).Encode(response)
if err != nil {
log.Error(err)
log.Error().Err(err).Msg("Error encoding response")
api.InternalErrorHandler(w)
return
}

defer conn.Close()
}
}
32 changes: 32 additions & 0 deletions internal/log/logger.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
package log

import (
"os"

"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/rs/zerolog/pkgerrors"
)

func InitLogger() {
// overrides zerolog global logger
log.Logger = NewLogger("default")
}

func NewLogger(name string) zerolog.Logger {
zerolog.ErrorStackMarshaler = pkgerrors.MarshalStack

level := zerolog.WarnLevel
if lvl, err := zerolog.ParseLevel(os.Getenv("LOG_LEVEL")); err == nil && lvl != zerolog.NoLevel {
level = lvl
}
zerolog.SetGlobalLevel(level)

prettify := os.Getenv("LOG_PRETTIFY") == "true"
logger := zerolog.New(os.Stderr).With().Timestamp().Str("component", name).Logger()
logger = logger.With().Caller().Logger()
if prettify {
logger = logger.Output(zerolog.ConsoleWriter{Out: os.Stderr})
}
return logger
}
10 changes: 5 additions & 5 deletions internal/orchestrator/commiter.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@ package orchestrator

import (
"fmt"
"log"
"os"
"sort"
"strconv"
"sync"
"time"

"github.com/rs/zerolog/log"
"github.com/thirdweb-dev/indexer/internal/common"
"github.com/thirdweb-dev/indexer/internal/storage"
)
Expand Down Expand Up @@ -44,18 +44,18 @@ func (c *Commiter) Start() {

go func() {
for t := range ticker.C {
fmt.Println("Commiter running at", t)
log.Debug().Msgf("Commiter running at %s", t)
blocksToCommit, err := c.getSequentialBlocksToCommit()
if err != nil {
log.Printf("Error getting blocks to commit: %v", err)
log.Error().Err(err).Msg("Error getting blocks to commit")
continue
}
if len(blocksToCommit) == 0 {
log.Println("No blocks to commit")
log.Debug().Msg("No blocks to commit")
continue
}
if err := c.commit(blocksToCommit); err != nil {
log.Printf("Error committing blocks: %v", err)
log.Error().Err(err).Msg("Error committing blocks")
}
}
}()
Expand Down
11 changes: 5 additions & 6 deletions internal/orchestrator/failure_recoverer.go
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
package orchestrator

import (
"fmt"
"log"
"os"
"strconv"
"time"

"github.com/rs/zerolog/log"
"github.com/thirdweb-dev/indexer/internal/common"
"github.com/thirdweb-dev/indexer/internal/storage"
"github.com/thirdweb-dev/indexer/internal/worker"
Expand Down Expand Up @@ -45,15 +44,15 @@ func (fr *FailureRecoverer) Start() {

go func() {
for t := range ticker.C {
fmt.Println("Failure Recovery running at", t)
log.Debug().Msgf("Failure Recovery running at %s", t)

blockFailures, err := fr.storage.OrchestratorStorage.GetBlockFailures(fr.failuresPerPoll)
if err != nil {
log.Printf("Failed to get block failures: %s", err)
log.Error().Err(err).Msg("Failed to get block failures")
continue
}

log.Printf("Triggering workers for %d block failures", len(blockFailures))
log.Debug().Msgf("Triggering workers for %d block failures", len(blockFailures))

blocksToTrigger := make([]uint64, 0, len(blockFailures))
for _, blockFailure := range blockFailures {
Expand All @@ -73,7 +72,7 @@ func (fr *FailureRecoverer) Start() {
func (fr *FailureRecoverer) handleBlockResults(blockFailures []common.BlockFailure, results []worker.BlockResult) {
err := fr.storage.OrchestratorStorage.DeleteBlockFailures(blockFailures)
if err != nil {
log.Printf("Error deleting block failures: %v", err)
log.Error().Err(err).Msg("Error deleting block failures")
return
}
blockFailureMap := make(map[uint64]common.BlockFailure)
Expand Down
Loading

0 comments on commit 7e11a98

Please sign in to comment.