update go ether + refactor trace call

.github/workflows/ci.yml

@@ -79,11 +79,11 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v3
         with:
-          go-version: "1.20.x"
+          go-version: "1.21.x"
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.51.1
+          version: v1.55.2
           args: --config=.golangci.yml
           skip-pkg-cache: true
           skip-build-cache: true
@@ -112,7 +112,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v3
         with:
-          go-version: "1.20.x"
+          go-version: "1.21.x"
       - name: Run test
         run: go test -race -v ./...
 

.github/workflows/release.yml

@@ -103,7 +103,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v3
         with:
-          go-version: "1.20.x"
+          go-version: "1.21.x"
 
       - name: Setup Git
         run: |

Dockerfile

@@ -1,5 +1,5 @@
 ## BUILDER
-FROM golang:1.20-bullseye as builder
+FROM golang:1.21-bullseye as builder
 
 WORKDIR /src
 

cmd/tradelogs/main.go

@@ -3,10 +3,15 @@ package main
 import (
 	"context"
 	"fmt"
-	"github.com/KyberNetwork/tradelogs/pkg/parser"
-	"github.com/KyberNetwork/tradelogs/pkg/parser/oneinch"
 	"log"
+	"net/http"
 	"os"
+	"time"
+
+	"github.com/KyberNetwork/tradelogs/pkg/parser"
+	"github.com/KyberNetwork/tradelogs/pkg/parser/oneinch"
+	"github.com/KyberNetwork/tradelogs/pkg/rpcnode"
+	"github.com/KyberNetwork/tradelogs/pkg/tracecall"
 
 	libapp "github.com/KyberNetwork/tradelogs/internal/app"
 	"github.com/KyberNetwork/tradelogs/internal/bigquery"
@@ -74,6 +79,18 @@ func run(c *cli.Context) error {
 		l.Errorw("Error while init listener service")
 		return err
 	}
+	httpClient := &http.Client{
+		Timeout: time.Second * 30,
+		Transport: &http.Transport{
+			IdleConnTimeout:       time.Second * 120,
+			ResponseHeaderTimeout: time.Second * 30,
+		},
+	}
+	rpcClient, err := rpcnode.NewClient(httpClient, c.String(libapp.RPCUrlFlagName))
+	if err != nil {
+		panic(err)
+	}
+	traceCalls := tracecall.NewCache(rpcClient)
 
 	w, err := worker.New(l, s, listener,
 		kyberswap.MustNewParser(),
@@ -85,12 +102,12 @@ func run(c *cli.Context) error {
 		native.MustNewParser(),
 		kyberswaprfq.MustNewParser(),
 		hashflowv3.MustNewParser(),
+		oneinch.MustNewParser(traceCalls),
 	)
 	if err != nil {
 		l.Errorw("Error while init worker")
 		return err
 	}
-
 	parserMap := map[string]parser.Parser{
 		"kyberswap":    kyberswap.MustNewParser(),
 		"zxotc":        zxotc.MustNewParser(),
@@ -101,22 +118,21 @@ func run(c *cli.Context) error {
 		"native":       native.MustNewParser(),
 		"kyberswaprfq": kyberswaprfq.MustNewParser(),
 		"hashflowv3":   hashflowv3.MustNewParser(),
-		"1inch":        oneinch.MustNewParser(c.String(libapp.RPCUrlFlagName)),
+		"1inch":        oneinch.MustNewParser(traceCalls),
 	}
 
 	backfillWorker, err := bigquery.NewWorker(libapp.BigqueryProjectIDFFromCli(c), s, parserMap)
 	if err != nil {
 		l.Errorw("Error while init backfillWorker")
-		return err
+	} else {
+		httpBackfill := backfill.New(c.String(libapp.HTTPBackfillServerFlag.Name), backfillWorker)
+		go func() {
+			if err := httpBackfill.Run(); err != nil {
+				panic(err)
+			}
+		}()
 	}
 
-	httpBackfill := backfill.New(c.String(libapp.HTTPBackfillServerFlag.Name), backfillWorker)
-	go func() {
-		if err := httpBackfill.Run(); err != nil {
-			panic(err)
-		}
-	}()
-
 	httpTradelogs := tradelogs.New(l, s, c.String(libapp.HTTPServerFlag.Name))
 	go func() {
 		if err := httpTradelogs.Run(); err != nil {

go.mod

@@ -1,113 +1,128 @@
 module github.com/KyberNetwork/tradelogs
 
-go 1.20
+go 1.21.1
 
 require (
-	cloud.google.com/go/bigquery v1.50.0
-	github.com/KyberNetwork/cclog v1.0.1
-	github.com/TheZeroSlave/zapsentry v1.12.0
-	github.com/ethereum/go-ethereum v1.11.5
-	github.com/getsentry/sentry-go v0.18.0
-	github.com/gin-contrib/pprof v1.3.0
+	cloud.google.com/go/bigquery v1.56.0
+	github.com/KyberNetwork/cclog v1.1.0
+	github.com/TheZeroSlave/zapsentry v1.20.2
+	github.com/ethereum/go-ethereum v1.13.14
+	github.com/gammazero/workerpool v1.1.3
+	github.com/getsentry/sentry-go v0.26.0
+	github.com/gin-contrib/pprof v1.4.0
 	github.com/gin-gonic/gin v1.9.1
-	github.com/golang-migrate/migrate/v4 v4.15.1
-	github.com/jmoiron/sqlx v1.3.4
+	github.com/golang-migrate/migrate/v4 v4.17.0
+	github.com/jmoiron/sqlx v1.3.5
 	github.com/joho/godotenv v1.4.0
-	github.com/lib/pq v1.10.4
+	github.com/lib/pq v1.10.9
 	github.com/pkg/errors v0.9.1
-	github.com/shopspring/decimal v1.2.0
-	github.com/stretchr/testify v1.8.3
-	github.com/urfave/cli v1.22.5
-	go.uber.org/zap v1.24.0
-	google.golang.org/api v0.114.0
+	github.com/shopspring/decimal v1.3.1
+	github.com/stretchr/testify v1.8.4
+	github.com/urfave/cli v1.22.14
+	go.uber.org/zap v1.26.0
+	google.golang.org/api v0.150.0
 )
 
 require (
-	cloud.google.com/go v0.110.0 // indirect
-	cloud.google.com/go/compute v1.19.0 // indirect
+	cloud.google.com/go v0.110.10 // indirect
+	cloud.google.com/go/compute v1.23.3 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	cloud.google.com/go/iam v0.13.0 // indirect
-	github.com/andybalholm/brotli v1.0.4 // indirect
-	github.com/apache/arrow/go/v11 v11.0.0 // indirect
+	cloud.google.com/go/iam v1.1.5 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/andybalholm/brotli v1.0.5 // indirect
+	github.com/apache/arrow/go/v12 v12.0.0 // indirect
 	github.com/apache/thrift v0.16.0 // indirect
-	github.com/bytedance/sonic v1.9.1 // indirect
+	github.com/bits-and-blooms/bitset v1.13.0 // indirect
+	github.com/bytedance/sonic v1.10.2 // indirect
+	github.com/cespare/cp v1.1.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
-	github.com/deckarep/golang-set/v2 v2.1.0 // indirect
+	github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
+	github.com/chenzhuoyu/iasm v0.9.1 // indirect
+	github.com/consensys/bavard v0.1.13 // indirect
+	github.com/consensys/gnark-crypto v0.12.1 // indirect
+	github.com/crate-crypto/go-kzg-4844 v0.7.0 // indirect
+	github.com/deckarep/golang-set/v2 v2.6.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/fsnotify/fsnotify v1.6.0 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
+	github.com/ethereum/c-kzg-4844 v0.4.0 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
 	github.com/gammazero/deque v0.2.0 // indirect
-	github.com/gammazero/workerpool v1.1.3 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
-	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/golang/snappy v0.0.4 // indirect
+	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
 	github.com/google/flatbuffers v2.0.8+incompatible // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
-	github.com/googleapis/gax-go/v2 v2.7.1 // indirect
-	github.com/holiman/uint256 v1.2.0 // indirect
+	github.com/google/s2a-go v0.1.7 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
+	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
+	github.com/holiman/uint256 v1.2.4 // indirect
 	github.com/klauspost/asmfmt v1.3.2 // indirect
-	github.com/klauspost/compress v1.15.15 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
+	github.com/klauspost/compress v1.16.0 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
 	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
 	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
 	github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
 	github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
-	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
-	github.com/pierrec/lz4/v4 v4.1.15 // indirect
+	github.com/mitchellh/mapstructure v1.4.2 // indirect
+	github.com/mmcloughlin/addchain v0.4.0 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.1 // indirect
+	github.com/pierrec/lz4/v3 v3.3.5 // indirect
+	github.com/pierrec/lz4/v4 v4.1.16 // indirect
+	github.com/supranational/blst v0.3.11 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
-	github.com/yusufpapurcu/wmi v1.2.2 // indirect
+	github.com/yusufpapurcu/wmi v1.2.3 // indirect
 	github.com/zeebo/xxh3 v1.0.2 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	golang.org/x/arch v0.3.0 // indirect
-	golang.org/x/mod v0.8.0 // indirect
-	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/oauth2 v0.6.0 // indirect
-	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/text v0.9.0 // indirect
-	golang.org/x/tools v0.6.0 // indirect
+	golang.org/x/arch v0.7.0 // indirect
+	golang.org/x/exp v0.0.0-20240119083558-1b970713d09a // indirect
+	golang.org/x/mod v0.14.0 // indirect
+	golang.org/x/net v0.20.0 // indirect
+	golang.org/x/oauth2 v0.14.0 // indirect
+	golang.org/x/sync v0.6.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.17.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea // indirect
-	google.golang.org/grpc v1.54.0 // indirect
+	google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect
+	google.golang.org/grpc v1.59.0 // indirect
+	rsc.io/tmplfunc v0.0.3 // indirect
 )
 
 require (
-	github.com/KyberNetwork/evmlistener v0.3.5
-	github.com/Masterminds/squirrel v1.5.3
+	github.com/KyberNetwork/evmlistener v0.4.7
+	github.com/Masterminds/squirrel v1.5.4
 	github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
-	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/go-ole/go-ole v1.3.0 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
-	github.com/go-playground/validator/v10 v10.14.0 // indirect
+	github.com/go-playground/validator/v10 v10.17.0 // indirect
 	github.com/go-redis/redis/v8 v8.11.5
-	github.com/go-stack/stack v1.8.1 // indirect
-	github.com/google/uuid v1.3.0 // indirect
-	github.com/gorilla/websocket v1.5.0 // indirect
-	github.com/hashicorp/errwrap v1.0.0 // indirect
-	github.com/hashicorp/go-multierror v1.1.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/gorilla/websocket v1.5.1 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/leodido/go-urn v1.2.4 // indirect
-	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
-	github.com/tklauser/go-sysconf v0.3.5 // indirect
-	github.com/tklauser/numcpus v0.2.2 // indirect
-	github.com/ugorji/go/codec v1.2.11 // indirect
-	go.uber.org/atomic v1.10.0 // indirect
-	go.uber.org/multierr v1.8.0 // indirect
-	golang.org/x/crypto v0.9.0 // indirect
-	golang.org/x/sys v0.8.0 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
-	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
+	github.com/tklauser/go-sysconf v0.3.13 // indirect
+	github.com/tklauser/numcpus v0.7.0 // indirect
+	github.com/ugorji/go/codec v1.2.12 // indirect
+	go.uber.org/atomic v1.11.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/crypto v0.18.0 // indirect
+	golang.org/x/sys v0.16.0 // indirect
+	google.golang.org/protobuf v1.32.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )