From e88a64286d4455cdbd8eb9827b4362c96a0b42bc Mon Sep 17 00:00:00 2001 From: Sean McGary Date: Thu, 12 Sep 2024 13:48:26 -0500 Subject: [PATCH] # This is a combination of 25 commits. # This is the 1st commit message: Calculate rewards Add state change table for AVS operators to make windowing easier Convert avs operator windows query to sqlite Queries for snapshots Testing OperatorAVSRegistrationWindows Window query passing Some cleanup Expand test dataset to be more than just up to 2024-09-01 Restaked strategy snapshots Test tweaks Generate snapshots rather than windows Add custom sum_big function Adding tests for operator share snapshots Change operatorShares to be a sparse table Change StakerShares to a sparse table Add snapshot generation for operator shares Calculate rewards remove git attributes # This is the commit message #2: Adding query stubs # This is the commit message #3: Adding more custom functions for math # This is the commit message #4: Copy over queries for gold table generation # This is the commit message #5: Use custom sql functions # This is the commit message #6: Use file-based db for testing rewards # This is the commit message #7: Staker amounts for nile and amazon forks # This is the commit message #8: Calculation testing # This is the commit message #9: Floating point math is the absolute worst # This is the commit message #10: Fix compliation # This is the commit message #11: More number tests # This is the commit message #12: Trying to get sqlite lib working # This is the commit message #13: This compiles and runs # This is the commit message #14: Also compiles # This is the commit message #15: Compiles and runs with added python stuff # This is the commit message #16: Delete compiled stuff # This is the commit message #17: Python seems to be working # This is the commit message #18: Working pre_nile_tokens_per_day # This is the commit message #19: Works with go example # This is the commit message #20: Port amazonStakerTokenRewards to native C # This is the commit message #21: Port nile_staker_token_rewards to native C # This is the commit message #22: Port amazon_operator_token_rewards and nile_operator_token_rewards to native C # This is the commit message #23: Rename yolo to calculations # This is the commit message #24: Add pure functions for easier testing # This is the commit message #25: Token calcuations test works --- .gitignore | 5 + .testdataVersion | 1 + Makefile | 56 +- cmd/debugger/main.go | 136 +++ cmd/run.go | 2 +- cmd/sidecar/main.go | 127 +++ cmd/sqlitedebug/db.py | 7 + cmd/sqlitedebug/main.go | 33 + go.mod | 39 +- go.sum | 96 ++- internal/config/config.go | 29 + .../sqliteContractStore_test.go | 2 +- .../eigenState/avsOperators/avsOperators.go | 48 ++ .../avsOperators/avsOperators_test.go | 46 +- internal/eigenState/eigenstate_test.go | 2 +- .../operatorShares/operatorShares.go | 109 +-- .../operatorShares/operatorShares_test.go | 3 +- .../rewardSubmissions/rewardSubmissions.go | 17 +- .../rewardSubmissions_test.go | 2 +- .../stakerDelegations/stakerDelegations.go | 46 + .../stakerDelegations_test.go | 3 +- .../eigenState/stakerShares/stakerShares.go | 108 +-- .../stakerShares/stakerShares_test.go | 5 +- .../stateManager/stateManager_test.go | 2 +- .../submittedDistributionRoots_test.go | 2 +- internal/indexer/restakedStrategies_test.go | 2 +- .../pipeline/pipeline_integration_test.go | 2 +- internal/python/lossyTokens.py | 597 +++++++++++++ .../202409161057_avsOperatorDeltas/up.go | 36 + .../202409181340_stakerDelegationDelta/up.go | 29 + .../202409191425_addRewardTypeColumn/up.go | 60 ++ internal/sqlite/migrations/migrator.go | 6 + internal/sqlite/sqlite.go | 124 ++- internal/sqlite/sqlite_test.go | 72 +- internal/storage/sqlite/sqlite_test.go | 2 +- .../tests/testdata/combinedRewards/README.md | 26 + .../README.md | 31 + .../generateExpectedResults.sql | 67 ++ .../operatorRestakedStrategies/README.md | 18 + .../generateExpectedResults.sql | 112 +++ .../testdata/operatorShareSnapshots/README.md | 17 + .../generateExpectedResults.sql | 47 + .../stakerDelegationSnapshots/README.md | 22 + .../generateExpectedResults.sql | 47 + .../testdata/stakerShareSnapshots/README.md | 15 + .../generateExpectedResults.sql | 48 ++ internal/tests/utils.go | 169 +++- internal/types/numbers/numbers.go | 104 ++- internal/types/numbers/numbersData_test.go | 559 ++++++++++++ internal/types/numbers/numbers_test.go | 813 +++++++++++++++++- internal/types/numbers/tokenCalculations.go | 127 +++ pkg/rewards/1_goldActiveRewards.go | 170 ++++ pkg/rewards/2_goldStakerRewardAmounts.go | 180 ++++ pkg/rewards/3_goldOperatorRewardAmounts.go | 65 ++ pkg/rewards/4_goldRewardsForAll.go | 101 +++ pkg/rewards/5_goldRfaeStakers.go | 141 +++ pkg/rewards/6_goldRfaeOperators.go | 65 ++ pkg/rewards/7_goldStaging.go | 107 +++ pkg/rewards/8_goldFinal.go | 39 + pkg/rewards/combinedRewards.go | 100 +++ pkg/rewards/combinedRewards_test.go | 114 +++ .../operatorAvsRegistrationSnaphots.go | 172 ++++ .../operatorAvsRegistrationSnaphots_test.go | 159 ++++ pkg/rewards/operatorAvsStrategySnapshots.go | 231 +++++ .../operatorAvsStrategySnapshots_test.go | 148 ++++ pkg/rewards/operatorShareSnapshots.go | 132 +++ pkg/rewards/operatorShareSnapshots_test.go | 142 +++ pkg/rewards/rewards.go | 173 ++++ pkg/rewards/rewards_test.go | 191 ++++ pkg/rewards/stakerDelegationSnapshots.go | 124 +++ pkg/rewards/stakerDelegationSnapshots_test.go | 143 +++ pkg/rewards/stakerShareSnapshots.go | 127 +++ pkg/rewards/stakerShareSnapshots_test.go | 140 +++ pkg/utils/utils.go | 9 + scripts/downloadTestData.sh | 13 + scripts/updateTestData.sh | 38 + sql/schema.sql | 737 ---------------- sqlite-extensions/README.md | 5 + sqlite-extensions/calculations.c | 310 +++++++ sqlite-extensions/calculations.h | 28 + sqlite-extensions/calculations.py | 74 ++ 81 files changed, 7254 insertions(+), 1002 deletions(-) create mode 100644 .testdataVersion create mode 100644 cmd/debugger/main.go create mode 100644 cmd/sidecar/main.go create mode 100644 cmd/sqlitedebug/db.py create mode 100644 cmd/sqlitedebug/main.go create mode 100644 internal/python/lossyTokens.py create mode 100644 internal/sqlite/migrations/202409161057_avsOperatorDeltas/up.go create mode 100644 internal/sqlite/migrations/202409181340_stakerDelegationDelta/up.go create mode 100644 internal/sqlite/migrations/202409191425_addRewardTypeColumn/up.go create mode 100644 internal/tests/testdata/combinedRewards/README.md create mode 100644 internal/tests/testdata/operatorAvsRegistrationSnapshots/README.md create mode 100644 internal/tests/testdata/operatorAvsRegistrationSnapshots/generateExpectedResults.sql create mode 100644 internal/tests/testdata/operatorRestakedStrategies/README.md create mode 100644 internal/tests/testdata/operatorRestakedStrategies/generateExpectedResults.sql create mode 100644 internal/tests/testdata/operatorShareSnapshots/README.md create mode 100644 internal/tests/testdata/operatorShareSnapshots/generateExpectedResults.sql create mode 100644 internal/tests/testdata/stakerDelegationSnapshots/README.md create mode 100644 internal/tests/testdata/stakerDelegationSnapshots/generateExpectedResults.sql create mode 100644 internal/tests/testdata/stakerShareSnapshots/README.md create mode 100644 internal/tests/testdata/stakerShareSnapshots/generateExpectedResults.sql create mode 100644 internal/types/numbers/numbersData_test.go create mode 100644 internal/types/numbers/tokenCalculations.go create mode 100644 pkg/rewards/1_goldActiveRewards.go create mode 100644 pkg/rewards/2_goldStakerRewardAmounts.go create mode 100644 pkg/rewards/3_goldOperatorRewardAmounts.go create mode 100644 pkg/rewards/4_goldRewardsForAll.go create mode 100644 pkg/rewards/5_goldRfaeStakers.go create mode 100644 pkg/rewards/6_goldRfaeOperators.go create mode 100644 pkg/rewards/7_goldStaging.go create mode 100644 pkg/rewards/8_goldFinal.go create mode 100644 pkg/rewards/combinedRewards.go create mode 100644 pkg/rewards/combinedRewards_test.go create mode 100644 pkg/rewards/operatorAvsRegistrationSnaphots.go create mode 100644 pkg/rewards/operatorAvsRegistrationSnaphots_test.go create mode 100644 pkg/rewards/operatorAvsStrategySnapshots.go create mode 100644 pkg/rewards/operatorAvsStrategySnapshots_test.go create mode 100644 pkg/rewards/operatorShareSnapshots.go create mode 100644 pkg/rewards/operatorShareSnapshots_test.go create mode 100644 pkg/rewards/rewards.go create mode 100644 pkg/rewards/rewards_test.go create mode 100644 pkg/rewards/stakerDelegationSnapshots.go create mode 100644 pkg/rewards/stakerDelegationSnapshots_test.go create mode 100644 pkg/rewards/stakerShareSnapshots.go create mode 100644 pkg/rewards/stakerShareSnapshots_test.go create mode 100755 scripts/downloadTestData.sh create mode 100755 scripts/updateTestData.sh delete mode 100644 sql/schema.sql create mode 100644 sqlite-extensions/README.md create mode 100644 sqlite-extensions/calculations.c create mode 100644 sqlite-extensions/calculations.h create mode 100644 sqlite-extensions/calculations.py diff --git a/.gitignore b/.gitignore index 39ced4dc..3ec43ad0 100644 --- a/.gitignore +++ b/.gitignore @@ -14,5 +14,10 @@ go-sidecar /internal/tests/testdata/**/*.json /internal/tests/testdata/*.sql !/internal/tests/testdata/**/generateExpectedResults.sql +!sqlite-extensions /*.tar /sidecar-data +__pycache__ +test-db +*.dylib +*.dylib.dSYM diff --git a/.testdataVersion b/.testdataVersion new file mode 100644 index 00000000..9b70f0f2 --- /dev/null +++ b/.testdataVersion @@ -0,0 +1 @@ +f8689176dbe66b2d7ce03665190298b9c9f7c8e9 \ No newline at end of file diff --git a/Makefile b/Makefile index d05d123e..2f9147a9 100644 --- a/Makefile +++ b/Makefile @@ -1,24 +1,27 @@ .PHONY: deps proto +args=CGO_ENABLED=1 +GO=$(shell which go) + deps/dev: - go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.61.0 - go install honnef.co/go/tools/cmd/staticcheck@latest - go install github.com/google/yamlfmt/cmd/yamlfmt@latest + ${GO} install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.61.0 + ${GO} install honnef.co/go/tools/cmd/staticcheck@latest + ${GO} install github.com/google/yamlfmt/cmd/yamlfmt@latest deps/go: - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 - go get \ + ${GO} install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28 + ${GO} install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2 + ${GO} get \ github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway \ github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2 \ google.golang.org/protobuf/cmd/protoc-gen-go \ google.golang.org/grpc/cmd/protoc-gen-go-grpc - go install \ + ${GO} install \ github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-grpc-gateway \ github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2 \ google.golang.org/protobuf/cmd/protoc-gen-go \ google.golang.org/grpc/cmd/protoc-gen-go-grpc - go mod tidy + ${GO} mod tidy curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.61.0 @@ -42,7 +45,7 @@ clean: .PHONY: build/cmd/sidecar build/cmd/sidecar: - CGO_ENABLED=1 go build -o bin/sidecar main.go + ${args} ${GO} build -o bin/sidecar main.go .PHONY: build build: build/cmd/sidecar @@ -88,3 +91,38 @@ staticcheck: .PHONY: ci-test ci-test: test + +test-rewards: + TEST_REWARDS=true TESTING=true ${GO} test ./pkg/rewards -v -p 1 + +# ----------------------------------------------------------------------------- +# SQLite extension build steps +# ----------------------------------------------------------------------------- +CC = gcc -g -fPIC -shared + +PYTHON_CONFIG = python3-config +PYTHON_VERSION = $(shell python3 -c "import sys; print('{}.{}'.format(sys.version_info.major, sys.version_info.minor))") +PYTHON_LIBDIR := $(shell $(PYTHON_CONFIG) --prefix)/lib + +# Base flags +CFLAGS = +LDFLAGS = + +INCLUDE_DIRS = +CFLAGS += $(foreach dir,$(INCLUDE_DIRS),-I$(dir)) + +# Python flags +PYTHON_CFLAGS := $(shell $(PYTHON_CONFIG) --includes) +PYTHON_LDFLAGS := $(shell $(PYTHON_CONFIG) --ldflags) + +SQLITE_DIR = /opt/homebrew/opt/sqlite +CFLAGS += -I$(SQLITE_DIR)/include +LDFLAGS += -L$(SQLITE_DIR)/lib -lsqlite3 + +CFLAGS += $(PYTHON_CFLAGS) +LDFLAGS += $(PYTHON_LDFLAGS) -L$(PYTHON_LIBDIR) -lpython$(PYTHON_VERSION) + + +.PHONY: sqlite-extensions +sqlite-extensions: + $(CC) $(CFLAGS) -o sqlite-extensions/libcalculations.dylib sqlite-extensions/calculations.c $(LDFLAGS) diff --git a/cmd/debugger/main.go b/cmd/debugger/main.go new file mode 100644 index 00000000..6166c55e --- /dev/null +++ b/cmd/debugger/main.go @@ -0,0 +1,136 @@ +package main + +import ( + "context" + "fmt" + "log" + + "github.com/Layr-Labs/go-sidecar/internal/clients/ethereum" + "github.com/Layr-Labs/go-sidecar/internal/clients/etherscan" + "github.com/Layr-Labs/go-sidecar/internal/config" + "github.com/Layr-Labs/go-sidecar/internal/contractCaller" + "github.com/Layr-Labs/go-sidecar/internal/contractManager" + "github.com/Layr-Labs/go-sidecar/internal/contractStore/sqliteContractStore" + "github.com/Layr-Labs/go-sidecar/internal/eigenState/avsOperators" + "github.com/Layr-Labs/go-sidecar/internal/eigenState/operatorShares" + "github.com/Layr-Labs/go-sidecar/internal/eigenState/stakerDelegations" + "github.com/Layr-Labs/go-sidecar/internal/eigenState/stakerShares" + "github.com/Layr-Labs/go-sidecar/internal/eigenState/stateManager" + "github.com/Layr-Labs/go-sidecar/internal/fetcher" + "github.com/Layr-Labs/go-sidecar/internal/indexer" + "github.com/Layr-Labs/go-sidecar/internal/logger" + "github.com/Layr-Labs/go-sidecar/internal/metrics" + "github.com/Layr-Labs/go-sidecar/internal/pipeline" + "github.com/Layr-Labs/go-sidecar/internal/sidecar" + "github.com/Layr-Labs/go-sidecar/internal/sqlite" + "github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations" + sqliteBlockStore "github.com/Layr-Labs/go-sidecar/internal/storage/sqlite" + "go.uber.org/zap" +) + +func main() { + ctx := context.Background() + cfg := config.NewConfig() + + l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) + + sdc, err := metrics.InitStatsdClient(cfg.StatsdUrl) + if err != nil { + l.Sugar().Fatal("Failed to setup statsd client", zap.Error(err)) + } + + etherscanClient := etherscan.NewEtherscanClient(cfg, l) + client := ethereum.NewClient(cfg.EthereumRpcConfig.BaseUrl, l) + + db := sqlite.NewSqlite(cfg.GetSqlitePath(), l) + + grm, err := sqlite.NewGormSqliteFromSqlite(db) + if err != nil { + l.Error("Failed to create gorm instance", zap.Error(err)) + panic(err) + } + + migrator := migrations.NewSqliteMigrator(grm, l) + if err = migrator.MigrateAll(); err != nil { + log.Fatalf("Failed to migrate: %v", err) + } + + contractStore := sqliteContractStore.NewSqliteContractStore(grm, l, cfg) + if err := contractStore.InitializeCoreContracts(); err != nil { + log.Fatalf("Failed to initialize core contracts: %v", err) + } + + cm := contractManager.NewContractManager(contractStore, etherscanClient, client, sdc, l) + + mds := sqliteBlockStore.NewSqliteBlockStore(grm, l, cfg) + if err != nil { + log.Fatalln(err) + } + + sm := stateManager.NewEigenStateManager(l, grm) + + if _, err := avsOperators.NewAvsOperators(sm, grm, l, cfg); err != nil { + l.Sugar().Fatalw("Failed to create AvsOperatorsModel", zap.Error(err)) + } + if _, err := operatorShares.NewOperatorSharesModel(sm, grm, l, cfg); err != nil { + l.Sugar().Fatalw("Failed to create OperatorSharesModel", zap.Error(err)) + } + if _, err := stakerDelegations.NewStakerDelegationsModel(sm, grm, l, cfg); err != nil { + l.Sugar().Fatalw("Failed to create StakerDelegationsModel", zap.Error(err)) + } + if _, err := stakerShares.NewStakerSharesModel(sm, grm, l, cfg); err != nil { + l.Sugar().Fatalw("Failed to create StakerSharesModel", zap.Error(err)) + } + + fetchr := fetcher.NewFetcher(client, cfg, l) + + cc := contractCaller.NewContractCaller(client, l) + + idxr := indexer.NewIndexer(mds, contractStore, etherscanClient, cm, client, fetchr, cc, l, cfg) + + p := pipeline.NewPipeline(fetchr, idxr, mds, sm, l) + + // Create new sidecar instance + sidecar := sidecar.NewSidecar(&sidecar.SidecarConfig{ + GenesisBlockNumber: cfg.GetGenesisBlockNumber(), + }, cfg, mds, p, sm, l, client) + + // RPC channel to notify the RPC server to shutdown gracefully + rpcChannel := make(chan bool) + err = sidecar.WithRpcServer(ctx, mds, sm, rpcChannel) + if err != nil { + l.Sugar().Fatalw("Failed to start RPC server", zap.Error(err)) + } + + block, err := fetchr.FetchBlock(ctx, 1215893) + if err != nil { + l.Sugar().Fatalw("Failed to fetch block", zap.Error(err)) + } + + transactionHash := "0xf6775c38af1d2802bcbc2b7c8959c0d5b48c63a14bfeda0261ba29d76c68c423" + transaction := ðereum.EthereumTransaction{} + + for _, tx := range block.Block.Transactions { + if tx.Hash.Value() == transactionHash { + transaction = tx + break + } + } + + logIndex := 4 + receipt := block.TxReceipts[transaction.Hash.Value()] + var interestingLog *ethereum.EthereumEventLog + + for _, log := range receipt.Logs { + if log.LogIndex.Value() == uint64(logIndex) { + fmt.Printf("Log: %+v\n", log) + interestingLog = log + } + } + + decodedLog, err := idxr.DecodeLogWithAbi(nil, receipt, interestingLog) + if err != nil { + l.Sugar().Fatalw("Failed to decode log", zap.Error(err)) + } + l.Sugar().Infof("Decoded log: %+v", decodedLog) +} diff --git a/cmd/run.go b/cmd/run.go index 18c38521..6671bb0e 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -59,7 +59,7 @@ var runCmd = &cobra.Command{ } } - db := sqlite.NewSqlite(cfg.GetSqlitePath()) + db := sqlite.NewSqlite(cfg.GetSqlitePath(), l) grm, err := sqlite.NewGormSqliteFromSqlite(db) if err != nil { diff --git a/cmd/sidecar/main.go b/cmd/sidecar/main.go new file mode 100644 index 00000000..7b5366b7 --- /dev/null +++ b/cmd/sidecar/main.go @@ -0,0 +1,127 @@ +package main + +import ( + "context" + "log" + "time" + + "github.com/Layr-Labs/go-sidecar/internal/clients/ethereum" + "github.com/Layr-Labs/go-sidecar/internal/clients/etherscan" + "github.com/Layr-Labs/go-sidecar/internal/config" + "github.com/Layr-Labs/go-sidecar/internal/contractCaller" + "github.com/Layr-Labs/go-sidecar/internal/contractManager" + "github.com/Layr-Labs/go-sidecar/internal/contractStore/sqliteContractStore" + "github.com/Layr-Labs/go-sidecar/internal/eigenState/avsOperators" + "github.com/Layr-Labs/go-sidecar/internal/eigenState/operatorShares" + "github.com/Layr-Labs/go-sidecar/internal/eigenState/rewardSubmissions" + "github.com/Layr-Labs/go-sidecar/internal/eigenState/stakerDelegations" + "github.com/Layr-Labs/go-sidecar/internal/eigenState/stakerShares" + "github.com/Layr-Labs/go-sidecar/internal/eigenState/stateManager" + "github.com/Layr-Labs/go-sidecar/internal/eigenState/submittedDistributionRoots" + "github.com/Layr-Labs/go-sidecar/internal/fetcher" + "github.com/Layr-Labs/go-sidecar/internal/indexer" + "github.com/Layr-Labs/go-sidecar/internal/logger" + "github.com/Layr-Labs/go-sidecar/internal/metrics" + "github.com/Layr-Labs/go-sidecar/internal/pipeline" + "github.com/Layr-Labs/go-sidecar/internal/shutdown" + "github.com/Layr-Labs/go-sidecar/internal/sidecar" + "github.com/Layr-Labs/go-sidecar/internal/sqlite" + "github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations" + sqliteBlockStore "github.com/Layr-Labs/go-sidecar/internal/storage/sqlite" + "go.uber.org/zap" +) + +func main() { + ctx := context.Background() + cfg := config.NewConfig() + + l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) + + sdc, err := metrics.InitStatsdClient(cfg.StatsdUrl) + if err != nil { + l.Sugar().Fatal("Failed to setup statsd client", zap.Error(err)) + } + + etherscanClient := etherscan.NewEtherscanClient(cfg, l) + client := ethereum.NewClient(cfg.EthereumRpcConfig.BaseUrl, l) + + db := sqlite.NewSqlite(cfg.GetSqlitePath(), l) + + grm, err := sqlite.NewGormSqliteFromSqlite(db) + if err != nil { + l.Error("Failed to create gorm instance", zap.Error(err)) + panic(err) + } + + migrator := migrations.NewSqliteMigrator(grm, l) + if err = migrator.MigrateAll(); err != nil { + log.Fatalf("Failed to migrate: %v", err) + } + + contractStore := sqliteContractStore.NewSqliteContractStore(grm, l, cfg) + if err := contractStore.InitializeCoreContracts(); err != nil { + log.Fatalf("Failed to initialize core contracts: %v", err) + } + + cm := contractManager.NewContractManager(contractStore, etherscanClient, client, sdc, l) + + mds := sqliteBlockStore.NewSqliteBlockStore(grm, l, cfg) + if err != nil { + log.Fatalln(err) + } + + sm := stateManager.NewEigenStateManager(l, grm) + + if _, err := avsOperators.NewAvsOperators(sm, grm, l, cfg); err != nil { + l.Sugar().Fatalw("Failed to create AvsOperatorsModel", zap.Error(err)) + } + if _, err := operatorShares.NewOperatorSharesModel(sm, grm, l, cfg); err != nil { + l.Sugar().Fatalw("Failed to create OperatorSharesModel", zap.Error(err)) + } + if _, err := stakerDelegations.NewStakerDelegationsModel(sm, grm, l, cfg); err != nil { + l.Sugar().Fatalw("Failed to create StakerDelegationsModel", zap.Error(err)) + } + if _, err := stakerShares.NewStakerSharesModel(sm, grm, l, cfg); err != nil { + l.Sugar().Fatalw("Failed to create StakerSharesModel", zap.Error(err)) + } + if _, err := submittedDistributionRoots.NewSubmittedDistributionRootsModel(sm, grm, l, cfg); err != nil { + l.Sugar().Fatalw("Failed to create SubmittedDistributionRootsModel", zap.Error(err)) + } + if _, err := rewardSubmissions.NewRewardSubmissionsModel(sm, grm, l, cfg); err != nil { + l.Sugar().Fatalw("Failed to create RewardSubmissionsModel", zap.Error(err)) + } + + fetchr := fetcher.NewFetcher(client, cfg, l) + + cc := contractCaller.NewContractCaller(client, l) + + idxr := indexer.NewIndexer(mds, contractStore, etherscanClient, cm, client, fetchr, cc, l, cfg) + + p := pipeline.NewPipeline(fetchr, idxr, mds, sm, l) + + // Create new sidecar instance + sidecar := sidecar.NewSidecar(&sidecar.SidecarConfig{ + GenesisBlockNumber: cfg.GetGenesisBlockNumber(), + }, cfg, mds, p, sm, l, client) + + // RPC channel to notify the RPC server to shutdown gracefully + rpcChannel := make(chan bool) + err = sidecar.WithRpcServer(ctx, mds, sm, rpcChannel) + if err != nil { + l.Sugar().Fatalw("Failed to start RPC server", zap.Error(err)) + } + + // Start the sidecar main process in a goroutine so that we can listen for a shutdown signal + go sidecar.Start(ctx) + + l.Sugar().Info("Started Sidecar") + + gracefulShutdown := shutdown.CreateGracefulShutdownChannel() + + done := make(chan bool) + shutdown.ListenForShutdown(gracefulShutdown, done, func() { + l.Sugar().Info("Shutting down...") + rpcChannel <- true + sidecar.ShutdownChan <- true + }, time.Second*5, l) +} diff --git a/cmd/sqlitedebug/db.py b/cmd/sqlitedebug/db.py new file mode 100644 index 00000000..5bb3c886 --- /dev/null +++ b/cmd/sqlitedebug/db.py @@ -0,0 +1,7 @@ +import sqlite3 + +conn = sqlite3.connect(":memory:") +conn.enable_load_extension(True) +conn.load_extension("/Users/seanmcgary/Code/sidecar/sqlite-extensions/yolo.dylib") +conn.execute("SELECT my_custom_function('foo')") +conn.close() diff --git a/cmd/sqlitedebug/main.go b/cmd/sqlitedebug/main.go new file mode 100644 index 00000000..0826af25 --- /dev/null +++ b/cmd/sqlitedebug/main.go @@ -0,0 +1,33 @@ +package main + +import ( + "database/sql" + "fmt" + "github.com/mattn/go-sqlite3" + _ "github.com/mattn/go-sqlite3" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +var extensionPath = "/Users/seanmcgary/Code/sidecar/sqlite-extensions/yolo.dylib" + +func main() { + sql.Register("sqlite3_with_extensions", &sqlite3.SQLiteDriver{ + Extensions: []string{extensionPath}, + }) + dialector := gorm.Dialector(sqlite.Dialector{ + DSN: "file:yourdatabase.db?cache=shared&_fk=1", + DriverName: "sqlite3_with_extensions", + }) + db, err := gorm.Open(dialector, &gorm.Config{}) + if err != nil { + panic("failed to connect database") + } + + var r string + res := db.Raw("SELECT pre_nile_tokens_per_day('500') as result").Scan(&r) + if res.Error != nil { + panic(fmt.Sprintf("failed to load extension: %v", err)) + } + fmt.Printf("Result: %s\n", r) +} diff --git a/go.mod b/go.mod index b3ab9abc..cc49f39e 100644 --- a/go.mod +++ b/go.mod @@ -5,11 +5,13 @@ go 1.22.2 require ( github.com/DataDog/datadog-go/v5 v5.5.0 github.com/ethereum/go-ethereum v1.14.9 + github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 github.com/jbrower95/multicall-go v0.0.0-20240923010412-060e37b98d03 github.com/mattn/go-sqlite3 v1.14.23 github.com/rs/cors v1.7.0 + github.com/shopspring/decimal v1.4.0 github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.19.0 @@ -27,31 +29,31 @@ require ( require ( github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/StackExchange/wmi v1.2.1 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect - github.com/bits-and-blooms/bitset v1.14.3 // indirect + github.com/bits-and-blooms/bitset v1.13.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect github.com/buger/jsonparser v1.1.1 // indirect - github.com/consensys/bavard v0.1.16 // indirect - github.com/consensys/gnark-crypto v0.14.0 // indirect - github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a // indirect - github.com/crate-crypto/go-kzg-4844 v1.1.0 // indirect + github.com/consensys/bavard v0.1.13 // indirect + github.com/consensys/gnark-crypto v0.12.1 // indirect + github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c // indirect + github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/deckarep/golang-set/v2 v2.6.0 // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect - github.com/ethereum/c-kzg-4844 v1.0.3 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect + github.com/ethereum/c-kzg-4844 v1.0.0 // indirect github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/gorilla/websocket v1.5.3 // indirect + github.com/gorilla/websocket v1.4.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/uint256 v1.3.1 // indirect github.com/iden3/go-iden3-crypto v0.0.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/klauspost/compress v1.17.9 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -61,22 +63,21 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect - github.com/shirou/gopsutil v3.21.11+incompatible // indirect + github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect - github.com/supranational/blst v0.3.13 // indirect - github.com/tklauser/go-sysconf v0.3.14 // indirect - github.com/tklauser/numcpus v0.8.0 // indirect - github.com/yusufpapurcu/wmi v1.2.4 // indirect + github.com/supranational/blst v0.3.11 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/crypto v0.27.0 // indirect - golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect - golang.org/x/net v0.29.0 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect + golang.org/x/net v0.26.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.25.0 // indirect - golang.org/x/text v0.18.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/text v0.17.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index fad0d458..404dfe43 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,8 @@ github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= +github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= @@ -14,8 +16,8 @@ github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xW github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bits-and-blooms/bitset v1.14.3 h1:Gd2c8lSNf9pKXom5JtD7AaKO8o7fGQ2LtFj1436qilA= -github.com/bits-and-blooms/bitset v1.14.3/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= +github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/btcsuite/btcd/btcec/v2 v2.3.4 h1:3EJjcN70HCu/mwqlUsGK8GcNVyLVxFDlWurTXGPFfiQ= github.com/btcsuite/btcd/btcec/v2 v2.3.4/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= @@ -41,32 +43,32 @@ github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwP github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= -github.com/consensys/bavard v0.1.16 h1:3+nT0BqzKg84VtCY9eNN2Glkf1X7dbS5yhh5849syJ8= -github.com/consensys/bavard v0.1.16/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= -github.com/consensys/gnark-crypto v0.14.0 h1:DDBdl4HaBtdQsq/wfMwJvZNE80sHidrK3Nfrefatm0E= -github.com/consensys/gnark-crypto v0.14.0/go.mod h1:CU4UijNPsHawiVGNxe9co07FkzCeWHHrb1li/n1XoU0= +github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= +github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= +github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= +github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a h1:W8mUrRp6NOVl3J+MYp5kPMoUZPp7aOYHtaua31lwRHg= -github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a/go.mod h1:sTwzHBvIzm2RfVCGNEBZgRyjwK40bVoun3ZnGOCafNM= -github.com/crate-crypto/go-kzg-4844 v1.1.0 h1:EN/u9k2TF6OWSHrCCDBBU6GLNMq88OspHHlMnHfoyU4= -github.com/crate-crypto/go-kzg-4844 v1.1.0/go.mod h1:JolLjpSff1tCCJKaJx4psrlEdlXuJEC996PL3tTAFks= +github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c h1:uQYC5Z1mdLRPrZhHjHxufI8+2UG/i25QG92j0Er9p6I= +github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs= +github.com/crate-crypto/go-kzg-4844 v1.0.0 h1:TsSgHwrkTKecKJ4kadtHi4b3xHW5dCFUDFnUp1TsawI= +github.com/crate-crypto/go-kzg-4844 v1.0.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= -github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= -github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= +github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ethereum/c-kzg-4844 v1.0.3 h1:IEnbOHwjixW2cTvKRUlAAUOeleV7nNM/umJR+qy4WDs= -github.com/ethereum/c-kzg-4844 v1.0.3/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= +github.com/ethereum/c-kzg-4844 v1.0.0 h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA= +github.com/ethereum/c-kzg-4844 v1.0.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/ethereum/go-ethereum v1.14.9 h1:J7iwXDrtUyE9FUjUYbd4c9tyzwMh6dTJsKzo9i6SrwA= github.com/ethereum/go-ethereum v1.14.9/go.mod h1:QeW+MtTpRdBEm2pUFoonByee8zfHv7kGp0wK0odvU1I= github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 h1:8NfxH2iXvJ60YRB8ChToFTUzl8awsc3cJ8CbLjGIl/A= @@ -81,10 +83,12 @@ github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 h1:FWNFq4fM1wPfcK40yHE5UO3RUdSNPaBC+j3PokzA6OQ= +github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -109,8 +113,8 @@ github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= -github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys= @@ -142,8 +146,8 @@ github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/ github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -154,8 +158,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/leanovate/gopter v0.2.11 h1:vRjThO1EKPb/1NsDXuDrzldR28RLkBflWYcU9CvzWu4= -github.com/leanovate/gopter v0.2.11/go.mod h1:aK3tzZP/C+p1m3SPRE4SYZFGP7jjkuSI4f7Xvpt0S9c= +github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= +github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -199,8 +203,8 @@ github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -209,8 +213,10 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= -github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= +github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= @@ -245,14 +251,14 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/supranational/blst v0.3.13 h1:AYeSxdOMacwu7FBmpfloBz5pbFXDmJL33RuwnKtmTjk= -github.com/supranational/blst v0.3.13/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= +github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= -github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= -github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= -github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYgY= -github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= @@ -266,8 +272,6 @@ github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsr github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= -github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -281,11 +285,11 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= -golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= -golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= +golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= +golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -302,8 +306,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= -golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -326,13 +330,15 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/internal/config/config.go b/internal/config/config.go index 70a1ba47..bd78cad5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,6 +1,7 @@ package config import ( + "errors" "fmt" "github.com/spf13/viper" "strings" @@ -13,10 +14,15 @@ type Environment int type Chain string +type ForkName string + const ( Chain_Mainnet Chain = "mainnet" Chain_Holesky Chain = "holesky" Chain_Preprod Chain = "preprod" + + Fork_Nile ForkName = "nile" + Fork_Amazon ForkName = "amazon" ) func parseListEnvVar(envVar string) []string { @@ -191,6 +197,29 @@ func (c *Config) GetGenesisBlockNumber() uint64 { } } +type ForkMap map[ForkName]string + +func (c *Config) GetForkDates() (ForkMap, error) { + switch c.Chain { + case Chain_Preprod: + return ForkMap{ + Fork_Amazon: "1970-01-01", // Amazon hard fork was never on preprod as we backfilled + Fork_Nile: "2024-08-14", // Last calculation end timestamp was 8-13: https://holesky.etherscan.io/tx/0xb5a6855e88c79312b7c0e1c9f59ae9890b97f157ea27e69e4f0fadada4712b64#eventlog + }, nil + case Chain_Holesky: + return ForkMap{ + Fork_Amazon: "1970-01-01", // Amazon hard fork was never on testnet as we backfilled + Fork_Nile: "2024-08-13", // Last calculation end timestamp was 8-12: https://holesky.etherscan.io/tx/0x5fc81b5ed2a78b017ef313c181d8627737a97fef87eee85acedbe39fc8708c56#eventlog + }, nil + case Chain_Mainnet: + return ForkMap{ + Fork_Amazon: "2024-08-02", // Last calculation end timestamp was 8-01: https://etherscan.io/tx/0x2aff6f7b0132092c05c8f6f41a5e5eeeb208aa0d95ebcc9022d7823e343dd012#eventlog + Fork_Nile: "2024-08-12", // Last calculation end timestamp was 8-11: https://etherscan.io/tx/0x922d29d93c02d189fc2332041f01a80e0007cd7a625a5663ef9d30082f7ef66f#eventlog + }, nil + } + return nil, errors.New("unsupported chain") +} + func (c *Config) GetEigenLayerGenesisBlockHeight() (uint64, error) { switch c.Chain { case Chain_Preprod, Chain_Holesky: diff --git a/internal/contractStore/sqliteContractStore/sqliteContractStore_test.go b/internal/contractStore/sqliteContractStore/sqliteContractStore_test.go index ce86f48a..35ac8b27 100644 --- a/internal/contractStore/sqliteContractStore/sqliteContractStore_test.go +++ b/internal/contractStore/sqliteContractStore/sqliteContractStore_test.go @@ -22,7 +22,7 @@ func setup() ( cfg := tests.GetConfig() l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) - db, err := tests.GetSqliteDatabaseConnection() + db, err := tests.GetInMemorySqliteDatabaseConnection(l) if err != nil { panic(err) } diff --git a/internal/eigenState/avsOperators/avsOperators.go b/internal/eigenState/avsOperators/avsOperators.go index 8d69e264..9a1f0ab5 100644 --- a/internal/eigenState/avsOperators/avsOperators.go +++ b/internal/eigenState/avsOperators/avsOperators.go @@ -2,6 +2,7 @@ package avsOperators import ( "database/sql" + "errors" "fmt" "slices" "sort" @@ -44,6 +45,14 @@ type RegisteredAvsOperatorDiff struct { Registered bool } +type AvsOperatorStateChange struct { + Avs string + Operator string + Registered bool + LogIndex uint64 + BlockNumber uint64 +} + func NewSlotID(avs string, operator string) types.SlotID { return types.SlotID(fmt.Sprintf("%s_%s", avs, operator)) } @@ -58,6 +67,9 @@ type AvsOperatorsModel struct { // Accumulates state changes for SlotIds, grouped by block number stateAccumulator map[uint64]map[types.SlotID]*AccumulatedStateChange + + // Keep track of each distinct change, rather than accumulated change, to add to the delta table + deltaAccumulator map[uint64][]*AvsOperatorStateChange } // NewAvsOperators creates a new AvsOperatorsModel. @@ -76,6 +88,8 @@ func NewAvsOperators( globalConfig: globalConfig, stateAccumulator: make(map[uint64]map[types.SlotID]*AccumulatedStateChange), + + deltaAccumulator: make(map[uint64][]*AvsOperatorStateChange), } esm.RegisterState(s, 0) return s, nil @@ -120,6 +134,15 @@ func (a *AvsOperatorsModel) GetStateTransitions() (types.StateTransitions[Accumu registered = uint64(val.(float64)) == 1 } + // Store the change in the delta accumulator + a.deltaAccumulator[log.BlockNumber] = append(a.deltaAccumulator[log.BlockNumber], &AvsOperatorStateChange{ + Avs: avs, + Operator: operator, + Registered: registered, + LogIndex: log.LogIndex, + BlockNumber: log.BlockNumber, + }) + slotID := NewSlotID(avs, operator) record, ok := a.stateAccumulator[log.BlockNumber][slotID] if !ok { @@ -173,6 +196,7 @@ func (a *AvsOperatorsModel) IsInterestingLog(log *storage.TransactionLog) bool { func (a *AvsOperatorsModel) InitBlockProcessing(blockNumber uint64) error { a.stateAccumulator[blockNumber] = make(map[types.SlotID]*AccumulatedStateChange) + a.deltaAccumulator[blockNumber] = make([]*AvsOperatorStateChange, 0) return nil } @@ -249,6 +273,24 @@ func (a *AvsOperatorsModel) prepareState(blockNumber uint64) ([]RegisteredAvsOpe return inserts, deletes, nil } +func (a *AvsOperatorsModel) writeDeltaRecordsToDeltaTable(blockNumber uint64) error { + records, ok := a.deltaAccumulator[blockNumber] + if !ok { + msg := "Delta accumulator was not initialized" + a.logger.Sugar().Errorw(msg, zap.Uint64("blockNumber", blockNumber)) + return errors.New(msg) + } + + if len(records) > 0 { + res := a.DB.Model(&AvsOperatorStateChange{}).Clauses(clause.Returning{}).Create(&records) + if res.Error != nil { + a.logger.Sugar().Errorw("Failed to insert delta records", zap.Error(res.Error)) + return res.Error + } + } + return nil +} + // CommitFinalState commits the final state for the given block number. func (a *AvsOperatorsModel) CommitFinalState(blockNumber uint64) error { err := a.clonePreviousBlocksToNewBlock(blockNumber) @@ -280,11 +322,17 @@ func (a *AvsOperatorsModel) CommitFinalState(blockNumber uint64) error { return res.Error } } + + if err = a.writeDeltaRecordsToDeltaTable(blockNumber); err != nil { + return err + } + return nil } func (a *AvsOperatorsModel) ClearAccumulatedState(blockNumber uint64) error { delete(a.stateAccumulator, blockNumber) + delete(a.deltaAccumulator, blockNumber) return nil } diff --git a/internal/eigenState/avsOperators/avsOperators_test.go b/internal/eigenState/avsOperators/avsOperators_test.go index d7cdddc1..b467d29f 100644 --- a/internal/eigenState/avsOperators/avsOperators_test.go +++ b/internal/eigenState/avsOperators/avsOperators_test.go @@ -25,7 +25,7 @@ func setup() ( cfg := tests.GetConfig() l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) - db, err := tests.GetSqliteDatabaseConnection() + db, err := tests.GetInMemorySqliteDatabaseConnection(l) if err != nil { panic(err) } @@ -40,6 +40,21 @@ func setup() ( func teardown(model *AvsOperatorsModel) { model.DB.Exec("delete from avs_operator_changes") model.DB.Exec("delete from registered_avs_operators") + model.DB.Exec("delete from avs_operator_state_changes") +} + +func getInsertedDeltaRecordsForBlock(blockNumber uint64, model *AvsOperatorsModel) ([]*AvsOperatorStateChange, error) { + results := []*AvsOperatorStateChange{} + + res := model.DB.Model(&AvsOperatorStateChange{}).Where("block_number = ?", blockNumber).Find(&results) + return results, res.Error +} + +func getInsertedDeltaRecords(model *AvsOperatorsModel) ([]*AvsOperatorStateChange, error) { + results := []*AvsOperatorStateChange{} + + res := model.DB.Model(&AvsOperatorStateChange{}).Order("block_number asc").Find(&results) + return results, res.Error } func Test_AvsOperatorState(t *testing.T) { @@ -84,7 +99,22 @@ func Test_AvsOperatorState(t *testing.T) { assert.Nil(t, err) assert.NotNil(t, res) - teardown(avsOperatorState) + err = avsOperatorState.CommitFinalState(blockNumber) + assert.Nil(t, err) + + inserted, err := getInsertedDeltaRecordsForBlock(blockNumber, avsOperatorState) + assert.Nil(t, err) + assert.Equal(t, 1, len(inserted)) + + assert.Equal(t, "0xdf25bdcdcdd9a3dd8c9069306c4dba8d90dd8e8e", inserted[0].Avs) + assert.Equal(t, "0x870679e138bcdf293b7ff14dd44b70fc97e12fc0", inserted[0].Operator) + assert.Equal(t, true, inserted[0].Registered) + assert.Equal(t, blockNumber, inserted[0].BlockNumber) + assert.Equal(t, uint64(400), inserted[0].LogIndex) + + t.Cleanup(func() { + teardown(avsOperatorState) + }) }) t.Run("Should register AvsOperatorState and generate the table for the block", func(t *testing.T) { esm := stateManager.NewEigenStateManager(l, grm) @@ -134,7 +164,9 @@ func Test_AvsOperatorState(t *testing.T) { assert.Nil(t, err) assert.True(t, len(stateRoot) > 0) - teardown(avsOperatorState) + t.Cleanup(func() { + teardown(avsOperatorState) + }) }) t.Run("Should correctly generate state across multiple blocks", func(t *testing.T) { esm := stateManager.NewEigenStateManager(l, grm) @@ -217,6 +249,12 @@ func Test_AvsOperatorState(t *testing.T) { assert.True(t, len(stateRoot) > 0) } - teardown(avsOperatorState) + inserted, err := getInsertedDeltaRecords(avsOperatorState) + assert.Nil(t, err) + assert.Equal(t, len(logs), len(inserted)) + + t.Cleanup(func() { + teardown(avsOperatorState) + }) }) } diff --git a/internal/eigenState/eigenstate_test.go b/internal/eigenState/eigenstate_test.go index 79ce5820..a42e47f6 100644 --- a/internal/eigenState/eigenstate_test.go +++ b/internal/eigenState/eigenstate_test.go @@ -23,7 +23,7 @@ func setup() ( cfg := tests.GetConfig() l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) - db, err := tests.GetSqliteDatabaseConnection() + db, err := tests.GetInMemorySqliteDatabaseConnection(l) if err != nil { panic(err) } diff --git a/internal/eigenState/operatorShares/operatorShares.go b/internal/eigenState/operatorShares/operatorShares.go index 7316aa84..71b6c260 100644 --- a/internal/eigenState/operatorShares/operatorShares.go +++ b/internal/eigenState/operatorShares/operatorShares.go @@ -4,6 +4,7 @@ import ( "database/sql" "encoding/json" "fmt" + pkgUtils "github.com/Layr-Labs/go-sidecar/pkg/utils" "math/big" "slices" "sort" @@ -212,32 +213,9 @@ func (osm *OperatorSharesModel) HandleStateChange(log *storage.TransactionLog) ( return nil, nil //nolint:nilnil } -func (osm *OperatorSharesModel) clonePreviousBlocksToNewBlock(blockNumber uint64) error { - query := ` - insert into operator_shares (operator, strategy, shares, block_number) - select - operator, - strategy, - shares, - @currentBlock as block_number - from operator_shares - where block_number = @previousBlock - ` - res := osm.DB.Exec(query, - sql.Named("currentBlock", blockNumber), - sql.Named("previousBlock", blockNumber-1), - ) - - if res.Error != nil { - osm.logger.Sugar().Errorw("Failed to clone previous block state to new block", zap.Error(res.Error)) - return res.Error - } - return nil -} - // prepareState prepares the state for commit by adding the new state to the existing state. -func (osm *OperatorSharesModel) prepareState(blockNumber uint64) ([]OperatorSharesDiff, error) { - preparedState := make([]OperatorSharesDiff, 0) +func (osm *OperatorSharesModel) prepareState(blockNumber uint64) ([]*OperatorSharesDiff, error) { + preparedState := make([]*OperatorSharesDiff, 0) accumulatedState, ok := osm.stateAccumulator[blockNumber] if !ok { @@ -253,19 +231,28 @@ func (osm *OperatorSharesModel) prepareState(blockNumber uint64) ([]OperatorShar // Find only the records from the previous block, that are modified in this block query := ` + with ranked_rows as ( + select + operator, + strategy, + shares, + block_number, + ROW_NUMBER() OVER (PARTITION BY operator, strategy ORDER BY block_number desc) as rn + from operator_shares + where + concat(operator, '_', strategy) in @slotIds + ) select - operator, - strategy, - shares - from operator_shares - where - block_number = @previousBlock - and concat(operator, '_', strategy) in @slotIds + lb.operator, + lb.strategy, + lb.shares, + lb.block_number + from ranked_rows as lb + where rn = 1 ` - existingRecords := make([]OperatorShares, 0) + existingRecords := make([]*OperatorShares, 0) res := osm.DB.Model(&OperatorShares{}). Raw(query, - sql.Named("previousBlock", blockNumber-1), sql.Named("slotIds", slotIds), ). Scan(&existingRecords) @@ -276,9 +263,8 @@ func (osm *OperatorSharesModel) prepareState(blockNumber uint64) ([]OperatorShar } // Map the existing records to a map for easier lookup - mappedRecords := make(map[types.SlotID]OperatorShares) + mappedRecords := make(map[types.SlotID]*OperatorShares) for _, record := range existingRecords { - fmt.Printf("Existing OperatorShares %+v\n", record) slotID := NewSlotID(record.Operator, record.Strategy) mappedRecords[slotID] = record } @@ -286,7 +272,7 @@ func (osm *OperatorSharesModel) prepareState(blockNumber uint64) ([]OperatorShar // Loop over our new state changes. // If the record exists in the previous block, add the shares to the existing shares for slotID, newState := range accumulatedState { - prepared := OperatorSharesDiff{ + prepared := &OperatorSharesDiff{ Operator: newState.Operator, Strategy: newState.Strategy, Shares: newState.Shares, @@ -317,56 +303,27 @@ func (osm *OperatorSharesModel) prepareState(blockNumber uint64) ([]OperatorShar } func (osm *OperatorSharesModel) CommitFinalState(blockNumber uint64) error { - // Clone the previous block state to give us a reference point. - err := osm.clonePreviousBlocksToNewBlock(blockNumber) - if err != nil { - return err - } - records, err := osm.prepareState(blockNumber) if err != nil { return err } - newRecords := make([]OperatorShares, 0) - updateRecords := make([]OperatorShares, 0) - - for _, record := range records { - r := &OperatorShares{ - Operator: record.Operator, - Strategy: record.Strategy, - Shares: record.Shares.String(), - BlockNumber: record.BlockNumber, - } - if record.IsNew { - newRecords = append(newRecords, *r) - } else { - updateRecords = append(updateRecords, *r) + recordToInsert := pkgUtils.Map(records, func(r *OperatorSharesDiff, i uint64) *OperatorShares { + return &OperatorShares{ + Operator: r.Operator, + Strategy: r.Strategy, + Shares: r.Shares.String(), + BlockNumber: blockNumber, } - } + }) - // Batch insert new records - if len(newRecords) > 0 { - res := osm.DB.Model(&OperatorShares{}).Clauses(clause.Returning{}).Create(&newRecords) + if len(recordToInsert) > 0 { + res := osm.DB.Model(&OperatorShares{}).Clauses(clause.Returning{}).Create(&recordToInsert) if res.Error != nil { osm.logger.Sugar().Errorw("Failed to create new operator_shares records", zap.Error(res.Error)) return res.Error } } - // Update existing records that were cloned from the previous block - if len(updateRecords) > 0 { - for _, record := range updateRecords { - res := osm.DB.Model(&OperatorShares{}). - Where("operator = ? and strategy = ? and block_number = ?", record.Operator, record.Strategy, record.BlockNumber). - Updates(map[string]interface{}{ - "shares": record.Shares, - }) - if res.Error != nil { - osm.logger.Sugar().Errorw("Failed to update operator_shares record", zap.Error(res.Error)) - return res.Error - } - } - } return nil } @@ -391,7 +348,7 @@ func (osm *OperatorSharesModel) GenerateStateRoot(blockNumber uint64) (types.Sta return types.StateRoot(utils.ConvertBytesToString(fullTree.Root())), nil } -func (osm *OperatorSharesModel) sortValuesForMerkleTree(diffs []OperatorSharesDiff) []*base.MerkleTreeInput { +func (osm *OperatorSharesModel) sortValuesForMerkleTree(diffs []*OperatorSharesDiff) []*base.MerkleTreeInput { inputs := make([]*base.MerkleTreeInput, 0) for _, diff := range diffs { inputs = append(inputs, &base.MerkleTreeInput{ diff --git a/internal/eigenState/operatorShares/operatorShares_test.go b/internal/eigenState/operatorShares/operatorShares_test.go index f5586822..af690c70 100644 --- a/internal/eigenState/operatorShares/operatorShares_test.go +++ b/internal/eigenState/operatorShares/operatorShares_test.go @@ -27,7 +27,7 @@ func setup() ( cfg := tests.GetConfig() l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) - db, err := tests.GetSqliteDatabaseConnection() + db, err := tests.GetInMemorySqliteDatabaseConnection(l) if err != nil { panic(err) } @@ -40,7 +40,6 @@ func setup() ( } func teardown(model *OperatorSharesModel) { - model.DB.Exec("delete from operator_share_changes") model.DB.Exec("delete from operator_shares") } diff --git a/internal/eigenState/rewardSubmissions/rewardSubmissions.go b/internal/eigenState/rewardSubmissions/rewardSubmissions.go index 473d02eb..10445df2 100644 --- a/internal/eigenState/rewardSubmissions/rewardSubmissions.go +++ b/internal/eigenState/rewardSubmissions/rewardSubmissions.go @@ -34,7 +34,7 @@ type RewardSubmission struct { EndTimestamp *time.Time `gorm:"type:DATETIME"` Duration uint64 BlockNumber uint64 - IsForAll bool + RewardType string // avs, all_stakers, all_earners } type RewardSubmissionDiff struct { @@ -151,6 +151,15 @@ func (rs *RewardSubmissionsModel) handleRewardSubmissionCreatedEvent(log *storag return nil, xerrors.Errorf("Failed to parse multiplier to Big257: %s", actualOuputData.Amount.String()) } + var rewardType string + if log.EventName == "RewardsSubmissionForAllCreated" || log.EventName == "RangePaymentForAllCreated" { + rewardType = "all_stakers" + } else if log.EventName == "RangePaymentCreated" || log.EventName == "AVSRewardsSubmissionCreated" { + rewardType = "avs" + } else { + return nil, xerrors.Errorf("Unknown event name: %s", log.EventName) + } + rewardSubmission := &RewardSubmission{ Avs: strings.ToLower(arguments[0].Value.(string)), RewardHash: strings.ToLower(arguments[2].Value.(string)), @@ -162,7 +171,7 @@ func (rs *RewardSubmissionsModel) handleRewardSubmissionCreatedEvent(log *storag EndTimestamp: &endTimestamp, Duration: actualOuputData.Duration, BlockNumber: log.BlockNumber, - IsForAll: log.EventName == "RewardsSubmissionForAllCreated" || log.EventName == "RangePaymentForAllCreated", + RewardType: rewardType, } rewardSubmissions = append(rewardSubmissions, rewardSubmission) } @@ -252,7 +261,7 @@ func (rs *RewardSubmissionsModel) HandleStateChange(log *storage.TransactionLog) func (rs *RewardSubmissionsModel) clonePreviousBlocksToNewBlock(blockNumber uint64) error { query := ` - insert into reward_submissions(avs, reward_hash, token, amount, strategy, strategy_index, multiplier, start_timestamp, end_timestamp, duration, is_for_all, block_number) + insert into reward_submissions(avs, reward_hash, token, amount, strategy, strategy_index, multiplier, start_timestamp, end_timestamp, duration, reward_type, block_number) select avs, reward_hash, @@ -264,7 +273,7 @@ func (rs *RewardSubmissionsModel) clonePreviousBlocksToNewBlock(blockNumber uint start_timestamp, end_timestamp, duration, - is_for_all, + reward_type, @currentBlock as block_number from reward_submissions where block_number = @previousBlock diff --git a/internal/eigenState/rewardSubmissions/rewardSubmissions_test.go b/internal/eigenState/rewardSubmissions/rewardSubmissions_test.go index 2310baa2..17c8aa76 100644 --- a/internal/eigenState/rewardSubmissions/rewardSubmissions_test.go +++ b/internal/eigenState/rewardSubmissions/rewardSubmissions_test.go @@ -27,7 +27,7 @@ func setup() ( cfg := tests.GetConfig() l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) - db, err := tests.GetSqliteDatabaseConnection() + db, err := tests.GetInMemorySqliteDatabaseConnection(l) if err != nil { panic(err) } diff --git a/internal/eigenState/stakerDelegations/stakerDelegations.go b/internal/eigenState/stakerDelegations/stakerDelegations.go index 72a92262..d55514a2 100644 --- a/internal/eigenState/stakerDelegations/stakerDelegations.go +++ b/internal/eigenState/stakerDelegations/stakerDelegations.go @@ -2,6 +2,7 @@ package stakerDelegations import ( "database/sql" + "errors" "fmt" "slices" "sort" @@ -36,6 +37,14 @@ type AccumulatedStateChange struct { Delegated bool } +type StakerDelegationChange struct { + Staker string + Operator string + BlockNumber uint64 + Delegated bool + LogIndex uint64 +} + func NewSlotID(staker string, operator string) types.SlotID { return types.SlotID(fmt.Sprintf("%s_%s", staker, operator)) } @@ -49,6 +58,8 @@ type StakerDelegationsModel struct { // Accumulates state changes for SlotIds, grouped by block number stateAccumulator map[uint64]map[types.SlotID]*AccumulatedStateChange + + deltaAccumulator map[uint64][]*StakerDelegationChange } type DelegatedStakersDiff struct { @@ -72,6 +83,8 @@ func NewStakerDelegationsModel( logger: logger, globalConfig: globalConfig, stateAccumulator: make(map[uint64]map[types.SlotID]*AccumulatedStateChange), + + deltaAccumulator: make(map[uint64][]*StakerDelegationChange), } esm.RegisterState(model, 2) @@ -123,6 +136,15 @@ func (s *StakerDelegationsModel) GetStateTransitions() (types.StateTransitions[A record.Delegated = true } + // Store the change in the delta accumulator + s.deltaAccumulator[log.BlockNumber] = append(s.deltaAccumulator[log.BlockNumber], &StakerDelegationChange{ + Staker: staker, + Operator: operator, + BlockNumber: log.BlockNumber, + Delegated: record.Delegated, + LogIndex: log.LogIndex, + }) + return record, nil } @@ -157,6 +179,7 @@ func (s *StakerDelegationsModel) IsInterestingLog(log *storage.TransactionLog) b // InitBlockProcessing initialize state accumulator for the block. func (s *StakerDelegationsModel) InitBlockProcessing(blockNumber uint64) error { s.stateAccumulator[blockNumber] = make(map[types.SlotID]*AccumulatedStateChange) + s.deltaAccumulator[blockNumber] = make([]*StakerDelegationChange, 0) return nil } @@ -229,6 +252,24 @@ func (s *StakerDelegationsModel) prepareState(blockNumber uint64) ([]DelegatedSt return inserts, deletes, nil } +func (s *StakerDelegationsModel) writeDeltaRecordsToDeltaTable(blockNumber uint64) error { + records, ok := s.deltaAccumulator[blockNumber] + if !ok { + msg := "Delta accumulator was not initialized" + s.logger.Sugar().Errorw(msg, zap.Uint64("blockNumber", blockNumber)) + return errors.New(msg) + } + + if len(records) > 0 { + res := s.DB.Model(&StakerDelegationChange{}).Clauses(clause.Returning{}).Create(&records) + if res.Error != nil { + s.logger.Sugar().Errorw("Failed to insert delta records", zap.Error(res.Error)) + return res.Error + } + } + return nil +} + func (s *StakerDelegationsModel) CommitFinalState(blockNumber uint64) error { // Clone the previous block state to give us a reference point. // @@ -264,12 +305,17 @@ func (s *StakerDelegationsModel) CommitFinalState(blockNumber uint64) error { return res.Error } } + + if err = s.writeDeltaRecordsToDeltaTable(blockNumber); err != nil { + return err + } return nil } // ClearAccumulatedState clears the accumulated state for the given block number to free up memory. func (s *StakerDelegationsModel) ClearAccumulatedState(blockNumber uint64) error { delete(s.stateAccumulator, blockNumber) + delete(s.deltaAccumulator, blockNumber) return nil } diff --git a/internal/eigenState/stakerDelegations/stakerDelegations_test.go b/internal/eigenState/stakerDelegations/stakerDelegations_test.go index e74bf03e..e312a465 100644 --- a/internal/eigenState/stakerDelegations/stakerDelegations_test.go +++ b/internal/eigenState/stakerDelegations/stakerDelegations_test.go @@ -24,7 +24,7 @@ func setup() ( cfg := tests.GetConfig() l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) - db, err := tests.GetSqliteDatabaseConnection() + db, err := tests.GetInMemorySqliteDatabaseConnection(l) if err != nil { panic(err) } @@ -39,6 +39,7 @@ func setup() ( func teardown(model *StakerDelegationsModel) { model.DB.Exec("delete from staker_delegation_changes") model.DB.Exec("delete from delegated_stakers") + model.DB.Exec("delete from staker_delegation_changes") } func Test_DelegatedStakersState(t *testing.T) { diff --git a/internal/eigenState/stakerShares/stakerShares.go b/internal/eigenState/stakerShares/stakerShares.go index 9c9a2d63..d580825c 100644 --- a/internal/eigenState/stakerShares/stakerShares.go +++ b/internal/eigenState/stakerShares/stakerShares.go @@ -18,6 +18,7 @@ import ( "github.com/Layr-Labs/go-sidecar/internal/storage" "github.com/Layr-Labs/go-sidecar/internal/types/numbers" "github.com/Layr-Labs/go-sidecar/internal/utils" + pkgUtils "github.com/Layr-Labs/go-sidecar/pkg/utils" "go.uber.org/zap" "golang.org/x/xerrors" "gorm.io/gorm" @@ -484,32 +485,9 @@ func (ss *StakerSharesModel) HandleStateChange(log *storage.TransactionLog) (int return nil, nil } -func (ss *StakerSharesModel) clonePreviousBlocksToNewBlock(blockNumber uint64) error { - query := ` - insert into staker_shares (staker, strategy, shares, block_number) - select - staker, - strategy, - shares, - @currentBlock as block_number - from staker_shares - where block_number = @previousBlock - ` - res := ss.DB.Exec(query, - sql.Named("currentBlock", blockNumber), - sql.Named("previousBlock", blockNumber-1), - ) - - if res.Error != nil { - ss.logger.Sugar().Errorw("Failed to clone previous block state to new block", zap.Error(res.Error)) - return res.Error - } - return nil -} - // prepareState prepares the state for commit by adding the new state to the existing state. -func (ss *StakerSharesModel) prepareState(blockNumber uint64) ([]StakerSharesDiff, error) { - preparedState := make([]StakerSharesDiff, 0) +func (ss *StakerSharesModel) prepareState(blockNumber uint64) ([]*StakerSharesDiff, error) { + preparedState := make([]*StakerSharesDiff, 0) accumulatedState, ok := ss.stateAccumulator[blockNumber] if !ok { @@ -525,19 +503,28 @@ func (ss *StakerSharesModel) prepareState(blockNumber uint64) ([]StakerSharesDif // Find only the records from the previous block, that are modified in this block query := ` + with ranked_rows as ( + select + staker, + strategy, + shares, + block_number, + ROW_NUMBER() OVER (PARTITION BY staker, strategy ORDER BY block_number desc) as rn + from staker_shares + where + concat(staker, '_', strategy) in @slotIds + ) select - staker, - strategy, - shares - from staker_shares - where - block_number = @previousBlock - and concat(staker, '_', strategy) in @slotIds + rr.staker, + rr.strategy, + rr.shares, + rr.block_number + from ranked_rows as rr + where rn = 1 ` existingRecords := make([]StakerShares, 0) res := ss.DB.Model(&StakerShares{}). Raw(query, - sql.Named("previousBlock", blockNumber-1), sql.Named("slotIds", slotIds), ). Scan(&existingRecords) @@ -557,12 +544,11 @@ func (ss *StakerSharesModel) prepareState(blockNumber uint64) ([]StakerSharesDif // Loop over our new state changes. // If the record exists in the previous block, add the shares to the existing shares for slotId, newState := range accumulatedState { - prepared := StakerSharesDiff{ + prepared := &StakerSharesDiff{ Staker: newState.Staker, Strategy: newState.Strategy, Shares: newState.Shares, BlockNumber: blockNumber, - IsNew: false, } if existingRecord, ok := mappedRecords[slotId]; ok { @@ -577,9 +563,6 @@ func (ss *StakerSharesModel) prepareState(blockNumber uint64) ([]StakerSharesDif continue } prepared.Shares = existingShares.Add(existingShares, newState.Shares) - } else { - // SlotID was not found in the previous block, so this is a new record - prepared.IsNew = true } preparedState = append(preparedState, prepared) @@ -588,56 +571,27 @@ func (ss *StakerSharesModel) prepareState(blockNumber uint64) ([]StakerSharesDif } func (ss *StakerSharesModel) CommitFinalState(blockNumber uint64) error { - // Clone the previous block state to give us a reference point. - err := ss.clonePreviousBlocksToNewBlock(blockNumber) - if err != nil { - return err - } - records, err := ss.prepareState(blockNumber) if err != nil { return err } - newRecords := make([]StakerShares, 0) - updateRecords := make([]StakerShares, 0) - - for _, record := range records { - r := &StakerShares{ - Staker: record.Staker, - Strategy: record.Strategy, - Shares: record.Shares.String(), - BlockNumber: record.BlockNumber, - } - if record.IsNew { - newRecords = append(newRecords, *r) - } else { - updateRecords = append(updateRecords, *r) + recordsToInsert := pkgUtils.Map(records, func(r *StakerSharesDiff, i uint64) *StakerShares { + return &StakerShares{ + Staker: r.Staker, + Strategy: r.Strategy, + Shares: r.Shares.String(), + BlockNumber: blockNumber, } - } + }) - // Batch insert new records - if len(newRecords) > 0 { - res := ss.DB.Model(&StakerShares{}).Clauses(clause.Returning{}).Create(&newRecords) + if len(recordsToInsert) > 0 { + res := ss.DB.Model(&StakerShares{}).Clauses(clause.Returning{}).Create(&recordsToInsert) if res.Error != nil { ss.logger.Sugar().Errorw("Failed to create new operator_shares records", zap.Error(res.Error)) return res.Error } } - // Update existing records that were cloned from the previous block - if len(updateRecords) > 0 { - for _, record := range updateRecords { - res := ss.DB.Model(&StakerShares{}). - Where("staker = ? and strategy = ? and block_number = ?", record.Staker, record.Strategy, record.BlockNumber). - Updates(map[string]interface{}{ - "shares": record.Shares, - }) - if res.Error != nil { - ss.logger.Sugar().Errorw("Failed to update operator_shares record", zap.Error(res.Error)) - return res.Error - } - } - } return nil } @@ -662,7 +616,7 @@ func (ss *StakerSharesModel) GenerateStateRoot(blockNumber uint64) (types.StateR return types.StateRoot(utils.ConvertBytesToString(fullTree.Root())), nil } -func (ss *StakerSharesModel) sortValuesForMerkleTree(diffs []StakerSharesDiff) []*base.MerkleTreeInput { +func (ss *StakerSharesModel) sortValuesForMerkleTree(diffs []*StakerSharesDiff) []*base.MerkleTreeInput { inputs := make([]*base.MerkleTreeInput, 0) for _, diff := range diffs { inputs = append(inputs, &base.MerkleTreeInput{ diff --git a/internal/eigenState/stakerShares/stakerShares_test.go b/internal/eigenState/stakerShares/stakerShares_test.go index 996ae9eb..526aabb7 100644 --- a/internal/eigenState/stakerShares/stakerShares_test.go +++ b/internal/eigenState/stakerShares/stakerShares_test.go @@ -27,7 +27,7 @@ func setup() ( cfg := tests.GetConfig() l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) - db, err := tests.GetSqliteDatabaseConnection() + db, err := tests.GetInMemorySqliteDatabaseConnection(l) if err != nil { panic(err) } @@ -313,9 +313,6 @@ func Test_StakerSharesState(t *testing.T) { assert.Equal(t, "0x298afb19a105d59e74658c4c334ff360bade6dd2", preparedChange[0].Strategy) assert.Equal(t, "246393621132195985", preparedChange[0].Shares.String()) - err = model.clonePreviousBlocksToNewBlock(blockNumber) - assert.Nil(t, err) - err = model.CommitFinalState(blockNumber) assert.Nil(t, err) diff --git a/internal/eigenState/stateManager/stateManager_test.go b/internal/eigenState/stateManager/stateManager_test.go index 8ed04672..e9437576 100644 --- a/internal/eigenState/stateManager/stateManager_test.go +++ b/internal/eigenState/stateManager/stateManager_test.go @@ -22,7 +22,7 @@ func setup() ( cfg := tests.GetConfig() l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) - db, err := tests.GetSqliteDatabaseConnection() + db, err := tests.GetInMemorySqliteDatabaseConnection(l) if err != nil { panic(err) } diff --git a/internal/eigenState/submittedDistributionRoots/submittedDistributionRoots_test.go b/internal/eigenState/submittedDistributionRoots/submittedDistributionRoots_test.go index 9cae8350..34224a19 100644 --- a/internal/eigenState/submittedDistributionRoots/submittedDistributionRoots_test.go +++ b/internal/eigenState/submittedDistributionRoots/submittedDistributionRoots_test.go @@ -24,7 +24,7 @@ func setup() ( cfg := tests.GetConfig() l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) - db, err := tests.GetSqliteDatabaseConnection() + db, err := tests.GetInMemorySqliteDatabaseConnection(l) if err != nil { panic(err) } diff --git a/internal/indexer/restakedStrategies_test.go b/internal/indexer/restakedStrategies_test.go index 0ad0ae1c..021d5ccb 100644 --- a/internal/indexer/restakedStrategies_test.go +++ b/internal/indexer/restakedStrategies_test.go @@ -41,7 +41,7 @@ func setup() ( cfg := tests.GetConfig() l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) - db, err := tests.GetSqliteDatabaseConnection() + db, err := tests.GetInMemorySqliteDatabaseConnection(l) if err != nil { panic(err) } diff --git a/internal/pipeline/pipeline_integration_test.go b/internal/pipeline/pipeline_integration_test.go index c9b0b910..b4f64dc3 100644 --- a/internal/pipeline/pipeline_integration_test.go +++ b/internal/pipeline/pipeline_integration_test.go @@ -62,7 +62,7 @@ func setup() ( client := ethereum.NewClient(rpcUrl, l) // database - grm, err := tests.GetSqliteDatabaseConnection() + grm, err := tests.GetInMemorySqliteDatabaseConnection(l) if err != nil { panic(err) } diff --git a/internal/python/lossyTokens.py b/internal/python/lossyTokens.py new file mode 100644 index 00000000..99fdf5f3 --- /dev/null +++ b/internal/python/lossyTokens.py @@ -0,0 +1,597 @@ +from decimal import Decimal, getcontext, ROUND_UP, ROUND_HALF_UP, ROUND_DOWN, ROUND_HALF_DOWN, ROUND_FLOOR + +def lossyAdd(tokens: str): + big_amount = float(tokens) + div = 0.999999999999999 + res = big_amount * div + + res_str = "{}".format(res) + return "{}".format(int(Decimal(res_str))) + +def amazonStakerTokenRewards(sp:str, tpd:str) -> str: + getcontext().prec = 15 + stakerProportion = Decimal(sp) + tokensPerDay = Decimal(tpd) + + decimal_res = Decimal(stakerProportion * tokensPerDay) + + getcontext().prec = 20 + rounded = decimal_res.quantize(Decimal('1'), rounding=ROUND_HALF_UP) + + return "{}".format(rounded) + + +def nilStakerTokenRewards(sp:str, tpd:str) -> str: + print(getcontext().prec) + # getcontext().prec = 38 + stakerProportion = Decimal(sp) + tokensPerDay = Decimal(tpd) + + print("{}, {}".format(stakerProportion, tokensPerDay)) + decimal_res = Decimal(stakerProportion * tokensPerDay) + print("Decimal: {}".format(decimal_res)) + truncated = decimal_res.quantize(Decimal('0.1'), rounding=ROUND_UP) + print("Truncated: {}".format(truncated)) + rounded = truncated.quantize(Decimal('1'), rounding=ROUND_HALF_UP) + + return "{}".format(rounded) + +def stakerTokenRewards(sp:str, tpd:str) -> str: + stakerProportion = float(sp) + tokensPerDay = int(tpd) + + decimal_res = stakerProportion * tokensPerDay + + parts = str(decimal_res).split("+") + # Need more precision + if len(parts) == 2 and int(parts[1]) > 16: + stakerProportion = Decimal(sp) + tokensPerDay = Decimal(tpd) + + getcontext().prec = 17 + getcontext().rounding = ROUND_DOWN + decimal_res = stakerProportion * tokensPerDay + + return "{}".format(int(decimal_res)) + + return "{}".format(int(decimal_res)) + +def stakerTokenRewardsAlt(sp:str, tpd:str) -> str: + stakerProportion = Decimal(sp) + tokensPerDay = Decimal(tpd) + + getcontext().prec = 17 + getcontext().rounding = ROUND_DOWN + print("{} {}".format(stakerProportion, tokensPerDay)) + decimal_res = stakerProportion * tokensPerDay + print("{}".format(decimal_res)) + + return "{}".format(int(decimal_res)) + + +def nileOperatorTokenRewards(totalStakerOperatorTokens:str) -> str: + if totalStakerOperatorTokens[-1] == "0": + return "{}".format(int(totalStakerOperatorTokens) // 10) + totalStakerOperatorTokens = Decimal(totalStakerOperatorTokens) + operatorTokens = Decimal(str(totalStakerOperatorTokens)) * Decimal(0.1) + rounded = operatorTokens.quantize(Decimal('1'), rounding=ROUND_HALF_UP) + return "{}".format(rounded) + +#print(lossyAdd("1428571428571428571428571428571428571.4142857142857143")) +#print(lossyCalc("0.092173497467823", "142857142857142700000000")) + +numbers = [ + ["2262498437749998","226249843775000"], + ["1077804256999999","107780425700000"], + ["1200335945499999","120033594550000"], + ["2579821510499997","257982151050000"], + ["1077804256999999","107780425700000"], + ["97982205000000","9798220500000"], + ["2731616159749997","273161615975000"], + ["1468013852499999","146801385250000"], + ["1460746763499999","146074676350000"], + ["97982205000000","9798220500000"], + ["2160308386499998","216030838650000"], + ["2173632495999998","217363249600000"], + ["979822051749999","97982205175000"], + ["1077804256999999","107780425700000"], + ["97982205000000","9798220500000"], + ["1205181123749999","120518112375000"], + ["1692326232749998","169232623275000"], + ["19596441000000","1959644100000"], + ["1077804256999999","107780425700000"], + ["2766886966999997","276688696700000"], + ["1421109574749999","142110957475000"], + ["1327025256749999","132702525675000"], + ["1513152979749999","151315297975000"], + ["1724319284249998","172431928425000"], + ["1881072278499998","188107227850000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["2767046562249997","276704656225000"], + ["1771544047249998","177154404725000"], + ["979822051749999","97982205175000"], + ["2766618839499997","276661883950000"], + ["1111499404999999","111149940500000"], + ["1256537372499999","125653737250000"], + ["1311309356749999","131130935675000"], + ["1077804256999999","107780425700000"], + ["1476639073749999","147663907375000"], + ["619368817749999","61936881775000"], + ["2351572924249998","235157292425000"], + ["1077804256999999","107780425700000"], + ["38824366794749960","3882436679474996"], + ["1329923441749999","132992344175000"], + ["1077804256999999","107780425700000"], + ["1171094469499999","117109446950000"], + ["1077804256999999","107780425700000"], + ["2164538729999998","216453873000000"], + ["2094265522499998","209426552250000"], + ["1448972310749999","144897231075000"], + ["1077804256999999","107780425700000"], + ["1403751361499999","140375136150000"], + ["979822051749999","97982205175000"], + ["979822051749999","97982205175000"], + ["1085802969749999","108580296975000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1124308426749999","112430842675000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1182901487499999","118290148750000"], + ["1372466011999999","137246601200000"], + ["979822051749999","97982205175000"], + ["1120320939499999","112032093950000"], + ["1077804256999999","107780425700000"], + ["2813743578249997","281374357825000"], + ["1997209544499998","199720954450000"], + ["342937718000000","34293771800000"], + ["1077804256999999","107780425700000"], + ["2504518724749998","250451872475000"], + ["489911025749999","48991102575000"], + ["1169357997749999","116935799775000"], + ["1077804256999999","107780425700000"], + ["1263158553749999","126315855375000"], + ["1094640640499999","109464064050000"], + ["1284065770249999","128406577025000"], + ["1521653686749998","152165368675000"], + ["1077804256999999","107780425700000"], + ["1695020089999998","169502009000000"], + ["1077804256999999","107780425700000"], + ["1089590665249999","108959066525000"], + ["4333353594249996","433335359425000"], + ["979822051749999","97982205175000"], + ["2766860861499997","276686086150000"], + ["3737419616999996","373741961700000"], + ["1682669036499998","168266903650000"], + ["979822051749999","97982205175000"], + ["718515954363749200","71851595436374920"], + ["1077804256999999","107780425700000"], + ["3350684550249997","335068455025000"], + ["979822051749999","97982205175000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["2766670760499997","276667076050000"], + ["1139779232999999","113977923300000"], + ["1294703182249999","129470318225000"], + ["1401943427499999","140194342750000"], + ["1077804256999999","107780425700000"], + ["8661942075499991","866194207549999"], + ["979822051749999","97982205175000"], + ["1077804256999999","107780425700000"], + ["979822051749999","97982205175000"], + ["9798220500000","979822050000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1329616345749999","132961634575000"], + ["2285721732999998","228572173300000"], + ["440830750000","44083075000"], + ["1309869932249999","130986993225000"], + ["1077804256999999","107780425700000"], + ["2767143131249997","276714313125000"], + ["1087449895749999","108744989575000"], + ["1440582461749999","144058246175000"], + ["1437596903499999","143759690350000"], + ["3739760468499996","373976046850000"], + ["1228930342749999","122893034275000"], + ["1668502454499998","166850245450000"], + ["1077804256999999","107780425700000"], + ["20400709468749976","2040070946874998"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1501152824749998","150115282475000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1588788984249999","158878898425000"], + ["97982205000000","9798220500000"], + ["2766955102999997","276695510300000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1213777201499999","121377720150000"], + ["2766522236999998","276652223700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1518724180249999","151872418025000"], + ["1077804256999999","107780425700000"], + ["102040547000000","10204054700000"], + ["1521663646499999","152166364650000"], + ["2766883311999998","276688331200000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1239913664499999","123991366450000"], + ["1083648329249999","108364832925000"], + ["2766567664499997","276656766450000"], + ["1077804256999999","107780425700000"], + ["121497934250000","12149793425000"], + ["1489329518749999","148932951875000"], + ["705471877249999","70547187725000"], + ["1365074446999999","136507444700000"], + ["148259503000000","14825950300000"], + ["120582017500000","12058201750000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["3458772690249997","345877269025000"], + ["1096846600499999","109684660050000"], + ["2642116312499998","264211631250000"], + ["1186049670749999","118604967075000"], + ["3379361777249997","337936177725000"], + ["1073027971999999","107302797200000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["2205003116999998","220500311700000"], + ["1400476333249999","140047633325000"], + ["1334754243499999","133475424350000"], + ["75000261135249920","7500026113524992"], + ["2766917160499997","276691716050000"], + ["1077804256999999","107780425700000"], + ["4081356354749996","408135635475000"], + ["115064965500000","11506496550000"], + ["140863986000000","14086398600000"], + ["1421544889999999","142154489000000"], + ["1249751425499999","124975142550000"], + ["1522711647499999","152271164750000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["2766928990999997","276692899100000"], + ["1251222885499999","125122288550000"], + ["1804670182999998","180467018300000"], + ["1286074936749999","128607493675000"], + ["22474595357499976","2247459535749998"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["2766772303499997","276677230350000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1636477811999998","163647781200000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["2766759761749998","276675976175000"], + ["1077804256999999","107780425700000"], + ["2367891235249998","236789123525000"], + ["2111355676249998","211135567625000"], + ["97982205000000","9798220500000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1335070157749999","133507015775000"], + ["1088824402499999","108882440250000"], + ["1591863078499998","159186307850000"], + ["1093209052249999","109320905225000"], + ["1231725361499999","123172536150000"], + ["1077804256999999","107780425700000"], + ["1188849744499999","118884974450000"], + ["1325974469749999","132597446975000"], + ["1077804256999999","107780425700000"], + ["1170727959499999","117072795950000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["2766687807749997","276668780775000"], + ["979822051749999","97982205175000"], + ["1077804256999999","107780425700000"], + ["1195382903249999","119538290325000"], + ["131740039000000","13174003900000"], + ["1077804256999999","107780425700000"], + ["1584454986249999","158445498625000"], + ["1496388154999999","149638815500000"], + ["2783380170999997","278338017100000"], + ["2766853466749997","276685346675000"], + ["97982205000000","9798220500000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1259437513249999","125943751325000"], + ["2559660094249998","255966009425000"], + ["97982205000000","9798220500000"], + ["97982205000000","9798220500000"], + ["1077804256999999","107780425700000"], + ["7828778194249992","782877819424999"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1138055195249999","113805519525000"], + ["979822051749999","97982205175000"], + ["1192936111999999","119293611200000"], + ["1117604526499999","111760452650000"], + ["1077804256999999","107780425700000"], + ["391928820500000","39192882050000"], + ["1671343943749998","167134394375000"], + ["1237619152499999","123761915250000"], + ["1461227295249999","146122729525000"], + ["536422250000","53642225000"], + ["1077804256999999","107780425700000"], + ["746783849749999","74678384975000"], + ["8686889065749991","868688906574999"], + ["979822051749999","97982205175000"], + ["1077804256999999","107780425700000"], + ["3577907333749997","357790733375000"], + ["9798220500000","979822050000"], + ["2766688656499997","276668865650000"], + ["1077804256999999","107780425700000"], + ["1190360713999999","119036071400000"], + ["1372170432249999","137217043225000"], + ["1180097737749999","118009773775000"], + ["979822051749999","97982205175000"], + ["1077804256999999","107780425700000"], + ["1231665739249999","123166573925000"], + ["1250928951499999","125092895150000"], + ["1077804256999999","107780425700000"], + ["1122446018749999","112244601875000"], + ["1077804256999999","107780425700000"], + ["3465020847749997","346502084775000"], + ["497502843250000","49750284325000"], + ["2766694622499997","276669462250000"], + ["1077804256999999","107780425700000"], + ["9799200250000","979920025000"], + ["1077804256999999","107780425700000"], + ["3393183871249997","339318387125000"], + ["1224777564749999","122477756475000"], + ["1077804256999999","107780425700000"], + ["2766784225749997","276678422575000"], + ["1084088666249999","108408866625000"], + ["979822051749999","97982205175000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["120671798250000","12067179825000"], + ["97982205000000","9798220500000"], + ["1207335820749999","120733582075000"], + ["979822051749999","97982205175000"], + ["1708717776499998","170871777650000"], + ["1077804256999999","107780425700000"], + ["2171666879249998","217166687925000"], + ["2766540531749997","276654053175000"], + ["97982205000000","9798220500000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1139514050249999","113951405025000"], + ["1347136987749999","134713698775000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["2373897634749998","237389763475000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1296568008249999","129656800825000"], + ["1747592574499998","174759257450000"], + ["1718446376999998","171844637700000"], + ["97982205000000","9798220500000"], + ["1077804256999999","107780425700000"], + ["1126212608249999","112621260825000"], + ["1382620828249999","138262082825000"], + ["1827529957249998","182752995725000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["2766997874749997","276699787475000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["3558586463999997","355858646400000"], + ["148139296000000","14813929600000"], + ["1228361717249999","122836171725000"], + ["1120021586249999","112002158625000"], + ["1077804256999999","107780425700000"], + ["75838000000","7583800000"], + ["2813176036999997","281317603700000"], + ["1126418676499999","112641867650000"], + ["1077804256999999","107780425700000"], + ["1168952628749999","116895262875000"], + ["1286002109999999","128600211000000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1106691471749999","110669147175000"], + ["1077804256999999","107780425700000"], + ["2767147036499997","276714703650000"], + ["2766894477499997","276689447750000"], + ["1077804256999999","107780425700000"], + ["1221210601249999","122121060125000"], + ["1077804256999999","107780425700000"], + ["1474944833499998","147494483350000"], + ["3355500944749997","335550094475000"], + ["36042871697249960","3604287169724996"], + ["1077804256999999","107780425700000"], + ["103861137250000","10386113725000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["7982080906749991","798208090674999"], + ["1077804256999999","107780425700000"], + ["6600292500000","660029250000"], + ["158317476750000","15831747675000"], + ["1077804256999999","107780425700000"], + ["1266130821999999","126613082200000"], + ["1087461430249999","108746143025000"], + ["1925894882749998","192589488275000"], + ["109489270750000","10948927075000"], + ["3011552248749997","301155224875000"], + ["5302500000","530250000"], + ["1077804256999999","107780425700000"], + ["1295497476749999","129549747675000"], + ["1514257759249998","151425775925000"], + ["1973412247749998","197341224775000"], + ["2612205589999997","261220559000000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1215310149499999","121531014950000"], + ["1077804256999999","107780425700000"], + ["1087895020499999","108789502050000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["147078692250000","14707869225000"], + ["2464874880999998","246487488100000"], + ["1166980098249999","116698009825000"], + ["734866538749999","73486653875000"], + ["2163704354999998","216370435500000"], + ["1281449592249999","128144959225000"], + ["1198409126249999","119840912625000"], + ["2767120898499997","276712089850000"], + ["1959088190749998","195908819075000"], + ["1077804256999999","107780425700000"], + ["1170951049999999","117095105000000"], + ["1154675041749999","115467504175000"], + ["1134384833499999","113438483350000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1194101009249999","119410100925000"], + ["1388429900249999","138842990025000"], + ["979822051749999","97982205175000"], + ["1122446018749999","112244601875000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1731852292499998","173185229250000"], + ["1396115605249999","139611560525000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1081755392249999","108175539225000"], + ["1115170505749999","111517050575000"], + ["1077804256999999","107780425700000"], + ["1880982576249998","188098257625000"], + ["1253732266249999","125373226625000"], + ["1077804256999999","107780425700000"], + ["117578912750000","11757891275000"], + ["2766816535749997","276681653575000"], + ["1151717784499999","115171778450000"], + ["1073440472499999","107344047250000"], + ["1842220677249998","184222067725000"], + ["1444291238499999","144429123850000"], + ["185920687750000","18592068775000"], + ["1077804256999999","107780425700000"], + ["2766739048249997","276673904825000"], + ["1077804256999999","107780425700000"], + ["1111127251249999","111112725125000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["2495212757249998","249521275725000"], + ["1109533228999999","110953322900000"], + ["1204528088999999","120452808900000"], + ["2767240998749997","276724099875000"], + ["1077804256999999","107780425700000"], + ["27022701521749972","2702270152174997"], + ["1133721705999999","113372170600000"], + ["2766637369749997","276663736975000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["979822051749999","97982205175000"], + ["861929137749999","86192913775000"], + ["1077804256999999","107780425700000"], + ["3503727170499996","350372717050000"], + ["1060787189499999","106078718950000"], + ["979822051749999","97982205175000"], + ["2045249932249998","204524993225000"], + ["482354676250000","48235467625000"], + ["1077804256999999","107780425700000"], + ["1397360907249999","139736090725000"], + ["1175341173499999","117534117350000"], + ["2767044838249997","276704483825000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1449542205249999","144954220525000"], + ["1077804256999999","107780425700000"], + ["2321707502749998","232170750275000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1513531123249999","151353112325000"], + ["2740460193499997","274046019350000"], + ["1077804256999999","107780425700000"], + ["1149339404999999","114933940500000"], + ["1151809466249999","115180946625000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1249904385249999","124990438525000"], + ["1096106635999999","109610663600000"], + ["6898005302249993","689800530224999"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["2766392857999997","276639285800000"], + ["1077804256999999","107780425700000"], + ["2921368330249997","292136833025000"], + ["1369704807999999","136970480800000"], + ["1077804256999999","107780425700000"], + ["5006256564499995","500625656450000"], + ["1077804256999999","107780425700000"], + ["109740069750000","10974006975000"], + ["1077804256999999","107780425700000"], + ["1346236941999999","134623694200000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1103601738249999","110360173825000"], + ["2704308862999998","270430886300000"], + ["864763585499999","86476358550000"], + ["979822051749999","97982205175000"], + ["1576305397999998","157630539800000"], + ["147299275500000","14729927550000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1321185449499999","132118544950000"], + ["1077804256999999","107780425700000"], + ["1244374005749999","124437400575000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["979822051749999","97982205175000"], + ["1077804256999999","107780425700000"], + ["2175404280999998","217540428100000"], + ["2324262255249998","232426225525000"], + ["1087602477499999","108760247750000"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["1226466551749999","122646655175000"], + ["2797069106499997","279706910650000"], + ["1122461615499999","112246161550000"], + ["1156738466749999","115673846675000"], + ["5920104740249994","592010474024999"], + ["1077804256999999","107780425700000"], + ["1077804256999999","107780425700000"], + ["13604498250000","1360449825000"], + ["1367625821749999","136762582175000"], +] +issues = 0 +indexes = [] +#for i, number in enumerate(numbers): +for i, number in enumerate(numbers): + res = nileOperatorTokenRewards(number[0]) + if res != number[1]: + issues += 1 + indexes.append(i) + print("{}".format(number[0])) + print("Expected: {}".format(number[1])) + print("Got: {}".format(res)) + print("--------") +print("issues: {} {}".format(issues, indexes)) diff --git a/internal/sqlite/migrations/202409161057_avsOperatorDeltas/up.go b/internal/sqlite/migrations/202409161057_avsOperatorDeltas/up.go new file mode 100644 index 00000000..40c42647 --- /dev/null +++ b/internal/sqlite/migrations/202409161057_avsOperatorDeltas/up.go @@ -0,0 +1,36 @@ +package _202409161057_avsOperatorDeltas + +import ( + "fmt" + "gorm.io/gorm" +) + +type SqliteMigration struct { +} + +func (m *SqliteMigration) Up(grm *gorm.DB) error { + queries := []string{ + `create table if not exists avs_operator_state_changes ( + operator TEXT NOT NULL, + avs TEXT NOT NULL, + block_number INTEGER NOT NULL, + log_index INTEGER NOT NULL, + created_at DATETIME default current_timestamp, + registered integer not null, + unique(operator, avs, block_number, log_index) + ); + `, + } + + for _, query := range queries { + if res := grm.Exec(query); res.Error != nil { + fmt.Printf("Failed to execute query: %s\n", query) + return res.Error + } + } + return nil +} + +func (m *SqliteMigration) GetName() string { + return "202409161057_avsOperatorDeltas" +} diff --git a/internal/sqlite/migrations/202409181340_stakerDelegationDelta/up.go b/internal/sqlite/migrations/202409181340_stakerDelegationDelta/up.go new file mode 100644 index 00000000..b93b4048 --- /dev/null +++ b/internal/sqlite/migrations/202409181340_stakerDelegationDelta/up.go @@ -0,0 +1,29 @@ +package _202409181340_stakerDelegationDelta + +import ( + "gorm.io/gorm" +) + +type SqliteMigration struct { +} + +func (m *SqliteMigration) Up(grm *gorm.DB) error { + query := ` + create table if not exists staker_delegation_changes ( + staker TEXT NOT NULL, + operator TEXT NOT NULL, + delegated INTEGER NOT NULL, + block_number INTEGER NOT NULL, + log_index INTEGER NOT NULL + ) + ` + res := grm.Exec(query) + if res.Error != nil { + return res.Error + } + return nil +} + +func (m *SqliteMigration) GetName() string { + return "202409181340_stakerDelegationDelta" +} diff --git a/internal/sqlite/migrations/202409191425_addRewardTypeColumn/up.go b/internal/sqlite/migrations/202409191425_addRewardTypeColumn/up.go new file mode 100644 index 00000000..4a3c52c3 --- /dev/null +++ b/internal/sqlite/migrations/202409191425_addRewardTypeColumn/up.go @@ -0,0 +1,60 @@ +package _202409191425_addRewardTypeColumn + +import ( + "gorm.io/gorm" +) + +type SqliteMigration struct { +} + +func (m *SqliteMigration) Up(grm *gorm.DB) error { + queries := []string{ + `create table if not exists reward_submissions_new ( + avs TEXT NOT NULL, + reward_hash TEST NOT NULL, + token TEXT NOT NULL, + amount TEXT NOT NULL, + strategy TEXT NOT NULL, + strategy_index INTEGER NOT NULL, + multiplier TEXT NOT NULL, + start_timestamp DATETIME NOT NULL, + end_timestamp DATETIME NOT NULL, + duration INTEGER NOT NULL, + block_number INTEGER NOT NULL, + reward_type string, + unique(reward_hash, strategy, block_number) + );`, + `insert into reward_submissions_new + select + avs, + reward_hash, + token, + amount, + strategy, + strategy_index, + multiplier, + start_timestamp, + end_timestamp, + duration, + block_number, + CASE is_for_all + WHEN 1 THEN 'all_stakers' + ELSE 'avs' + END as reward_type + from reward_submissions; + `, + `drop table reward_submissions;`, + `alter table reward_submissions_new rename to reward_submissions;`, + } + + for _, query := range queries { + if err := grm.Exec(query).Error; err != nil { + return err + } + } + return nil +} + +func (m *SqliteMigration) GetName() string { + return "202409191425_addRewardTypeColumn" +} diff --git a/internal/sqlite/migrations/migrator.go b/internal/sqlite/migrations/migrator.go index 332ceec6..065632ab 100644 --- a/internal/sqlite/migrations/migrator.go +++ b/internal/sqlite/migrations/migrator.go @@ -3,6 +3,9 @@ package migrations import ( "database/sql" "fmt" + _202409161057_avsOperatorDeltas "github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations/202409161057_avsOperatorDeltas" + _202409181340_stakerDelegationDelta "github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations/202409181340_stakerDelegationDelta" + _202409191425_addRewardTypeColumn "github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations/202409191425_addRewardTypeColumn" "time" _202409061249_bootstrapDb "github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations/202409061249_bootstrapDb" @@ -52,6 +55,9 @@ func (m *SqliteMigrator) MigrateAll() error { &_202409101144_submittedDistributionRoot.SqliteMigration{}, &_202409101540_rewardSubmissions.SqliteMigration{}, &_202409111509_removeOperatorRestakedStrategiesBlockConstraint.SqliteMigration{}, + &_202409161057_avsOperatorDeltas.SqliteMigration{}, + &_202409181340_stakerDelegationDelta.SqliteMigration{}, + &_202409191425_addRewardTypeColumn.SqliteMigration{}, } m.Logger.Sugar().Info("Running migrations") diff --git a/internal/sqlite/sqlite.go b/internal/sqlite/sqlite.go index 281d48f3..63d6177b 100644 --- a/internal/sqlite/sqlite.go +++ b/internal/sqlite/sqlite.go @@ -5,6 +5,9 @@ import ( "encoding/hex" "encoding/json" "fmt" + "github.com/Layr-Labs/go-sidecar/internal/types/numbers" + "github.com/shopspring/decimal" + "go.uber.org/zap" "os" "path/filepath" "regexp" @@ -42,12 +45,121 @@ func InitSqliteDir(path string) error { return nil } -func NewSqlite(path string) gorm.Dialector { - sql.Register("sqlite3_with_extensions", &goSqlite.SQLiteDriver{ - ConnectHook: func(conn *goSqlite.SQLiteConn) error { - return conn.RegisterFunc("bytes_to_hex", bytesToHex, true) - }, - }) +type SumBigNumbers struct { + total decimal.Decimal +} + +func NewSumBigNumbers() *SumBigNumbers { + zero, _ := decimal.NewFromString("0") + return &SumBigNumbers{total: zero} +} + +func (s *SumBigNumbers) Step(value any) { + bigValue, err := decimal.NewFromString(value.(string)) + if err != nil { + return + } + s.total = s.total.Add(bigValue) +} + +func (s *SumBigNumbers) Done() (string, error) { + return s.total.String(), nil +} + +var hasRegisteredExtensions = false + +const SqliteInMemoryPath = "file::memory:?cache=shared" + +func NewInMemorySqlite(l *zap.Logger) gorm.Dialector { + return NewSqlite(SqliteInMemoryPath, l) +} + +func NewInMemorySqliteWithName(name string, l *zap.Logger) gorm.Dialector { + path := fmt.Sprintf("file:%s?mode=memory&cache=shared", name) + return NewSqlite(path, l) +} + +func NewSqlite(path string, l *zap.Logger) gorm.Dialector { + if !hasRegisteredExtensions { + sql.Register("sqlite3_with_extensions", &goSqlite.SQLiteDriver{ + ConnectHook: func(conn *goSqlite.SQLiteConn) error { + // Generic functions + if err := conn.RegisterAggregator("sum_big", NewSumBigNumbers, true); err != nil { + l.Sugar().Errorw("Failed to register aggregator sum_big", "error", err) + return err + } + if err := conn.RegisterFunc("subtract_big", numbers.SubtractBig, true); err != nil { + l.Sugar().Errorw("Failed to register function subtract_big", "error", err) + return err + } + if err := conn.RegisterFunc("numeric_multiply", numbers.NumericMultiply, true); err != nil { + l.Sugar().Errorw("Failed to register function NumericMultiply", "error", err) + return err + } + if err := conn.RegisterFunc("big_gt", numbers.BigGreaterThan, true); err != nil { + l.Sugar().Errorw("Failed to register function BigGreaterThan", "error", err) + return err + } + + if err := conn.RegisterFunc("bytes_to_hex", bytesToHex, true); err != nil { + l.Sugar().Errorw("Failed to register function bytes_to_hex", "error", err) + return err + } + // Raw tokens per day + if err := conn.RegisterFunc("calc_raw_tokens_per_day", numbers.CalcRawTokensPerDay, true); err != nil { + l.Sugar().Errorw("Failed to register function calc_raw_tokens_per_day", "error", err) + return err + } + // Forked tokens per day + if err := conn.RegisterFunc("post_nile_tokens_per_day", numbers.PostNileTokensPerDay, true); err != nil { + l.Sugar().Errorw("Failed to register function PostNileTokensPerDay", "error", err) + return err + } + if err := conn.RegisterFunc("pre_nile_tokens_per_day", numbers.PreNileTokensPerDay, true); err != nil { + l.Sugar().Errorw("Failed to register function PreNileTokensPerDay", "error", err) + return err + } + // Staker proportion and weight + if err := conn.RegisterFunc("calc_staker_proportion", numbers.CalculateStakerProportion, true); err != nil { + l.Sugar().Errorw("Failed to register function CalculateStakerProportion", "error", err) + return err + } + if err := conn.RegisterFunc("calc_staker_weight", numbers.CalculateStakerWeight, true); err != nil { + l.Sugar().Errorw("Failed to register function CalculateStakerWeight", "error", err) + return err + } + // Forked rewards for stakers + if err := conn.RegisterFunc("amazon_token_rewards", numbers.CalculateAmazonStakerTokenRewards, true); err != nil { + l.Sugar().Errorw("Failed to register function CalculateAmazonStakerTokenRewards", "error", err) + return err + } + if err := conn.RegisterFunc("nile_token_rewards", numbers.CalculateNileStakerTokenRewards, true); err != nil { + l.Sugar().Errorw("Failed to register function CalculateNileStakerTokenRewards", "error", err) + return err + } + if err := conn.RegisterFunc("post_nile_token_rewards", numbers.CalculatePostNileStakerTokenRewards, true); err != nil { + l.Sugar().Errorw("Failed to register function CalculatePostNileStakerTokenRewards", "error", err) + return err + } + // Operator tokens + if err := conn.RegisterFunc("amazon_operator_tokens", numbers.CalculateAmazonOperatorTokens, true); err != nil { + l.Sugar().Errorw("Failed to register function CalculateAmazonOperatorTokens", "error", err) + return err + } + if err := conn.RegisterFunc("nile_operator_tokens", numbers.CalculateNileOperatorTokens, true); err != nil { + l.Sugar().Errorw("Failed to register function CalculateNileOperatorTokens", "error", err) + return err + } + if err := conn.RegisterFunc("post_nile_operator_tokens", numbers.CalculatePostNileOperatorTokens, true); err != nil { + l.Sugar().Errorw("Failed to register function CalculatePostNileOperatorTokens", "error", err) + return err + } + return nil + }, + }) + hasRegisteredExtensions = true + } + return &sqlite.Dialector{ DriverName: "sqlite3_with_extensions", DSN: path, diff --git a/internal/sqlite/sqlite_test.go b/internal/sqlite/sqlite_test.go index 736b02f9..b7f98500 100644 --- a/internal/sqlite/sqlite_test.go +++ b/internal/sqlite/sqlite_test.go @@ -2,12 +2,17 @@ package sqlite import ( "encoding/hex" + "fmt" + "github.com/Layr-Labs/go-sidecar/internal/logger" "github.com/stretchr/testify/assert" + "math/big" "strings" "testing" ) func Test_Sqlite(t *testing.T) { + l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: true}) + t.Run("Should use the bytesToHex function", func(t *testing.T) { query := ` with json_values as ( @@ -20,7 +25,7 @@ func Test_Sqlite(t *testing.T) { from json_values limit 1 ` - s := NewSqlite("file::memory:?cache=shared") + s := NewSqlite("file::memory:?cache=shared", l) grm, err := NewGormSqliteFromSqlite(s) assert.Nil(t, err) @@ -36,4 +41,69 @@ func Test_Sqlite(t *testing.T) { assert.Nil(t, res.Error) assert.Equal(t, strings.ToLower(hex.EncodeToString(expectedBytes)), hexValue.WithdrawalHex) }) + t.Run("Should sum two really big numbers that are stored as strings", func(t *testing.T) { + shares1 := "1670000000000000000000" + shares2 := "1670000000000000000000" + + type shares struct { + Shares string + } + + operatorShares := []*shares{ + &shares{ + Shares: shares1, + }, + &shares{ + Shares: shares2, + }, + } + + s := NewSqlite("file::memory:?cache=shared", l) + grm, err := NewGormSqliteFromSqlite(s) + assert.Nil(t, err) + + createQuery := ` + create table shares ( + shares TEXT NOT NULL + ) + ` + res := grm.Exec(createQuery) + assert.Nil(t, res.Error) + + res = grm.Model(&shares{}).Create(&operatorShares) + assert.Nil(t, res.Error) + + query := ` + select + sum_big(shares) as total + from shares + ` + var total string + res = grm.Raw(query).Scan(&total) + assert.Nil(t, res.Error) + fmt.Printf("Total: %s\n", total) + + shares1Big, _ := new(big.Int).SetString(shares1, 10) + shares2Big, _ := new(big.Int).SetString(shares2, 10) + + expectedTotal := shares1Big.Add(shares1Big, shares2Big) + + assert.Equal(t, expectedTotal.String(), total) + }) + t.Run("Custom functions", func(t *testing.T) { + t.Run("Should call calc_raw_tokens_per_day", func(t *testing.T) { + query := `select calc_raw_tokens_per_day('100', 100) as amt` + + s := NewSqlite("file::memory:?cache=shared", l) + grm, err := NewGormSqliteFromSqlite(s) + assert.Nil(t, err) + + var result string + res := grm.Raw(query).Scan(&result) + assert.Nil(t, res.Error) + + assert.Equal(t, "86400.0000000000080179", result) + + }) + }) } diff --git a/internal/storage/sqlite/sqlite_test.go b/internal/storage/sqlite/sqlite_test.go index 41e43cc3..7109afb1 100644 --- a/internal/storage/sqlite/sqlite_test.go +++ b/internal/storage/sqlite/sqlite_test.go @@ -20,7 +20,7 @@ import ( func setup() (*gorm.DB, *zap.Logger, *config.Config) { cfg := config.NewConfig() l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: true}) - db, err := tests.GetSqliteDatabaseConnection() + db, err := tests.GetInMemorySqliteDatabaseConnection(l) if err != nil { panic(err) } diff --git a/internal/tests/testdata/combinedRewards/README.md b/internal/tests/testdata/combinedRewards/README.md new file mode 100644 index 00000000..7433b824 --- /dev/null +++ b/internal/tests/testdata/combinedRewards/README.md @@ -0,0 +1,26 @@ +## Source query + +```sql +with filtered as ( + select * from dbt_testnet_holesky_rewards.rewards_combined + where block_time < '2024-09-17' +), +expanded as ( + select + f.avs, + f.reward_hash, + f.token, + f.amount::text as amount, + f.strategy, + f.strategy_index, + f.multiplier::text as multiplier, + f.start_timestamp, + f.end_timestamp, + f.reward_type, + f.duration, + f.block_number as block_number + from filtered as f +) +select * from expanded + +``` diff --git a/internal/tests/testdata/operatorAvsRegistrationSnapshots/README.md b/internal/tests/testdata/operatorAvsRegistrationSnapshots/README.md new file mode 100644 index 00000000..cd153efe --- /dev/null +++ b/internal/tests/testdata/operatorAvsRegistrationSnapshots/README.md @@ -0,0 +1,31 @@ +## Source query: + +```sql +with filtered as ( + SELECT + lower(t.arguments #>> '{0,Value}') as operator, + lower(t.arguments #>> '{1,Value}') as avs, + (t.output_data -> 'status')::int as status, + t.transaction_hash, + t.log_index, + b.block_time, + to_char(b.block_time, 'YYYY-MM-DD') AS block_date, + t.block_number + FROM transaction_logs t + LEFT JOIN blocks b ON t.block_sequence_id = b.id + WHERE t.address = '0x055733000064333caddbc92763c58bf0192ffebf' + AND t.event_name = 'OperatorAVSRegistrationStatusUpdated' + AND date_trunc('day', b.block_time) < TIMESTAMP '2024-09-17' +) +select + operator, + avs, + status as registered, + log_index, + block_number +from filtered +``` + +## Expected results + +_See `generateExpectedResults.sql`_ diff --git a/internal/tests/testdata/operatorAvsRegistrationSnapshots/generateExpectedResults.sql b/internal/tests/testdata/operatorAvsRegistrationSnapshots/generateExpectedResults.sql new file mode 100644 index 00000000..3a338ea1 --- /dev/null +++ b/internal/tests/testdata/operatorAvsRegistrationSnapshots/generateExpectedResults.sql @@ -0,0 +1,67 @@ +COPY ( + with filtered as ( + select * from dbt_testnet_holesky_rewards.operator_avs_status + where block_time < '2024-09-17' +), +marked_statuses AS ( + SELECT + operator, + avs, + registered, + block_time, + block_date, + LEAD(block_time) OVER (PARTITION BY operator, avs ORDER BY block_time ASC, log_index ASC) AS next_block_time, + LEAD(registered) OVER (PARTITION BY operator, avs ORDER BY block_time ASC, log_index ASC) AS next_registration_status, + LEAD(block_date) OVER (PARTITION BY operator, avs ORDER BY block_time ASC, log_index ASC) AS next_block_date, + LAG(registered) OVER (PARTITION BY operator, avs ORDER BY block_time ASC, log_index ASC) AS prev_registered, + LAG(block_date) OVER (PARTITION BY operator, avs ORDER BY block_time ASC, log_index ASC) AS prev_block_date + FROM filtered +), + removed_same_day_deregistrations AS ( + SELECT * from marked_statuses + WHERE NOT ( + (registered = TRUE AND + COALESCE(next_registration_status = FALSE, false) AND + COALESCE(block_date = next_block_date, false)) OR + (registered = FALSE AND + COALESCE(prev_registered = TRUE, false) and + COALESCE(block_date = prev_block_date, false) + ) + ) + ), + registration_periods AS ( + SELECT + operator, + avs, + block_time AS start_time, + COALESCE(next_block_time, TIMESTAMP '2024-09-01') AS end_time, + registered + FROM removed_same_day_deregistrations + WHERE registered = TRUE + ), + registration_windows_extra as ( + SELECT + operator, + avs, + date_trunc('day', start_time) + interval '1' day as start_time, + date_trunc('day', end_time) as end_time + FROM registration_periods + ), + operator_avs_registration_windows as ( + SELECT * from registration_windows_extra + WHERE start_time != end_time + ), + cleaned_records AS ( + SELECT * FROM operator_avs_registration_windows + WHERE start_time < end_time + ), + final_results as ( + SELECT + operator, + avs, + day AS snapshot + FROM cleaned_records + CROSS JOIN generate_series(DATE(start_time), DATE(end_time) - interval '1' day, interval '1' day) AS day + ) + select * from final_results +) TO STDOUT WITH DELIMITER ',' CSV HEADER; diff --git a/internal/tests/testdata/operatorRestakedStrategies/README.md b/internal/tests/testdata/operatorRestakedStrategies/README.md new file mode 100644 index 00000000..ab0dad6d --- /dev/null +++ b/internal/tests/testdata/operatorRestakedStrategies/README.md @@ -0,0 +1,18 @@ +## Source + +```sql +select + block_number, + operator, + avs, + strategy, + block_time, + avs_directory_address +from operator_restaked_strategies +where avs_directory_address = '0x055733000064333caddbc92763c58bf0192ffebf' +and block_time < '2024-09-17' +``` + +## Expected results + +_See `generateExpectedResults.sql`_ diff --git a/internal/tests/testdata/operatorRestakedStrategies/generateExpectedResults.sql b/internal/tests/testdata/operatorRestakedStrategies/generateExpectedResults.sql new file mode 100644 index 00000000..5236cc2d --- /dev/null +++ b/internal/tests/testdata/operatorRestakedStrategies/generateExpectedResults.sql @@ -0,0 +1,112 @@ +copy (with ranked_records AS ( + SELECT + lower(operator) as operator, + lower(avs) as avs, + lower(strategy) as strategy, + block_time, + date_trunc('day', CAST(block_time as timestamp(6))) + interval '1' day as start_time, + ROW_NUMBER() OVER ( + PARTITION BY operator, avs, strategy, date_trunc('day', CAST(block_time as timestamp(6))) + interval '1' day + ORDER BY block_time DESC + ) AS rn + FROM public.operator_restaked_strategies + WHERE avs_directory_address = lower('0x055733000064333caddbc92763c58bf0192ffebf') + and block_time < '2024-09-17' +), + latest_records AS ( + SELECT + operator, + avs, + strategy, + start_time, + block_time + FROM ranked_records + WHERE rn = 1 + ), + grouped_records AS ( + SELECT + operator, + avs, + strategy, + start_time, + LEAD(start_time) OVER ( + PARTITION BY operator, avs, strategy + ORDER BY start_time ASC + ) AS next_start_time + FROM latest_records + ), + parsed_ranges AS ( + SELECT + operator, + avs, + strategy, + start_time, + CASE + WHEN next_start_time IS NULL OR next_start_time > start_time + INTERVAL '1' DAY THEN start_time + ELSE next_start_time + END AS end_time + FROM grouped_records + ), + active_windows as ( + SELECT * + FROM parsed_ranges + WHERE start_time != end_time + ), + gaps_and_islands AS ( + SELECT + operator, + avs, + strategy, + start_time, + end_time, + LAG(end_time) OVER(PARTITION BY operator, avs, strategy ORDER BY start_time) as prev_end_time + FROM active_windows + ), + island_detection AS ( + SELECT operator, avs, strategy, start_time, end_time, prev_end_time, + CASE + WHEN prev_end_time = start_time THEN 0 + ELSE 1 + END as new_island + FROM gaps_and_islands + ), + island_groups AS ( + SELECT + operator, + avs, + strategy, + start_time, + end_time, + SUM(new_island) OVER ( + PARTITION BY operator, avs, strategy ORDER BY start_time + ) AS island_id + FROM island_detection + ), + operator_avs_strategy_windows AS ( + SELECT + operator, + avs, + strategy, + MIN(start_time) AS start_time, + MAX(end_time) AS end_time + FROM island_groups + GROUP BY operator, avs, strategy, island_id + ORDER BY operator, avs, strategy, start_time + ), + cleaned_records AS ( + SELECT * FROM operator_avs_strategy_windows + WHERE start_time < end_time + ), + final_results as ( +SELECT + operator, + avs, + strategy, + cast(day AS DATE) AS snapshot +FROM + cleaned_records + CROSS JOIN + generate_series(DATE(start_time), DATE(end_time) - interval '1' day, interval '1' day) AS day +) +select * from final_results +) to STDOUT DELIMITER ',' CSV HEADER; diff --git a/internal/tests/testdata/operatorShareSnapshots/README.md b/internal/tests/testdata/operatorShareSnapshots/README.md new file mode 100644 index 00000000..641ef007 --- /dev/null +++ b/internal/tests/testdata/operatorShareSnapshots/README.md @@ -0,0 +1,17 @@ +## Source data + +```sql +select + operator, + strategy, + block_number, + sum(shares)::text as shares +from dbt_testnet_holesky_rewards.operator_shares +where block_time < '2024-09-17' +group by 1, 2, 3 +``` + +## Expected results + +_See `generateExpectedResults.sql`_ + diff --git a/internal/tests/testdata/operatorShareSnapshots/generateExpectedResults.sql b/internal/tests/testdata/operatorShareSnapshots/generateExpectedResults.sql new file mode 100644 index 00000000..9d49693f --- /dev/null +++ b/internal/tests/testdata/operatorShareSnapshots/generateExpectedResults.sql @@ -0,0 +1,47 @@ +COPY ( + WITH operator_shares as ( + select * + FROM dbt_testnet_holesky_rewards.operator_shares + where block_time < '2024-09-17' +), +ranked_operator_records as ( + SELECT *, + ROW_NUMBER() OVER (PARTITION BY operator, strategy, cast(block_time AS DATE) ORDER BY block_time DESC, log_index DESC) AS rn + FROM operator_shares +), + snapshotted_records as ( + SELECT + operator, + strategy, + shares, + block_time, + date_trunc('day', block_time) + INTERVAL '1' day as snapshot_time + from ranked_operator_records + where rn = 1 + ), + operator_share_windows as ( + SELECT + operator, strategy, shares, snapshot_time as start_time, + CASE + WHEN LEAD(snapshot_time) OVER (PARTITION BY operator, strategy ORDER BY snapshot_time) is null THEN date_trunc('day', TIMESTAMP '2024-09-01') + ELSE LEAD(snapshot_time) OVER (PARTITION BY operator, strategy ORDER BY snapshot_time) + END AS end_time + FROM snapshotted_records + ), + cleaned_records as ( + SELECT * FROM operator_share_windows + WHERE start_time < end_time + ), + final_results as ( + SELECT + operator, + strategy, + shares::text, + cast(day AS DATE) AS snapshot + FROM + cleaned_records + CROSS JOIN + generate_series(DATE(start_time), DATE(end_time) - interval '1' day, interval '1' day) AS day + ) + select * from final_results +) TO STDOUT WITH DELIMITER ',' CSV HEADER diff --git a/internal/tests/testdata/stakerDelegationSnapshots/README.md b/internal/tests/testdata/stakerDelegationSnapshots/README.md new file mode 100644 index 00000000..bd1609a8 --- /dev/null +++ b/internal/tests/testdata/stakerDelegationSnapshots/README.md @@ -0,0 +1,22 @@ +## Source + +```sql +SELECT + staker, + operator, + log_index, + block_number, + case when src = 'undelegations' THEN false ELSE true END AS delegated +FROM ( + SELECT *, 'undelegations' AS src FROM dbt_testnet_holesky_rewards.staker_undelegations + UNION ALL + SELECT *, 'delegations' AS src FROM dbt_testnet_holesky_rewards.staker_delegations + ) as delegations_combined +where block_time < '2024-09-17' + +``` + + +```bash +psql --host localhost --port 5435 --user blocklake --dbname blocklake --password < internal/tests/testdata/stakerDelegationSnapshots/generateExpectedResults.sql > internal/tests/testdata/stakerDelegationSnapshots/expectedResults.csv +``` diff --git a/internal/tests/testdata/stakerDelegationSnapshots/generateExpectedResults.sql b/internal/tests/testdata/stakerDelegationSnapshots/generateExpectedResults.sql new file mode 100644 index 00000000..bfee6a53 --- /dev/null +++ b/internal/tests/testdata/stakerDelegationSnapshots/generateExpectedResults.sql @@ -0,0 +1,47 @@ +COPY ( + with delegated_stakers as ( + select + * + from dbt_testnet_holesky_rewards.staker_delegation_status + where block_time < '2024-09-17' +), +ranked_delegations as ( + SELECT *, + ROW_NUMBER() OVER (PARTITION BY staker, cast(block_time AS DATE) ORDER BY block_time DESC, log_index DESC) AS rn + FROM delegated_stakers +), + snapshotted_records as ( + SELECT + staker, + operator, + block_time, + date_trunc('day', block_time) + INTERVAL '1' day AS snapshot_time + from ranked_delegations + where rn = 1 + ), + staker_delegation_windows as ( + SELECT + staker, operator, snapshot_time as start_time, + CASE + -- If the range does not have the end, use the cutoff date truncated to 0 UTC + WHEN LEAD(snapshot_time) OVER (PARTITION BY staker ORDER BY snapshot_time) is null THEN date_trunc('day', TIMESTAMP '2024-09-01') + ELSE LEAD(snapshot_time) OVER (PARTITION BY staker ORDER BY snapshot_time) + END AS end_time + FROM snapshotted_records + ), +cleaned_records as ( + SELECT * FROM staker_delegation_windows + WHERE start_time < end_time +), +final_results as ( + SELECT + staker, + operator, + cast(day AS DATE) AS snapshot +FROM + cleaned_records + CROSS JOIN + generate_series(DATE(start_time), DATE(end_time) - interval '1' day, interval '1' day) AS day +) +select * from final_results +) TO STDOUT WITH DELIMITER ',' CSV HEADER; diff --git a/internal/tests/testdata/stakerShareSnapshots/README.md b/internal/tests/testdata/stakerShareSnapshots/README.md new file mode 100644 index 00000000..b92c0b6d --- /dev/null +++ b/internal/tests/testdata/stakerShareSnapshots/README.md @@ -0,0 +1,15 @@ +## Source data + +```sql +select + staker, + strategy, + block_number, + sum(shares)::TEXT as shares +from dbt_testnet_holesky_rewards.staker_shares +group by 1, 2, 3 +``` + +## Expected results + +_See `generateExpectedResults.sql`_ diff --git a/internal/tests/testdata/stakerShareSnapshots/generateExpectedResults.sql b/internal/tests/testdata/stakerShareSnapshots/generateExpectedResults.sql new file mode 100644 index 00000000..f766612f --- /dev/null +++ b/internal/tests/testdata/stakerShareSnapshots/generateExpectedResults.sql @@ -0,0 +1,48 @@ +COPY ( + with staker_shares as ( + select + * + from dbt_testnet_holesky_rewards.staker_shares + where block_time < '2024-09-17' +), +ranked_staker_records as ( + SELECT *, + ROW_NUMBER() OVER (PARTITION BY staker, strategy, cast(block_time AS DATE) ORDER BY block_time DESC, log_index DESC) AS rn + FROM staker_shares +), + snapshotted_records as ( + SELECT + staker, + strategy, + shares, + block_time, + date_trunc('day', block_time) + INTERVAL '1' day AS snapshot_time + from ranked_staker_records + where rn = 1 + ), + staker_share_windows as ( + SELECT + staker, strategy, shares, snapshot_time as start_time, + CASE + WHEN LEAD(snapshot_time) OVER (PARTITION BY staker, strategy ORDER BY snapshot_time) is null THEN date_trunc('day', TIMESTAMP '2024-09-01') + ELSE LEAD(snapshot_time) OVER (PARTITION BY staker, strategy ORDER BY snapshot_time) + END AS end_time + FROM snapshotted_records + ), + cleaned_records as ( + SELECT * FROM staker_share_windows + WHERE start_time < end_time +), +final_results as ( + SELECT + staker, + strategy, + shares::text, + cast(day AS DATE) AS snapshot + FROM + cleaned_records + CROSS JOIN + generate_series(DATE(start_time), DATE(end_time) - interval '1' day, interval '1' day) AS day + ) +SELECT * from final_results +) TO STDOUT WITH DELIMITER ',' CSV HEADER diff --git a/internal/tests/utils.go b/internal/tests/utils.go index b4b8af01..c217e76b 100644 --- a/internal/tests/utils.go +++ b/internal/tests/utils.go @@ -1,27 +1,55 @@ package tests import ( - "os" - + "fmt" "github.com/Layr-Labs/go-sidecar/internal/config" sqlite2 "github.com/Layr-Labs/go-sidecar/internal/sqlite" + "github.com/gocarina/gocsv" + "github.com/google/uuid" + "go.uber.org/zap" "gorm.io/gorm" + "os" + "path/filepath" + "strings" ) func GetConfig() *config.Config { return config.NewConfig() } -const sqliteInMemoryPath = "file::memory:?cache=shared" - -func GetSqliteDatabaseConnection() (*gorm.DB, error) { - db, err := sqlite2.NewGormSqliteFromSqlite(sqlite2.NewSqlite(sqliteInMemoryPath)) +func GetInMemorySqliteDatabaseConnection(l *zap.Logger) (*gorm.DB, error) { + db, err := sqlite2.NewGormSqliteFromSqlite(sqlite2.NewSqlite(sqlite2.SqliteInMemoryPath, l)) if err != nil { panic(err) } return db, nil } +func GetFileBasedSqliteDatabaseConnection(l *zap.Logger) (string, *gorm.DB, error) { + fileName, err := uuid.NewRandom() + if err != nil { + panic(err) + } + basePath := fmt.Sprintf("%s%s", os.TempDir(), fileName) + if err := os.MkdirAll(basePath, os.ModePerm); err != nil { + return "", nil, err + } + + filePath := fmt.Sprintf("%s/test.db", basePath) + fmt.Printf("File path: %s\n", filePath) + db, err := sqlite2.NewGormSqliteFromSqlite(sqlite2.NewSqlite(filePath, l)) + if err != nil { + panic(err) + } + return filePath, db, nil +} + +func DeleteTestSqliteDB(filePath string) { + if err := os.Remove(filePath); err != nil { + panic(err) + } +} + func ReplaceEnv(newValues map[string]string, previousValues *map[string]string) { for k, v := range newValues { (*previousValues)[k] = os.Getenv(k) @@ -34,3 +62,132 @@ func RestoreEnv(previousValues map[string]string) { os.Setenv(k, v) } } + +func getTestdataPathFromProjectRoot(projectRoot string, fileName string) string { + p, err := filepath.Abs(fmt.Sprintf("%s/internal/tests/testdata%s", projectRoot, fileName)) + if err != nil { + panic(err) + } + return p +} + +func getSqlFile(filePath string) (string, error) { + contents, err := os.ReadFile(filePath) + + if err != nil { + return "", err + } + + return strings.Trim(string(contents), "\n"), nil +} +func getExpectedResultsCsvFile[T any](filePath string) ([]*T, error) { + results := make([]*T, 0) + file, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer file.Close() + + if err := gocsv.UnmarshalFile(file, &results); err != nil { + panic(err) + } + return results, nil +} + +func GetAllBlocksSqlFile(projectBase string) (string, error) { + path := getTestdataPathFromProjectRoot(projectBase, "/allBlocks.sql") + fmt.Printf("Path: %v\n", path) + return getSqlFile(path) +} + +func GetOperatorAvsRegistrationsSqlFile(projectBase string) (string, error) { + path := getTestdataPathFromProjectRoot(projectBase, "/operatorAvsRegistrationSnapshots/operatorAvsRegistrations.sql") + return getSqlFile(path) +} + +type ExpectedOperatorAvsRegistrationSnapshot struct { + Operator string `csv:"operator"` + Avs string `csv:"avs"` + Snapshot string `csv:"snapshot"` +} + +func GetExpectedOperatorAvsSnapshotResults(projectBase string) ([]*ExpectedOperatorAvsRegistrationSnapshot, error) { + path := getTestdataPathFromProjectRoot(projectBase, "/operatorAvsRegistrationSnapshots/expectedResults.csv") + return getExpectedResultsCsvFile[ExpectedOperatorAvsRegistrationSnapshot](path) +} + +func GetOperatorAvsRestakedStrategiesSqlFile(projectBase string) (string, error) { + path := getTestdataPathFromProjectRoot(projectBase, "/operatorRestakedStrategies/operatorRestakedStrategies.sql") + return getSqlFile(path) +} + +type ExpectedOperatorAvsSnapshot struct { + Operator string `csv:"operator"` + Avs string `csv:"avs"` + Strategy string `csv:"strategy"` + Snapshot string `csv:"snapshot"` +} + +func GetExpectedOperatorAvsSnapshots(projectBase string) ([]*ExpectedOperatorAvsSnapshot, error) { + path := getTestdataPathFromProjectRoot(projectBase, "/operatorRestakedStrategies/expectedResults.csv") + return getExpectedResultsCsvFile[ExpectedOperatorAvsSnapshot](path) +} + +// OperatorShares snapshots +func GetOperatorSharesSqlFile(projectBase string) (string, error) { + path := getTestdataPathFromProjectRoot(projectBase, "/operatorShareSnapshots/operatorShares.sql") + return getSqlFile(path) +} + +type OperatorShareExpectedResult struct { + Operator string `csv:"operator"` + Strategy string `csv:"strategy"` + Snapshot string `csv:"snapshot"` + Shares string `csv:"shares"` +} + +func GetOperatorSharesExpectedResults(projectBase string) ([]*OperatorShareExpectedResult, error) { + path := getTestdataPathFromProjectRoot(projectBase, "/operatorShareSnapshots/expectedResults.csv") + return getExpectedResultsCsvFile[OperatorShareExpectedResult](path) +} + +// StakerShareSnapshots +func GetStakerSharesSqlFile(projectBase string) (string, error) { + path := getTestdataPathFromProjectRoot(projectBase, "/stakerShareSnapshots/stakerShares.sql") + return getSqlFile(path) +} + +type StakerShareExpectedResult struct { + Staker string `csv:"staker"` + Strategy string `csv:"strategy"` + Snapshot string `csv:"snapshot"` + Shares string `csv:"shares"` +} + +func GetStakerSharesExpectedResults(projectBase string) ([]*StakerShareExpectedResult, error) { + path := getTestdataPathFromProjectRoot(projectBase, "/stakerShareSnapshots/expectedResults.csv") + return getExpectedResultsCsvFile[StakerShareExpectedResult](path) +} + +// StakerDelegationSnapshots +func GetStakerDelegationsSqlFile(projectBase string) (string, error) { + path := getTestdataPathFromProjectRoot(projectBase, "/stakerDelegationSnapshots/stakerDelegations.sql") + return getSqlFile(path) +} + +type StakerDelegationExpectedResult struct { + Staker string `csv:"staker"` + Operator string `csv:"operator"` + Snapshot string `csv:"snapshot"` +} + +func GetStakerDelegationExpectedResults(projectBase string) ([]*StakerDelegationExpectedResult, error) { + path := getTestdataPathFromProjectRoot(projectBase, "/stakerDelegationSnapshots/expectedResults.csv") + return getExpectedResultsCsvFile[StakerDelegationExpectedResult](path) +} + +// CombinedRewards +func GetCombinedRewardsSqlFile(projectBase string) (string, error) { + path := getTestdataPathFromProjectRoot(projectBase, "/combinedRewards/combinedRewards.sql") + return getSqlFile(path) +} diff --git a/internal/types/numbers/numbers.go b/internal/types/numbers/numbers.go index 10e481f9..95e9e5b5 100644 --- a/internal/types/numbers/numbers.go +++ b/internal/types/numbers/numbers.go @@ -1,6 +1,11 @@ package numbers -import "math/big" +import "C" +import ( + "fmt" + "github.com/shopspring/decimal" + "math/big" +) // NewBig257 returns a new big.Int with a size of 257 bits // This allows us to fully support math on uint256 numbers @@ -8,3 +13,100 @@ import "math/big" func NewBig257() *big.Int { return big.NewInt(257) } + +// NumericMultiply take two huge numbers, stored as strings, and multiplies them +func NumericMultiply(a, b string) (string, error) { + na, err := decimal.NewFromString(a) + if err != nil { + return "", err + } + nb, err := decimal.NewFromString(b) + if err != nil { + return "", err + } + + return na.Mul(nb).String(), nil +} + +func SubtractBig(a, b string) (string, error) { + na, err := decimal.NewFromString(a) + if err != nil { + return "", err + } + nb, err := decimal.NewFromString(b) + if err != nil { + return "", err + } + + return na.Sub(nb).String(), nil +} + +func BigGreaterThan(a, b string) (bool, error) { + na, err := decimal.NewFromString(a) + if err != nil { + return false, err + } + nb, err := decimal.NewFromString(b) + if err != nil { + return false, err + } + + return na.GreaterThan(nb), nil +} + +// CalcRawTokensPerDay calculates the raw tokens per day for a given amount and duration +// Returns the raw tokens per day in decimal format as a string +func CalcRawTokensPerDay(amountStr string, duration uint64) (string, error) { + amount, err := decimal.NewFromString(amountStr) + if err != nil { + fmt.Printf("CalcRawTokensPerDay Error: %s\n", err) + return "", err + } + + rawTokensPerDay := amount.Div(decimal.NewFromFloat(float64(duration) / 86400)) + + return rawTokensPerDay.String(), nil +} + +// PostNileTokensPerDay calculates the tokens per day for post-nile rewards +// Simply truncates the decimal portion of the of the raw tokens per day +func PostNileTokensPerDay(tokensPerDay string) (string, error) { + fmt.Printf("PostNileTokensPerDay: %s\n", tokensPerDay) + tpd, err := decimal.NewFromString(tokensPerDay) + if err != nil { + fmt.Printf("PostNileTokensPerDay Error: %s\n", err) + return "", err + } + + return tpd.BigInt().String(), nil +} + +// CalculateStakerProportion calculates the staker weight for a given staker and total weight +func CalculateStakerProportion(stakerWeightStr string, totalWeightStr string) (string, error) { + stakerWeight, err := decimal.NewFromString(stakerWeightStr) + if err != nil { + return "", err + } + totalWeight, err := decimal.NewFromString(totalWeightStr) + if err != nil { + return "", err + } + + res := ((stakerWeight.Div(totalWeight)).Mul(decimal.NewFromInt(1000000000000000))). + Div(decimal.NewFromInt(1000000000000000)). + Floor() + return res.String(), nil +} + +func CalculateStakerWeight(multiplier string, shares string) (string, error) { + m, err := decimal.NewFromString(multiplier) + if err != nil { + return "", err + } + s, err := decimal.NewFromString(shares) + if err != nil { + return "", err + } + + return m.Mul(s).String(), nil +} diff --git a/internal/types/numbers/numbersData_test.go b/internal/types/numbers/numbersData_test.go new file mode 100644 index 00000000..3fcc00c2 --- /dev/null +++ b/internal/types/numbers/numbersData_test.go @@ -0,0 +1,559 @@ +package numbers + +var ( + operatorAmazonTokens = [][]string{ + []string{"2262498437750000", "226249843775000"}, + []string{"1077804257000000", "107780425700000"}, + []string{"1200335945500000", "120033594550000"}, + []string{"2579821510500000", "257982151050000"}, + []string{"1077804257000000", "107780425700000"}, + []string{"97982205000000", "9798220500000"}, + []string{"2731616159750000", "273161615975000"}, + []string{"1468013852500000", "146801385250000"}, + []string{"1460746763500000", "146074676350000"}, + []string{"97982205000000", "9798220500000"}, + []string{"2160308386500000", "216030838650000"}, + []string{"2173632496000000", "217363249600000"}, + []string{"979822051749999", "97982205175000"}, + []string{"1077804257000000", "107780425700000"}, + []string{"97982205000000", "9798220500000"}, + []string{"1205181123750000", "120518112375000"}, + []string{"1692326232750000", "169232623275000"}, + []string{"19596441000000", "1959644100000"}, + []string{"1077804257000000", "107780425700000"}, + []string{"2766886967000000", "276688696700000"}, + []string{"1421109574750000", "142110957475000"}, + []string{"1327025256750000", "132702525675000"}, + []string{"1513152979750000", "151315297975000"}, + []string{"1724319284250000", "172431928425000"}, + []string{"1881072278500000", "188107227850000"}, + []string{"1077804257000000", "107780425700000"}, + []string{"1077804257000000", "107780425700000"}, + []string{"2767046562250000", "276704656225000"}, + []string{"1771544047250000", "177154404725000"}, + []string{"979822051749999", "97982205175000"}, + []string{"2766618839500000", "276661883950000"}, + []string{"1111499405000000", "111149940500000"}, + []string{"1256537372500000", "125653737250000"}, + []string{"1311309356750000", "131130935675000"}, + []string{"1077804257000000", "107780425700000"}, + []string{"1476639073750000", "147663907375000"}, + []string{"619368817749999", "61936881775000"}, + []string{"2351572924250000", "235157292425000"}, + []string{"1077804257000000", "107780425700000"}, + []string{"38824366794750000", "3882436679475000"}, + []string{"1329923441750000", "132992344175000"}, + []string{"1077804257000000", "107780425700000"}, + []string{"1171094469500000", "117109446950000"}, + []string{"1077804257000000", "107780425700000"}, + []string{"2164538730000000", "216453873000000"}, + []string{"2094265522500000", "209426552250000"}, + []string{"1448972310750000", "144897231075000"}, + []string{"1077804257000000", "107780425700000"}, + []string{"1403751361500000", "140375136150000"}, + []string{"979822051749999", "97982205175000"}, + } + + operatorNileTokens = [][]string{ + []string{"2262498437749998", "226249843775000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1200335945499999", "120033594550000"}, + []string{"2579821510499997", "257982151050000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"97982205000000", "9798220500000"}, + []string{"2731616159749997", "273161615975000"}, + []string{"1468013852499999", "146801385250000"}, + []string{"1460746763499999", "146074676350000"}, + []string{"97982205000000", "9798220500000"}, + []string{"2160308386499998", "216030838650000"}, + []string{"2173632495999998", "217363249600000"}, + []string{"979822051749999", "97982205175000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"97982205000000", "9798220500000"}, + []string{"1205181123749999", "120518112375000"}, + []string{"1692326232749998", "169232623275000"}, + []string{"19596441000000", "1959644100000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2766886966999997", "276688696700000"}, + []string{"1421109574749999", "142110957475000"}, + []string{"1327025256749999", "132702525675000"}, + []string{"1513152979749999", "151315297975000"}, + []string{"1724319284249998", "172431928425000"}, + []string{"1881072278499998", "188107227850000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2767046562249997", "276704656225000"}, + []string{"1771544047249998", "177154404725000"}, + []string{"979822051749999", "97982205175000"}, + []string{"2766618839499997", "276661883950000"}, + []string{"1111499404999999", "111149940500000"}, + []string{"1256537372499999", "125653737250000"}, + []string{"1311309356749999", "131130935675000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1476639073749999", "147663907375000"}, + []string{"619368817749999", "61936881775000"}, + []string{"2351572924249998", "235157292425000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"38824366794749960", "3882436679474996"}, + []string{"1329923441749999", "132992344175000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1171094469499999", "117109446950000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2164538729999998", "216453873000000"}, + []string{"2094265522499998", "209426552250000"}, + []string{"1448972310749999", "144897231075000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1403751361499999", "140375136150000"}, + []string{"979822051749999", "97982205175000"}, + []string{"979822051749999", "97982205175000"}, + []string{"1085802969749999", "108580296975000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1124308426749999", "112430842675000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1182901487499999", "118290148750000"}, + []string{"1372466011999999", "137246601200000"}, + []string{"979822051749999", "97982205175000"}, + []string{"1120320939499999", "112032093950000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2813743578249997", "281374357825000"}, + []string{"1997209544499998", "199720954450000"}, + []string{"342937718000000", "34293771800000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2504518724749998", "250451872475000"}, + []string{"489911025749999", "48991102575000"}, + []string{"1169357997749999", "116935799775000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1263158553749999", "126315855375000"}, + []string{"1094640640499999", "109464064050000"}, + []string{"1284065770249999", "128406577025000"}, + []string{"1521653686749998", "152165368675000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1695020089999998", "169502009000000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1089590665249999", "108959066525000"}, + []string{"4333353594249996", "433335359425000"}, + []string{"979822051749999", "97982205175000"}, + []string{"2766860861499997", "276686086150000"}, + []string{"3737419616999996", "373741961700000"}, + []string{"1682669036499998", "168266903650000"}, + []string{"979822051749999", "97982205175000"}, + []string{"718515954363749200", "71851595436374920"}, + []string{"1077804256999999", "107780425700000"}, + []string{"3350684550249997", "335068455025000"}, + []string{"979822051749999", "97982205175000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2766670760499997", "276667076050000"}, + []string{"1139779232999999", "113977923300000"}, + []string{"1294703182249999", "129470318225000"}, + []string{"1401943427499999", "140194342750000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"8661942075499991", "866194207549999"}, + []string{"979822051749999", "97982205175000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"979822051749999", "97982205175000"}, + []string{"9798220500000", "979822050000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1329616345749999", "132961634575000"}, + []string{"2285721732999998", "228572173300000"}, + []string{"440830750000", "44083075000"}, + []string{"1309869932249999", "130986993225000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2767143131249997", "276714313125000"}, + []string{"1087449895749999", "108744989575000"}, + []string{"1440582461749999", "144058246175000"}, + []string{"1437596903499999", "143759690350000"}, + []string{"3739760468499996", "373976046850000"}, + []string{"1228930342749999", "122893034275000"}, + []string{"1668502454499998", "166850245450000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"20400709468749976", "2040070946874998"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1501152824749998", "150115282475000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1588788984249999", "158878898425000"}, + []string{"97982205000000", "9798220500000"}, + []string{"2766955102999997", "276695510300000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1213777201499999", "121377720150000"}, + []string{"2766522236999998", "276652223700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1518724180249999", "151872418025000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"102040547000000", "10204054700000"}, + []string{"1521663646499999", "152166364650000"}, + []string{"2766883311999998", "276688331200000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1239913664499999", "123991366450000"}, + []string{"1083648329249999", "108364832925000"}, + []string{"2766567664499997", "276656766450000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"121497934250000", "12149793425000"}, + []string{"1489329518749999", "148932951875000"}, + []string{"705471877249999", "70547187725000"}, + []string{"1365074446999999", "136507444700000"}, + []string{"148259503000000", "14825950300000"}, + []string{"120582017500000", "12058201750000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"3458772690249997", "345877269025000"}, + []string{"1096846600499999", "109684660050000"}, + []string{"2642116312499998", "264211631250000"}, + []string{"1186049670749999", "118604967075000"}, + []string{"3379361777249997", "337936177725000"}, + []string{"1073027971999999", "107302797200000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2205003116999998", "220500311700000"}, + []string{"1400476333249999", "140047633325000"}, + []string{"1334754243499999", "133475424350000"}, + []string{"75000261135249920", "7500026113524992"}, + []string{"2766917160499997", "276691716050000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"4081356354749996", "408135635475000"}, + []string{"115064965500000", "11506496550000"}, + []string{"140863986000000", "14086398600000"}, + []string{"1421544889999999", "142154489000000"}, + []string{"1249751425499999", "124975142550000"}, + []string{"1522711647499999", "152271164750000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2766928990999997", "276692899100000"}, + []string{"1251222885499999", "125122288550000"}, + []string{"1804670182999998", "180467018300000"}, + []string{"1286074936749999", "128607493675000"}, + []string{"22474595357499976", "2247459535749998"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2766772303499997", "276677230350000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1636477811999998", "163647781200000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2766759761749998", "276675976175000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2367891235249998", "236789123525000"}, + []string{"2111355676249998", "211135567625000"}, + []string{"97982205000000", "9798220500000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1335070157749999", "133507015775000"}, + []string{"1088824402499999", "108882440250000"}, + []string{"1591863078499998", "159186307850000"}, + []string{"1093209052249999", "109320905225000"}, + []string{"1231725361499999", "123172536150000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1188849744499999", "118884974450000"}, + []string{"1325974469749999", "132597446975000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1170727959499999", "117072795950000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2766687807749997", "276668780775000"}, + []string{"979822051749999", "97982205175000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1195382903249999", "119538290325000"}, + []string{"131740039000000", "13174003900000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1584454986249999", "158445498625000"}, + []string{"1496388154999999", "149638815500000"}, + []string{"2783380170999997", "278338017100000"}, + []string{"2766853466749997", "276685346675000"}, + []string{"97982205000000", "9798220500000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1259437513249999", "125943751325000"}, + []string{"2559660094249998", "255966009425000"}, + []string{"97982205000000", "9798220500000"}, + []string{"97982205000000", "9798220500000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"7828778194249992", "782877819424999"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1138055195249999", "113805519525000"}, + []string{"979822051749999", "97982205175000"}, + []string{"1192936111999999", "119293611200000"}, + []string{"1117604526499999", "111760452650000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"391928820500000", "39192882050000"}, + []string{"1671343943749998", "167134394375000"}, + []string{"1237619152499999", "123761915250000"}, + []string{"1461227295249999", "146122729525000"}, + []string{"536422250000", "53642225000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"746783849749999", "74678384975000"}, + []string{"8686889065749991", "868688906574999"}, + []string{"979822051749999", "97982205175000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"3577907333749997", "357790733375000"}, + []string{"9798220500000", "979822050000"}, + []string{"2766688656499997", "276668865650000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1190360713999999", "119036071400000"}, + []string{"1372170432249999", "137217043225000"}, + []string{"1180097737749999", "118009773775000"}, + []string{"979822051749999", "97982205175000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1231665739249999", "123166573925000"}, + []string{"1250928951499999", "125092895150000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1122446018749999", "112244601875000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"3465020847749997", "346502084775000"}, + []string{"497502843250000", "49750284325000"}, + []string{"2766694622499997", "276669462250000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"9799200250000", "979920025000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"3393183871249997", "339318387125000"}, + []string{"1224777564749999", "122477756475000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2766784225749997", "276678422575000"}, + []string{"1084088666249999", "108408866625000"}, + []string{"979822051749999", "97982205175000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"120671798250000", "12067179825000"}, + []string{"97982205000000", "9798220500000"}, + []string{"1207335820749999", "120733582075000"}, + []string{"979822051749999", "97982205175000"}, + []string{"1708717776499998", "170871777650000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2171666879249998", "217166687925000"}, + []string{"2766540531749997", "276654053175000"}, + []string{"97982205000000", "9798220500000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1139514050249999", "113951405025000"}, + []string{"1347136987749999", "134713698775000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2373897634749998", "237389763475000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1296568008249999", "129656800825000"}, + []string{"1747592574499998", "174759257450000"}, + []string{"1718446376999998", "171844637700000"}, + []string{"97982205000000", "9798220500000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1126212608249999", "112621260825000"}, + []string{"1382620828249999", "138262082825000"}, + []string{"1827529957249998", "182752995725000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2766997874749997", "276699787475000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"3558586463999997", "355858646400000"}, + []string{"148139296000000", "14813929600000"}, + []string{"1228361717249999", "122836171725000"}, + []string{"1120021586249999", "112002158625000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"75838000000", "7583800000"}, + []string{"2813176036999997", "281317603700000"}, + []string{"1126418676499999", "112641867650000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1168952628749999", "116895262875000"}, + []string{"1286002109999999", "128600211000000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1106691471749999", "110669147175000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2767147036499997", "276714703650000"}, + []string{"2766894477499997", "276689447750000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1221210601249999", "122121060125000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1474944833499998", "147494483350000"}, + []string{"3355500944749997", "335550094475000"}, + []string{"36042871697249960", "3604287169724996"}, + []string{"1077804256999999", "107780425700000"}, + []string{"103861137250000", "10386113725000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"7982080906749991", "798208090674999"}, + []string{"1077804256999999", "107780425700000"}, + []string{"6600292500000", "660029250000"}, + []string{"158317476750000", "15831747675000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1266130821999999", "126613082200000"}, + []string{"1087461430249999", "108746143025000"}, + []string{"1925894882749998", "192589488275000"}, + []string{"109489270750000", "10948927075000"}, + []string{"3011552248749997", "301155224875000"}, + []string{"5302500000", "530250000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1295497476749999", "129549747675000"}, + []string{"1514257759249998", "151425775925000"}, + []string{"1973412247749998", "197341224775000"}, + []string{"2612205589999997", "261220559000000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1215310149499999", "121531014950000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1087895020499999", "108789502050000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"147078692250000", "14707869225000"}, + []string{"2464874880999998", "246487488100000"}, + []string{"1166980098249999", "116698009825000"}, + []string{"734866538749999", "73486653875000"}, + []string{"2163704354999998", "216370435500000"}, + []string{"1281449592249999", "128144959225000"}, + []string{"1198409126249999", "119840912625000"}, + []string{"2767120898499997", "276712089850000"}, + []string{"1959088190749998", "195908819075000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1170951049999999", "117095105000000"}, + []string{"1154675041749999", "115467504175000"}, + []string{"1134384833499999", "113438483350000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1194101009249999", "119410100925000"}, + []string{"1388429900249999", "138842990025000"}, + []string{"979822051749999", "97982205175000"}, + []string{"1122446018749999", "112244601875000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1731852292499998", "173185229250000"}, + []string{"1396115605249999", "139611560525000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1081755392249999", "108175539225000"}, + []string{"1115170505749999", "111517050575000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1880982576249998", "188098257625000"}, + []string{"1253732266249999", "125373226625000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"117578912750000", "11757891275000"}, + []string{"2766816535749997", "276681653575000"}, + []string{"1151717784499999", "115171778450000"}, + []string{"1073440472499999", "107344047250000"}, + []string{"1842220677249998", "184222067725000"}, + []string{"1444291238499999", "144429123850000"}, + []string{"185920687750000", "18592068775000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2766739048249997", "276673904825000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1111127251249999", "111112725125000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2495212757249998", "249521275725000"}, + []string{"1109533228999999", "110953322900000"}, + []string{"1204528088999999", "120452808900000"}, + []string{"2767240998749997", "276724099875000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"27022701521749972", "2702270152174997"}, + []string{"1133721705999999", "113372170600000"}, + []string{"2766637369749997", "276663736975000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"979822051749999", "97982205175000"}, + []string{"861929137749999", "86192913775000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"3503727170499996", "350372717050000"}, + []string{"1060787189499999", "106078718950000"}, + []string{"979822051749999", "97982205175000"}, + []string{"2045249932249998", "204524993225000"}, + []string{"482354676250000", "48235467625000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1397360907249999", "139736090725000"}, + []string{"1175341173499999", "117534117350000"}, + []string{"2767044838249997", "276704483825000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1449542205249999", "144954220525000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2321707502749998", "232170750275000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1513531123249999", "151353112325000"}, + []string{"2740460193499997", "274046019350000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1149339404999999", "114933940500000"}, + []string{"1151809466249999", "115180946625000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1249904385249999", "124990438525000"}, + []string{"1096106635999999", "109610663600000"}, + []string{"6898005302249993", "689800530224999"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2766392857999997", "276639285800000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2921368330249997", "292136833025000"}, + []string{"1369704807999999", "136970480800000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"5006256564499995", "500625656450000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"109740069750000", "10974006975000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1346236941999999", "134623694200000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1103601738249999", "110360173825000"}, + []string{"2704308862999998", "270430886300000"}, + []string{"864763585499999", "86476358550000"}, + []string{"979822051749999", "97982205175000"}, + []string{"1576305397999998", "157630539800000"}, + []string{"147299275500000", "14729927550000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1321185449499999", "132118544950000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1244374005749999", "124437400575000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"979822051749999", "97982205175000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"2175404280999998", "217540428100000"}, + []string{"2324262255249998", "232426225525000"}, + []string{"1087602477499999", "108760247750000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1226466551749999", "122646655175000"}, + []string{"2797069106499997", "279706910650000"}, + []string{"1122461615499999", "112246161550000"}, + []string{"1156738466749999", "115673846675000"}, + []string{"5920104740249994", "592010474024999"}, + []string{"1077804256999999", "107780425700000"}, + []string{"1077804256999999", "107780425700000"}, + []string{"13604498250000", "1360449825000"}, + []string{"1367625821749999", "136762582175000"}, + } +) diff --git a/internal/types/numbers/numbers_test.go b/internal/types/numbers/numbers_test.go index 681a0f88..7e7cb82d 100644 --- a/internal/types/numbers/numbers_test.go +++ b/internal/types/numbers/numbers_test.go @@ -1,12 +1,49 @@ package numbers import ( + "fmt" + "github.com/shopspring/decimal" "github.com/stretchr/testify/assert" + "math" "math/big" "testing" ) -func Test_numbers(t *testing.T) { +func roundToEven(f *big.Float) *big.Float { + // Get the integer part of the float + intPart, _ := f.Int(nil) + + // Create a big.Float from the integer part + rounded := new(big.Float).SetInt(intPart) + + // Calculate the difference (fractional part) + diff := new(big.Float).Sub(f, rounded) + + // Create a big.Float representing 0.5 + half := big.NewFloat(0.5) + + // Check if the fractional part is exactly 0.5 + if diff.Cmp(half) == 0 { + // Check if the integer part is even + intPartMod := new(big.Int).Mod(intPart, big.NewInt(2)) + if intPartMod.Cmp(big.NewInt(0)) != 0 { + // If odd, round up + rounded.Add(rounded, big.NewFloat(1)) + } + } else if diff.Cmp(half) > 0 { + // If the fractional part is greater than 0.5, round up + rounded.Add(rounded, big.NewFloat(1)) + } + + return rounded +} + +func Test_Numbers(t *testing.T) { + if err := InitPython(); err != nil { + t.Error(err) + } + defer FinalizePython() + t.Run("Test that big.Int can produce negative numbers", func(t *testing.T) { startingNum := big.Int{} startingNum.SetString("10", 10) @@ -45,4 +82,778 @@ func Test_numbers(t *testing.T) { assert.Equal(t, "13389173345999999999999980", amountToSubtract.Sub(startingNum, amountToSubtract).String()) }) + t.Run("Test floating point math to see if its correct", func(t *testing.T) { + t.Skip("math/big is not accurate enough for this test") + duration := float64(6048000) + amountStr := "99999999999999999999999999999999999999" + expectedRawTokensPerDay := "1428571428571428571428571428571428571" + //expectedFloorValue := "1428571428571428571428571428571428571" + //expectedRoundedValue := "1428571428571427000000000000000000000" + // 1428571428571428500000000000000000000 + + amount, _ := NewBig257().SetString(amountStr, 10) + //amount, _, err := big.ParseFloat("99999999999999999999999999999999999999", 10, 38, big.ToPositiveInf) + //assert.Nil(t, err) + + fmt.Printf("Amount: %+v\n", amount.String()) + + amountFloat := big.NewFloat(0) + amountFloat.SetInt(amount) + amountFloat.SetMode(big.ToNearestEven) + //amountFloat.SetPrec(256) + + divisor := big.NewFloat(float64(duration) / 86400) + divisor.SetMode(big.ToNearestEven) + fmt.Printf("Divisor: %+v\n", divisor.String()) + tokensPerDay := amountFloat.Quo(amountFloat, divisor) + tokensPerDay = roundToEven(tokensPerDay) + + rawTokensPerDay := tokensPerDay.Text('f', 0) + assert.Equal(t, expectedRawTokensPerDay, rawTokensPerDay) + + // We use floor to ensure we are always underesimating total tokens per day + tokensPerDayFloored := NewBig257() + tokensPerDayFloored, _ = tokensPerDay.Int(tokensPerDayFloored) + + precision := (math.Pow(10, 15) - float64(1)) / math.Pow(10, 15) + + // Round down to 15 sigfigs for double precision, ensuring know errouneous round up or down + tokensPerDayDecimal := tokensPerDay.Mul(tokensPerDay, big.NewFloat(precision)) + + fmt.Printf("tokensPerDayFloored: %s\n", tokensPerDayFloored.String()) + fmt.Printf("tokensPerDayDecimal: %s\n", tokensPerDayDecimal.String()) + + }) + t.Run("Test floating point math to see if its correct with shopspring/decimal", func(t *testing.T) { + duration := float64(6048000) + amountStr := "99999999999999999999999999999999999999" + expectedRawTokensPerDay := "1428571428571428571428571428571428571" + expectedFloorValue := "1428571428571428571428571428571428571" + // expectedRoundedValue := "1428571428571427000000000000000000000" + // 1428571428571428500000000000000000000 + + amount, err := decimal.NewFromString(amountStr) + assert.Nil(t, err) + + fmt.Printf("Amount: %+v\n", amount.String()) + + rawTokensPerDay := amount.Div(decimal.NewFromFloat(duration / 86400)) + fmt.Printf("Raw tokens per day: %+v\n", rawTokensPerDay.String()) + // rawTokensPerDayForRounding := amount.Div(decimal.NewFromFloat(duration / 86400)) + + assert.Equal(t, expectedRawTokensPerDay, rawTokensPerDay.BigInt().String()) + + tokensPerDayFloored := rawTokensPerDay.BigInt() + + assert.Equal(t, expectedFloorValue, tokensPerDayFloored.String()) + + one := (decimal.NewFromInt(10).Pow(decimal.NewFromInt(15))).Sub(decimal.NewFromInt(1)) + two := decimal.NewFromInt(10).Pow(decimal.NewFromInt(15)) + + fmt.Printf("one/two :%v\n", one.Div(two).String()) + assert.Equal(t, "0.999999999999999", one.Div(two).String()) + + }) + + t.Run("Test CalcRawTokensPerDay", func(t *testing.T) { + amountStr := "99999999999999999999999999999999999999" + duration := uint64(6048000) + + amount, err := CalcRawTokensPerDay(amountStr, duration) + assert.Nil(t, err) + assert.Equal(t, "1428571428571428571428571428571428571.4142857142857143", amount) + }) + t.Run("Test PostNileTokensPerDay", func(t *testing.T) { + duration := uint64(6048000) + amountStr := "99999999999999999999999999999999999999" + expectedFloorValue := "1428571428571428571428571428571428571" + + amount, err := CalcRawTokensPerDay(amountStr, duration) + assert.Nil(t, err) + + amount, err = PostNileTokensPerDay(amount) + assert.Nil(t, err) + assert.Equal(t, expectedFloorValue, amount) + }) + t.Run("Test PreNileTokensPerDay", func(t *testing.T) { + duration := uint64(6048000) + amountStr := "99999999999999999999999999999999999999" + expectedRoundedValue := "1428571428571427000000000000000000000" + + amount, err := CalcRawTokensPerDay(amountStr, duration) + assert.Nil(t, err) + + amount, err = PreNileTokensPerDay(amount) + assert.Nil(t, err) + assert.Equal(t, expectedRoundedValue, amount) + }) + + t.Run("Staker tokens", func(t *testing.T) { + t.Run("Should correctly calculate the amazon token rewards for stakers", func(t *testing.T) { + values := [][]string{ + []string{"0.000009049993751000000000", "249999999999999740000", "2262498437750000"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804257000000"}, + []string{"0.000004801343782000000000", "249999999999999740000", "1200335945500000"}, + []string{"0.000010319286042000000000", "249999999999999740000", "2579821510500000"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804257000000"}, + []string{"0.000000391928820000000000", "249999999999999740000", "97982205000000"}, + []string{"0.000010926464639000000000", "249999999999999740000", "2731616159750000"}, + []string{"0.000005872055410000000000", "249999999999999740000", "1468013852500000"}, + []string{"0.000005842987054000000000", "249999999999999740000", "1460746763500000"}, + []string{"0.000000391928820000000000", "249999999999999740000", "97982205000000"}, + []string{"0.000008641233546000000000", "249999999999999740000", "2160308386500000"}, + []string{"0.000008694529984000000000", "249999999999999740000", "2173632496000000"}, + []string{"0.000003919288207000000000", "249999999999999740000", "979822051749999"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804257000000"}, + []string{"0.000000391928820000000000", "249999999999999740000", "97982205000000"}, + []string{"0.000004820724495000000000", "249999999999999740000", "1205181123750000"}, + []string{"0.000006769304931000000000", "249999999999999740000", "1692326232750000"}, + []string{"0.000000078385764000000000", "249999999999999740000", "19596441000000"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804257000000"}, + []string{"0.000011067547868000000000", "249999999999999740000", "2766886967000000"}, + []string{"0.000005684438299000000000", "249999999999999740000", "1421109574750000"}, + []string{"0.000005308101027000000000", "249999999999999740000", "1327025256750000"}, + []string{"0.000006052611919000000000", "249999999999999740000", "1513152979750000"}, + []string{"0.000006897277137000000000", "249999999999999740000", "1724319284250000"}, + []string{"0.000007524289114000000000", "249999999999999740000", "1881072278500000"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804257000000"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804257000000"}, + []string{"0.000011068186249000000000", "249999999999999740000", "2767046562250000"}, + []string{"0.000007086176189000000000", "249999999999999740000", "1771544047250000"}, + []string{"0.000003919288207000000000", "249999999999999740000", "979822051749999"}, + []string{"0.000011066475358000000000", "249999999999999740000", "2766618839500000"}, + []string{"0.000004445997620000000000", "249999999999999740000", "1111499405000000"}, + []string{"0.000005026149490000000000", "249999999999999740000", "1256537372500000"}, + []string{"0.000005245237427000000000", "249999999999999740000", "1311309356750000"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804257000000"}, + []string{"0.000005906556295000000000", "249999999999999740000", "1476639073750000"}, + []string{"0.000002477475271000000000", "249999999999999740000", "619368817749999"}, + []string{"0.000009406291697000000000", "249999999999999740000", "2351572924250000"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804257000000"}, + []string{"0.00015529746717900000", "249999999999999740000", "38824366794750000"}, + []string{"0.000005319693767000000000", "249999999999999740000", "1329923441750000"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804257000000"}, + []string{"0.000004684377878000000000", "249999999999999740000", "1171094469500000"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804257000000"}, + []string{"0.000008658154920000000000", "249999999999999740000", "2164538730000000"}, + []string{"0.000008377062090000000000", "249999999999999740000", "2094265522500000"}, + []string{"0.000005795889243000000000", "249999999999999740000", "1448972310750000"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804257000000"}, + []string{"0.000005615005446000000000", "249999999999999740000", "1403751361500000"}, + []string{"0.000003919288207000000000", "249999999999999740000", "979822051749999"}, + } + + for i, value := range values { + stakerProportion := value[0] + tokensPerDay := value[1] + expectedValue := value[2] + result, err := CalculateAmazonStakerTokenRewards(stakerProportion, tokensPerDay) + assert.Nil(t, err) + + assert.Equal(t, expectedValue, result, fmt.Sprintf("row %d: '%s' - '%s' - '%s'", i, stakerProportion, tokensPerDay, expectedValue)) + } + + }) + t.Run("Should correctly calculate the nile token rewards for stakers", func(t *testing.T) { + values := [][]string{ + []string{"0.000009049993751000000000", "249999999999999740000", "2262498437749998"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804256999999"}, + []string{"0.000004801343782000000000", "249999999999999740000", "1200335945499999"}, + []string{"0.000010319286042000000000", "249999999999999740000", "2579821510499997"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804256999999"}, + []string{"0.000000391928820000000000", "249999999999999740000", "97982205000000"}, + []string{"0.000010926464639000000000", "249999999999999740000", "2731616159749997"}, + []string{"0.000005872055410000000000", "249999999999999740000", "1468013852499999"}, + []string{"0.000005842987054000000000", "249999999999999740000", "1460746763499999"}, + []string{"0.000000391928820000000000", "249999999999999740000", "97982205000000"}, + []string{"0.000008641233546000000000", "249999999999999740000", "2160308386499998"}, + []string{"0.000008694529984000000000", "249999999999999740000", "2173632495999998"}, + []string{"0.000003919288207000000000", "249999999999999740000", "979822051749999"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804256999999"}, + []string{"0.000000391928820000000000", "249999999999999740000", "97982205000000"}, + []string{"0.000004820724495000000000", "249999999999999740000", "1205181123749999"}, + []string{"0.000006769304931000000000", "249999999999999740000", "1692326232749998"}, + []string{"0.000000078385764000000000", "249999999999999740000", "19596441000000"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804256999999"}, + []string{"0.000011067547868000000000", "249999999999999740000", "2766886966999997"}, + []string{"0.000005684438299000000000", "249999999999999740000", "1421109574749999"}, + []string{"0.000005308101027000000000", "249999999999999740000", "1327025256749999"}, + []string{"0.000006052611919000000000", "249999999999999740000", "1513152979749999"}, + []string{"0.000006897277137000000000", "249999999999999740000", "1724319284249998"}, + []string{"0.000007524289114000000000", "249999999999999740000", "1881072278499998"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804256999999"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804256999999"}, + []string{"0.000011068186249000000000", "249999999999999740000", "2767046562249997"}, + []string{"0.000007086176189000000000", "249999999999999740000", "1771544047249998"}, + []string{"0.000003919288207000000000", "249999999999999740000", "979822051749999"}, + []string{"0.000011066475358000000000", "249999999999999740000", "2766618839499997"}, + []string{"0.000004445997620000000000", "249999999999999740000", "1111499404999999"}, + []string{"0.000005026149490000000000", "249999999999999740000", "1256537372499999"}, + []string{"0.000005245237427000000000", "249999999999999740000", "1311309356749999"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804256999999"}, + []string{"0.000005906556295000000000", "249999999999999740000", "1476639073749999"}, + []string{"0.000002477475271000000000", "249999999999999740000", "619368817749999"}, + []string{"0.000009406291697000000000", "249999999999999740000", "2351572924249998"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804256999999"}, + []string{"0.00015529746717900000", "249999999999999740000", "38824366794749960"}, + []string{"0.000005319693767000000000", "249999999999999740000", "1329923441749999"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804256999999"}, + []string{"0.000004684377878000000000", "249999999999999740000", "1171094469499999"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804256999999"}, + []string{"0.000008658154920000000000", "249999999999999740000", "2164538729999998"}, + []string{"0.000008377062090000000000", "249999999999999740000", "2094265522499998"}, + []string{"0.000005795889243000000000", "249999999999999740000", "1448972310749999"}, + []string{"0.000004311217028000000000", "249999999999999740000", "1077804256999999"}, + []string{"0.000005615005446000000000", "249999999999999740000", "1403751361499999"}, + []string{"0.000003919288207000000000", "249999999999999740000", "979822051749999"}, + } + + for i, value := range values { + stakerProportion := value[0] + tokensPerDay := value[1] + expectedValue := value[2] + result, err := CalculateNileStakerTokenRewards(stakerProportion, tokensPerDay) + assert.Nil(t, err) + + assert.Equal(t, expectedValue, result, fmt.Sprintf("row %d: '%s' - '%s' - '%s'", i, stakerProportion, tokensPerDay, expectedValue)) + } + }) + t.Run("Should correctly calculate the post-nile token rewards for stakers", func(t *testing.T) { + values := [][]string{ + []string{"0.000009049993751", "249999999999999740000", "2262498437749997", "2262498437749997.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004801343782", "249999999999999740000", "1200335945499998", "1200335945499998.8"}, + []string{"0.000010319286042", "249999999999999740000", "2579821510499997", "2579821510499997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.00000039192882", "249999999999999740000", "97982204999999", "97982204999999.89"}, + []string{"0.000010926464639", "249999999999999740000", "2731616159749997", "2731616159749997"}, + []string{"0.00000587205541", "249999999999999740000", "1468013852499998", "1468013852499998.5"}, + []string{"0.000005842987054", "249999999999999740000", "1460746763499998", "1460746763499998.5"}, + []string{"0.00000039192882", "249999999999999740000", "97982204999999", "97982204999999.89"}, + []string{"0.000008641233546", "249999999999999740000", "2160308386499998", "2160308386499998"}, + []string{"0.000008694529984", "249999999999999740000", "2173632495999997", "2173632495999997.8"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.00000039192882", "249999999999999740000", "97982204999999", "97982204999999.89"}, + []string{"0.000004820724495", "249999999999999740000", "1205181123749998", "1205181123749998.8"}, + []string{"0.000006769304931", "249999999999999740000", "1692326232749998", "1692326232749998.2"}, + []string{"0.000000078385764", "249999999999999740000", "19596440999999", "19596440999999.98"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011067547868", "249999999999999740000", "2766886966999997", "2766886966999997"}, + []string{"0.000005684438299", "249999999999999740000", "1421109574749998", "1421109574749998.5"}, + []string{"0.000005308101027", "249999999999999740000", "1327025256749998", "1327025256749998.5"}, + []string{"0.000006052611919", "249999999999999740000", "1513152979749998", "1513152979749998.5"}, + []string{"0.000006897277137", "249999999999999740000", "1724319284249998", "1724319284249998.2"}, + []string{"0.000007524289114", "249999999999999740000", "1881072278499998", "1881072278499998"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011068186249", "249999999999999740000", "2767046562249997", "2767046562249997"}, + []string{"0.000007086176189", "249999999999999740000", "1771544047249998", "1771544047249998.2"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000011066475358", "249999999999999740000", "2766618839499997", "2766618839499997"}, + []string{"0.00000444599762", "249999999999999740000", "1111499404999998", "1111499404999998.8"}, + []string{"0.00000502614949", "249999999999999740000", "1256537372499998", "1256537372499998.8"}, + []string{"0.000005245237427", "249999999999999740000", "1311309356749998", "1311309356749998.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005906556295", "249999999999999740000", "1476639073749998", "1476639073749998.5"}, + []string{"0.000002477475271", "249999999999999740000", "619368817749999", "619368817749999.4"}, + []string{"0.000009406291697", "249999999999999740000", "2351572924249997", "2351572924249997.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000155297467179", "249999999999999740000", "38824366794749960", "38824366794749960"}, + []string{"0.000005319693767", "249999999999999740000", "1329923441749998", "1329923441749998.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004684377878", "249999999999999740000", "1171094469499998", "1171094469499998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.00000865815492", "249999999999999740000", "2164538729999997", "2164538729999997.5"}, + []string{"0.00000837706209", "249999999999999740000", "2094265522499997", "2094265522499997.8"}, + []string{"0.000005795889243", "249999999999999740000", "1448972310749998", "1448972310749998.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005615005446", "249999999999999740000", "1403751361499998", "1403751361499998.5"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000004343211879", "249999999999999740000", "1085802969749998", "1085802969749998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004497233707", "249999999999999740000", "1124308426749998", "1124308426749998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.00000473160595", "249999999999999740000", "1182901487499998", "1182901487499998.8"}, + []string{"0.000005489864048", "249999999999999740000", "1372466011999998", "1372466011999998.5"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000004481283758", "249999999999999740000", "1120320939499998", "1120320939499998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011254974313", "249999999999999740000", "2813743578249997", "2813743578249997"}, + []string{"0.000007988838178", "249999999999999740000", "1997209544499998", "1997209544499998"}, + []string{"0.000001371750872", "249999999999999740000", "342937717999999", "342937717999999.6"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000010018074899", "249999999999999740000", "2504518724749997", "2504518724749997.5"}, + []string{"0.000001959644103", "249999999999999740000", "489911025749999", "489911025749999.44"}, + []string{"0.000004677431991", "249999999999999740000", "1169357997749998", "1169357997749998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005052634215", "249999999999999740000", "1263158553749998", "1263158553749998.8"}, + []string{"0.000004378562562", "249999999999999740000", "1094640640499998", "1094640640499998.9"}, + []string{"0.000005136263081", "249999999999999740000", "1284065770249998", "1284065770249998.5"}, + []string{"0.000006086614747", "249999999999999740000", "1521653686749998", "1521653686749998.2"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.00000678008036", "249999999999999740000", "1695020089999998", "1695020089999998"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004358362661", "249999999999999740000", "1089590665249998", "1089590665249998.8"}, + []string{"0.000017333414377", "249999999999999740000", "4333353594249995", "4333353594249995.5"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000011067443446", "249999999999999740000", "2766860861499997", "2766860861499997"}, + []string{"0.000014949678468", "249999999999999740000", "3737419616999996", "3737419616999996"}, + []string{"0.000006730676146", "249999999999999740000", "1682669036499998", "1682669036499998.2"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.002874063817455", "249999999999999740000", "718515954363749250", "718515954363749250"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000013402738201", "249999999999999740000", "3350684550249996", "3350684550249996.5"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011066683042", "249999999999999740000", "2766670760499997", "2766670760499997"}, + []string{"0.000004559116932", "249999999999999740000", "1139779232999998", "1139779232999998.8"}, + []string{"0.000005178812729", "249999999999999740000", "1294703182249998", "1294703182249998.8"}, + []string{"0.00000560777371", "249999999999999740000", "1401943427499998", "1401943427499998.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000034647768302", "249999999999999740000", "8661942075499991", "8661942075499991"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000000039192882", "249999999999999740000", "9798220499999", "9798220499999.99"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005318465383", "249999999999999740000", "1329616345749998", "1329616345749998.8"}, + []string{"0.000009142886932", "249999999999999740000", "2285721732999997", "2285721732999997.5"}, + []string{"0.000000001763323", "249999999999999740000", "440830749999", "440830749999.9995"}, + []string{"0.000005239479729", "249999999999999740000", "1309869932249998", "1309869932249998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011068572525", "249999999999999740000", "2767143131249997", "2767143131249997"}, + []string{"0.000004349799583", "249999999999999740000", "1087449895749998", "1087449895749998.9"}, + []string{"0.000005762329847", "249999999999999740000", "1440582461749998", "1440582461749998.5"}, + []string{"0.000005750387614", "249999999999999740000", "1437596903499998", "1437596903499998.5"}, + []string{"0.000014959041874", "249999999999999740000", "3739760468499996", "3739760468499996"}, + []string{"0.000004915721371", "249999999999999740000", "1228930342749998", "1228930342749998.8"}, + []string{"0.000006674009818", "249999999999999740000", "1668502454499998", "1668502454499998.2"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000081602837875", "249999999999999740000", "20400709468749976", "20400709468749976"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000006004611299", "249999999999999740000", "1501152824749998", "1501152824749998.2"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000006355155937", "249999999999999740000", "1588788984249998", "1588788984249998.5"}, + []string{"0.00000039192882", "249999999999999740000", "97982204999999", "97982204999999.89"}, + []string{"0.000011067820412", "249999999999999740000", "2766955102999997", "2766955102999997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004855108806", "249999999999999740000", "1213777201499998", "1213777201499998.8"}, + []string{"0.000011066088948", "249999999999999740000", "2766522236999997", "2766522236999997.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000006074896721", "249999999999999740000", "1518724180249998", "1518724180249998.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000000408162188", "249999999999999740000", "102040546999999", "102040546999999.89"}, + []string{"0.000006086654586", "249999999999999740000", "1521663646499998", "1521663646499998.5"}, + []string{"0.000011067533248", "249999999999999740000", "2766883311999997", "2766883311999997.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004959654658", "249999999999999740000", "1239913664499998", "1239913664499998.8"}, + []string{"0.000004334593317", "249999999999999740000", "1083648329249998", "1083648329249998.9"}, + []string{"0.000011066270658", "249999999999999740000", "2766567664499997", "2766567664499997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000000485991737", "249999999999999740000", "121497934249999", "121497934249999.86"}, + []string{"0.000005957318075", "249999999999999740000", "1489329518749998", "1489329518749998.5"}, + []string{"0.000002821887509", "249999999999999740000", "705471877249999", "705471877249999.2"}, + []string{"0.000005460297788", "249999999999999740000", "1365074446999998", "1365074446999998.5"}, + []string{"0.000000593038012", "249999999999999740000", "148259502999999", "148259502999999.84"}, + []string{"0.00000048232807", "249999999999999740000", "120582017499999", "120582017499999.88"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000013835090761", "249999999999999740000", "3458772690249996", "3458772690249996.5"}, + []string{"0.000004387386402", "249999999999999740000", "1096846600499998", "1096846600499998.9"}, + []string{"0.00001056846525", "249999999999999740000", "2642116312499997", "2642116312499997.5"}, + []string{"0.000004744198683", "249999999999999740000", "1186049670749998", "1186049670749998.8"}, + []string{"0.000013517447109", "249999999999999740000", "3379361777249996", "3379361777249996.5"}, + []string{"0.000004292111888", "249999999999999740000", "1073027971999998", "1073027971999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000008820012468", "249999999999999740000", "2205003116999997", "2205003116999997.5"}, + []string{"0.000005601905333", "249999999999999740000", "1400476333249998", "1400476333249998.5"}, + []string{"0.000005339016974", "249999999999999740000", "1334754243499998", "1334754243499998.5"}, + []string{"0.000300001044541", "249999999999999740000", "75000261135249920", "75000261135249920"}, + []string{"0.000011067668642", "249999999999999740000", "2766917160499997", "2766917160499997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000016325425419", "249999999999999740000", "4081356354749995", "4081356354749995.5"}, + []string{"0.000000460259862", "249999999999999740000", "115064965499999", "115064965499999.88"}, + []string{"0.000000563455944", "249999999999999740000", "140863985999999", "140863985999999.88"}, + []string{"0.00000568617956", "249999999999999740000", "1421544889999998", "1421544889999998.5"}, + []string{"0.000004999005702", "249999999999999740000", "1249751425499998", "1249751425499998.5"}, + []string{"0.00000609084659", "249999999999999740000", "1522711647499998", "1522711647499998.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011067715964", "249999999999999740000", "2766928990999997", "2766928990999997"}, + []string{"0.000005004891542", "249999999999999740000", "1251222885499998", "1251222885499998.8"}, + []string{"0.000007218680732", "249999999999999740000", "1804670182999998", "1804670182999998"}, + []string{"0.000005144299747", "249999999999999740000", "1286074936749998", "1286074936749998.8"}, + []string{"0.00008989838143", "249999999999999740000", "22474595357499976", "22474595357499976"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011067089214", "249999999999999740000", "2766772303499997", "2766772303499997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000006545911248", "249999999999999740000", "1636477811999998", "1636477811999998.2"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011067039047", "249999999999999740000", "2766759761749997", "2766759761749997.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000009471564941", "249999999999999740000", "2367891235249997", "2367891235249997.5"}, + []string{"0.000008445422705", "249999999999999740000", "2111355676249997", "2111355676249997.8"}, + []string{"0.00000039192882", "249999999999999740000", "97982204999999", "97982204999999.89"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005340280631", "249999999999999740000", "1335070157749998", "1335070157749998.5"}, + []string{"0.00000435529761", "249999999999999740000", "1088824402499998", "1088824402499998.9"}, + []string{"0.000006367452314", "249999999999999740000", "1591863078499998", "1591863078499998.2"}, + []string{"0.000004372836209", "249999999999999740000", "1093209052249998", "1093209052249998.9"}, + []string{"0.000004926901446", "249999999999999740000", "1231725361499998", "1231725361499998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004755398978", "249999999999999740000", "1188849744499998", "1188849744499998.8"}, + []string{"0.000005303897879", "249999999999999740000", "1325974469749998", "1325974469749998.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004682911838", "249999999999999740000", "1170727959499998", "1170727959499998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011066751231", "249999999999999740000", "2766687807749997", "2766687807749997"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004781531613", "249999999999999740000", "1195382903249998", "1195382903249998.8"}, + []string{"0.000000526960156", "249999999999999740000", "131740038999999", "131740038999999.86"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000006337819945", "249999999999999740000", "1584454986249998", "1584454986249998.5"}, + []string{"0.00000598555262", "249999999999999740000", "1496388154999998", "1496388154999998.5"}, + []string{"0.000011133520684", "249999999999999740000", "2783380170999997", "2783380170999997"}, + []string{"0.000011067413867", "249999999999999740000", "2766853466749997", "2766853466749997"}, + []string{"0.00000039192882", "249999999999999740000", "97982204999999", "97982204999999.89"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005037750053", "249999999999999740000", "1259437513249998", "1259437513249998.8"}, + []string{"0.000010238640377", "249999999999999740000", "2559660094249997", "2559660094249997.5"}, + []string{"0.00000039192882", "249999999999999740000", "97982204999999", "97982204999999.89"}, + []string{"0.00000039192882", "249999999999999740000", "97982204999999", "97982204999999.89"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000031315112777", "249999999999999740000", "7828778194249992", "7828778194249992"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004552220781", "249999999999999740000", "1138055195249998", "1138055195249998.8"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000004771744448", "249999999999999740000", "1192936111999998", "1192936111999998.8"}, + []string{"0.000004470418106", "249999999999999740000", "1117604526499998", "1117604526499998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000001567715282", "249999999999999740000", "391928820499999", "391928820499999.56"}, + []string{"0.000006685375775", "249999999999999740000", "1671343943749998", "1671343943749998.2"}, + []string{"0.00000495047661", "249999999999999740000", "1237619152499998", "1237619152499998.8"}, + []string{"0.000005844909181", "249999999999999740000", "1461227295249998", "1461227295249998.5"}, + []string{"0.000000002145689", "249999999999999740000", "536422249999", "536422249999.99945"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000002987135399", "249999999999999740000", "746783849749999", "746783849749999.2"}, + []string{"0.000034747556263", "249999999999999740000", "8686889065749991", "8686889065749991"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000014311629335", "249999999999999740000", "3577907333749996", "3577907333749996.5"}, + []string{"0.000000039192882", "249999999999999740000", "9798220499999", "9798220499999.99"}, + []string{"0.000011066754626", "249999999999999740000", "2766688656499997", "2766688656499997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004761442856", "249999999999999740000", "1190360713999998", "1190360713999998.8"}, + []string{"0.000005488681729", "249999999999999740000", "1372170432249998", "1372170432249998.5"}, + []string{"0.000004720390951", "249999999999999740000", "1180097737749998", "1180097737749998.8"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004926662957", "249999999999999740000", "1231665739249998", "1231665739249998.8"}, + []string{"0.000005003715806", "249999999999999740000", "1250928951499998", "1250928951499998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004489784075", "249999999999999740000", "1122446018749998", "1122446018749998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000013860083391", "249999999999999740000", "3465020847749996", "3465020847749996.5"}, + []string{"0.000001990011373", "249999999999999740000", "497502843249999", "497502843249999.5"}, + []string{"0.00001106677849", "249999999999999740000", "2766694622499997", "2766694622499997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000000039196801", "249999999999999740000", "9799200249999", "9799200249999.99"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000013572735485", "249999999999999740000", "3393183871249996", "3393183871249996.5"}, + []string{"0.000004899110259", "249999999999999740000", "1224777564749998", "1224777564749998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011067136903", "249999999999999740000", "2766784225749997", "2766784225749997"}, + []string{"0.000004336354665", "249999999999999740000", "1084088666249998", "1084088666249998.9"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000000482687193", "249999999999999740000", "120671798249999", "120671798249999.86"}, + []string{"0.00000039192882", "249999999999999740000", "97982204999999", "97982204999999.89"}, + []string{"0.000004829343283", "249999999999999740000", "1207335820749998", "1207335820749998.8"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000006834871106", "249999999999999740000", "1708717776499998", "1708717776499998.2"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000008686667517", "249999999999999740000", "2171666879249998", "2171666879249998"}, + []string{"0.000011066162127", "249999999999999740000", "2766540531749997", "2766540531749997"}, + []string{"0.00000039192882", "249999999999999740000", "97982204999999", "97982204999999.89"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004558056201", "249999999999999740000", "1139514050249999", "1139514050249999"}, + []string{"0.000005388547951", "249999999999999740000", "1347136987749998", "1347136987749998.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000009495590539", "249999999999999740000", "2373897634749997", "2373897634749997.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005186272033", "249999999999999740000", "1296568008249998", "1296568008249998.8"}, + []string{"0.000006990370298", "249999999999999740000", "1747592574499998", "1747592574499998"}, + []string{"0.000006873785508", "249999999999999740000", "1718446376999998", "1718446376999998"}, + []string{"0.00000039192882", "249999999999999740000", "97982204999999", "97982204999999.89"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004504850433", "249999999999999740000", "1126212608249998", "1126212608249998.8"}, + []string{"0.000005530483313", "249999999999999740000", "1382620828249998", "1382620828249998.5"}, + []string{"0.000007310119829", "249999999999999740000", "1827529957249998", "1827529957249998.2"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011067991499", "249999999999999740000", "2766997874749997", "2766997874749997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000014234345856", "249999999999999740000", "3558586463999996", "3558586463999996.5"}, + []string{"0.000000592557184", "249999999999999740000", "148139295999999", "148139295999999.84"}, + []string{"0.000004913446869", "249999999999999740000", "1228361717249998", "1228361717249998.8"}, + []string{"0.000004480086345", "249999999999999740000", "1120021586249998", "1120021586249998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000000000303352", "249999999999999740000", "75837999999", "75837999999.99992"}, + []string{"0.000011252704148", "249999999999999740000", "2813176036999997", "2813176036999997"}, + []string{"0.000004505674706", "249999999999999740000", "1126418676499999", "1126418676499999"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004675810515", "249999999999999740000", "1168952628749998", "1168952628749998.8"}, + []string{"0.00000514400844", "249999999999999740000", "1286002109999998", "1286002109999998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004426765887", "249999999999999740000", "1106691471749998", "1106691471749998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011068588146", "249999999999999740000", "2767147036499997", "2767147036499997"}, + []string{"0.00001106757791", "249999999999999740000", "2766894477499997", "2766894477499997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004884842405", "249999999999999740000", "1221210601249998", "1221210601249998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005899779334", "249999999999999740000", "1474944833499998", "1474944833499998.2"}, + []string{"0.000013422003779", "249999999999999740000", "3355500944749996", "3355500944749996.5"}, + []string{"0.000144171486789", "249999999999999740000", "36042871697249960", "36042871697249960"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000000415444549", "249999999999999740000", "103861137249999", "103861137249999.89"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000031928323627", "249999999999999740000", "7982080906749991", "7982080906749991"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.00000002640117", "249999999999999740000", "6600292499999", "6600292499999.993"}, + []string{"0.000000633269907", "249999999999999740000", "158317476749999", "158317476749999.84"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005064523288", "249999999999999740000", "1266130821999998", "1266130821999998.8"}, + []string{"0.000004349845721", "249999999999999740000", "1087461430249999", "1087461430249999"}, + []string{"0.000007703579531", "249999999999999740000", "1925894882749998", "1925894882749998.2"}, + []string{"0.000000437957083", "249999999999999740000", "109489270749999", "109489270749999.89"}, + []string{"0.000012046208995", "249999999999999740000", "3011552248749997", "3011552248749997"}, + []string{"0.00000000002121", "249999999999999740000", "5302499999", "5302499999.999994"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005181989907", "249999999999999740000", "1295497476749998", "1295497476749998.5"}, + []string{"0.000006057031037", "249999999999999740000", "1514257759249998", "1514257759249998.2"}, + []string{"0.000007893648991", "249999999999999740000", "1973412247749997", "1973412247749997.8"}, + []string{"0.00001044882236", "249999999999999740000", "2612205589999997", "2612205589999997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004861240598", "249999999999999740000", "1215310149499998", "1215310149499998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004351580082", "249999999999999740000", "1087895020499999", "1087895020499999"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000000588314769", "249999999999999740000", "147078692249999", "147078692249999.84"}, + []string{"0.000009859499524", "249999999999999740000", "2464874880999997", "2464874880999997.5"}, + []string{"0.000004667920393", "249999999999999740000", "1166980098249999", "1166980098249999"}, + []string{"0.000002939466155", "249999999999999740000", "734866538749999", "734866538749999.2"}, + []string{"0.00000865481742", "249999999999999740000", "2163704354999997", "2163704354999997.8"}, + []string{"0.000005125798369", "249999999999999740000", "1281449592249998", "1281449592249998.5"}, + []string{"0.000004793636505", "249999999999999740000", "1198409126249998", "1198409126249998.8"}, + []string{"0.000011068483594", "249999999999999740000", "2767120898499997", "2767120898499997"}, + []string{"0.000007836352763", "249999999999999740000", "1959088190749998", "1959088190749998"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.0000046838042", "249999999999999740000", "1170951049999998", "1170951049999998.8"}, + []string{"0.000004618700167", "249999999999999740000", "1154675041749998", "1154675041749998.8"}, + []string{"0.000004537539334", "249999999999999740000", "1134384833499998", "1134384833499998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004776404037", "249999999999999740000", "1194101009249998", "1194101009249998.8"}, + []string{"0.000005553719601", "249999999999999740000", "1388429900249998", "1388429900249998.5"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000004489784075", "249999999999999740000", "1122446018749998", "1122446018749998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.00000692740917", "249999999999999740000", "1731852292499998", "1731852292499998.2"}, + []string{"0.000005584462421", "249999999999999740000", "1396115605249998", "1396115605249998.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004327021569", "249999999999999740000", "1081755392249998", "1081755392249998.9"}, + []string{"0.000004460682023", "249999999999999740000", "1115170505749998", "1115170505749998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000007523930305", "249999999999999740000", "1880982576249998", "1880982576249998"}, + []string{"0.000005014929065", "249999999999999740000", "1253732266249998", "1253732266249998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000000470315651", "249999999999999740000", "117578912749999", "117578912749999.88"}, + []string{"0.000011067266143", "249999999999999740000", "2766816535749997", "2766816535749997"}, + []string{"0.000004606871138", "249999999999999740000", "1151717784499998", "1151717784499998.8"}, + []string{"0.00000429376189", "249999999999999740000", "1073440472499998", "1073440472499998.9"}, + []string{"0.000007368882709", "249999999999999740000", "1842220677249998", "1842220677249998"}, + []string{"0.000005777164954", "249999999999999740000", "1444291238499998", "1444291238499998.5"}, + []string{"0.000000743682751", "249999999999999740000", "185920687749999", "185920687749999.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011066956193", "249999999999999740000", "2766739048249997", "2766739048249997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004444509005", "249999999999999740000", "1111127251249998", "1111127251249998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000009980851029", "249999999999999740000", "2495212757249997", "2495212757249997.5"}, + []string{"0.000004438132916", "249999999999999740000", "1109533228999998", "1109533228999998.9"}, + []string{"0.000004818112356", "249999999999999740000", "1204528088999998", "1204528088999998.8"}, + []string{"0.000011068963995", "249999999999999740000", "2767240998749997", "2767240998749997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000108090806087", "249999999999999740000", "27022701521749972", "27022701521749972"}, + []string{"0.000004534886824", "249999999999999740000", "1133721705999998", "1133721705999998.8"}, + []string{"0.000011066549479", "249999999999999740000", "2766637369749997", "2766637369749997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000003447716551", "249999999999999740000", "861929137749999", "861929137749999.1"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000014014908682", "249999999999999740000", "3503727170499996", "3503727170499996"}, + []string{"0.000004243148758", "249999999999999740000", "1060787189499998", "1060787189499998.9"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000008180999729", "249999999999999740000", "2045249932249997", "2045249932249997.8"}, + []string{"0.000001929418705", "249999999999999740000", "482354676249999", "482354676249999.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005589443629", "249999999999999740000", "1397360907249998", "1397360907249998.5"}, + []string{"0.000004701364694", "249999999999999740000", "1175341173499998", "1175341173499998.8"}, + []string{"0.000011068179353", "249999999999999740000", "2767044838249997", "2767044838249997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005798168821", "249999999999999740000", "1449542205249998", "1449542205249998.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000009286830011", "249999999999999740000", "2321707502749997", "2321707502749997.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000006054124493", "249999999999999740000", "1513531123249998", "1513531123249998.5"}, + []string{"0.000010961840774", "249999999999999740000", "2740460193499997", "2740460193499997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.00000459735762", "249999999999999740000", "1149339404999998", "1149339404999998.8"}, + []string{"0.000004607237865", "249999999999999740000", "1151809466249998", "1151809466249998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004999617541", "249999999999999740000", "1249904385249998", "1249904385249998.8"}, + []string{"0.000004384426544", "249999999999999740000", "1096106635999998", "1096106635999998.8"}, + []string{"0.000027592021209", "249999999999999740000", "6898005302249993", "6898005302249993"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011065571432", "249999999999999740000", "2766392857999997", "2766392857999997"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000011685473321", "249999999999999740000", "2921368330249997", "2921368330249997"}, + []string{"0.000005478819232", "249999999999999740000", "1369704807999998", "1369704807999998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000020025026258", "249999999999999740000", "5006256564499995", "5006256564499995"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000000438960279", "249999999999999740000", "109740069749999", "109740069749999.89"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005384947768", "249999999999999740000", "1346236941999998", "1346236941999998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004414406953", "249999999999999740000", "1103601738249998", "1103601738249998.8"}, + []string{"0.000010817235452", "249999999999999740000", "2704308862999997", "2704308862999997.5"}, + []string{"0.000003459054342", "249999999999999740000", "864763585499999", "864763585499999"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000006305221592", "249999999999999740000", "1576305397999998", "1576305397999998.2"}, + []string{"0.000000589197102", "249999999999999740000", "147299275499999", "147299275499999.84"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000005284741798", "249999999999999740000", "1321185449499998", "1321185449499998.5"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004977496023", "249999999999999740000", "1244374005749998", "1244374005749998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000003919288207", "249999999999999740000", "979822051749999", "979822051749999"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000008701617124", "249999999999999740000", "2175404280999997", "2175404280999997.8"}, + []string{"0.000009297049021", "249999999999999740000", "2324262255249997", "2324262255249997.5"}, + []string{"0.00000435040991", "249999999999999740000", "1087602477499998", "1087602477499998.8"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004905866207", "249999999999999740000", "1226466551749998", "1226466551749998.8"}, + []string{"0.000011188276426", "249999999999999740000", "2797069106499997", "2797069106499997"}, + []string{"0.000004489846462", "249999999999999740000", "1122461615499998", "1122461615499998.9"}, + []string{"0.000004626953867", "249999999999999740000", "1156738466749999", "1156738466749999"}, + []string{"0.000023680418961", "249999999999999740000", "5920104740249994", "5920104740249994"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000004311217028", "249999999999999740000", "1077804256999998", "1077804256999998.9"}, + []string{"0.000000054417993", "249999999999999740000", "13604498249999", "13604498249999.986"}, + []string{"0.000005470503287", "249999999999999740000", "1367625821749998", "1367625821749998.5"}, + } + + for i, value := range values { + stakerProportion := value[0] + tokensPerDayDecimal := value[1] + expectedValue := value[2] + result, err := CalculatePostNileStakerTokenRewards(stakerProportion, tokensPerDayDecimal) + assert.Nil(t, err) + + assert.Equal(t, expectedValue, result, fmt.Sprintf("row %d: '%s' - '%s' - '%s'", i, stakerProportion, tokensPerDayDecimal, expectedValue)) + } + }) + }) + + t.Run("Operator tokens", func(t *testing.T) { + t.Run("Should calculate operator amazon fork tokens correctly", func(t *testing.T) { + for _, value := range operatorAmazonTokens { + totalStakerOperatorPayout := value[0] + expectedValue := value[1] + + result, err := CalculateAmazonOperatorTokens(totalStakerOperatorPayout) + assert.Nil(t, err) + assert.Equal(t, expectedValue, result) + } + }) + t.Run("Should calculate nile fork tokens correctly", func(t *testing.T) { + for _, value := range operatorNileTokens { + totalStakerOperatorPayout := value[0] + expectedValue := value[1] + + result, err := CalculateNileOperatorTokens(totalStakerOperatorPayout) + assert.Nil(t, err) + assert.Equal(t, expectedValue, result) + } + }) + }) } diff --git a/internal/types/numbers/tokenCalculations.go b/internal/types/numbers/tokenCalculations.go new file mode 100644 index 00000000..1aa778be --- /dev/null +++ b/internal/types/numbers/tokenCalculations.go @@ -0,0 +1,127 @@ +package numbers + +import ( + "errors" + "fmt" + "github.com/shopspring/decimal" +) + +/* +#cgo darwin CFLAGS: -I/opt/homebrew/opt/python@3.12/Frameworks/Python.framework/Versions/3.12/include/python3.12 -I/Users/seanmcgary/Code/sidecar/sqlite-extensions +#cgo darwin LDFLAGS: -L/opt/homebrew/opt/python@3.12/Frameworks/Python.framework/Versions/3.12/lib -lpython3.12 -L/Users/seanmcgary/Code/sidecar/sqlite-extensions -lcalculations +#cgo darwin LDFLAGS: -Wl,-rpath,/Users/seanmcgary/Code/sidecar/sqlite-extensions +#include +#include "calculations.h" +*/ +import "C" +import "unsafe" + +func InitPython() error { + if C.init_python() == 0 { + return errors.New("failed to initialize python") + } + return nil +} + +func FinalizePython() { + C.finalize_python() +} + +// CalculateAmazonStakerTokenRewards calculates the Amazon token rewards for a given staker proportion and tokens per day +// cast(staker_proportion * tokens_per_day AS DECIMAL(38,0)) +func CalculateAmazonStakerTokenRewards(stakerProportion string, tokensPerDay string) (string, error) { + cSp := C.CString(stakerProportion) + cTpd := C.CString(tokensPerDay) + defer C.free(unsafe.Pointer(cSp)) + defer C.free(unsafe.Pointer(cTpd)) + + cResult := C._amazon_staker_token_rewards(cSp, cTpd) + defer C.free(unsafe.Pointer(cResult)) + + return C.GoString(cResult), nil +} + +// CalculateNileStakerTokenRewards calculates the tokens to be rewarded for a given staker proportion and tokens per day +// (staker_proportion * tokens_per_day)::text::decimal(38,0) +func CalculateNileStakerTokenRewards(stakerProportion string, tokensPerDay string) (string, error) { + cSp := C.CString(stakerProportion) + cTpd := C.CString(tokensPerDay) + defer C.free(unsafe.Pointer(cSp)) + defer C.free(unsafe.Pointer(cTpd)) + + cResult := C._nile_staker_token_rewards(cSp, cTpd) + defer C.free(unsafe.Pointer(cResult)) + + return C.GoString(cResult), nil +} + +// CalculatePostNileStakerTokenRewards calculates the tokens to be rewarded for a given staker proportion and tokens per day +// FLOOR(staker_proportion * tokens_per_day_decimal) +func CalculatePostNileStakerTokenRewards(stakerProportion string, tokensPerDay string) (string, error) { + cSp := C.CString(stakerProportion) + cTpd := C.CString(tokensPerDay) + defer C.free(unsafe.Pointer(cSp)) + defer C.free(unsafe.Pointer(cTpd)) + + cResult := C._staker_token_rewards(cSp, cTpd) + defer C.free(unsafe.Pointer(cResult)) + + return C.GoString(cResult), nil +} + +// CalculateAmazonOperatorTokens calculates the operator payout portion for rewards (10% of total) +// +// cast(total_staker_operator_payout * 0.10 AS DECIMAL(38,0)) +func CalculateAmazonOperatorTokens(totalStakerPayout string) (string, error) { + tsp := C.CString(totalStakerPayout) + defer C.free(unsafe.Pointer(tsp)) + + cResult := C._amazon_operator_token_rewards(tsp) + defer C.free(unsafe.Pointer(cResult)) + + return C.GoString(cResult), nil +} + +// CalculateNileOperatorTokens calculates the operator payout portion for rewards (10% of total) +// +// (total_staker_operator_payout * 0.10)::text::decimal(38,0) +func CalculateNileOperatorTokens(totalStakerPayout string) (string, error) { + tsp := C.CString(totalStakerPayout) + defer C.free(unsafe.Pointer(tsp)) + + cResult := C._nile_operator_token_rewards(tsp) + defer C.free(unsafe.Pointer(cResult)) + + return C.GoString(cResult), nil +} + +// CalculatePostNileOperatorTokens calculates the operator payout portion for rewards (10% of total) +// +// floor(total_staker_operator_payout * 0.10) +func CalculatePostNileOperatorTokens(totalStakerPayout string) (string, error) { + tpd, err := decimal.NewFromString(totalStakerPayout) + if err != nil { + return "", err + } + + return tpd.Mul(decimal.NewFromFloat(0.10)).Floor().String(), nil +} + +// PreNileTokensPerDay calculates the tokens per day for pre-nile rewards, rounded to 15 sigfigs +// +// Not gonna lie, this is pretty annoying that it has to be this way, but in order to support backwards compatibility +// with the current/old rewards system where postgres was lossy, we have to do this. +func PreNileTokensPerDay(tokensPerDay string) (string, error) { + fmt.Printf("PreNileTokensPerDay: %s\n", tokensPerDay) + cTokens := C.CString(tokensPerDay) + defer C.free(unsafe.Pointer(cTokens)) + + fmt.Printf("Calling pre_nile_tokens_per_day\n") + result := C._pre_nile_tokens_per_day(cTokens) + fmt.Printf("Successfully called") + defer C.free(unsafe.Pointer(result)) + + resultStr := C.GoString(result) + fmt.Printf("PreNileTokensPerDay Result: %+v\n", resultStr) + return resultStr, nil +} diff --git a/pkg/rewards/1_goldActiveRewards.go b/pkg/rewards/1_goldActiveRewards.go new file mode 100644 index 00000000..d9f04ced --- /dev/null +++ b/pkg/rewards/1_goldActiveRewards.go @@ -0,0 +1,170 @@ +package rewards + +import ( + "database/sql" + "fmt" +) + +var _1_goldActiveRewardsQuery = ` +-- insert into gold_1_active_rewards +WITH active_rewards_modified as ( + SELECT + *, + calc_raw_tokens_per_day(amount, duration) as tokens_per_day, + DATETIME(@cutoffDate) as global_end_inclusive -- Inclusive means we DO USE this day as a snapshot + FROM combined_rewards + WHERE end_timestamp >= DATETIME(@rewardsStart) and start_timestamp <= DATETIME(@cutoffDate) +), +-- Cut each reward's start and end windows to handle the global range +active_rewards_updated_end_timestamps as ( + SELECT + avs, + -- Cut the start and end windows to handle + -- A. Retroactive rewards that came recently whose start date is less than start_timestamp + -- B. Don't make any rewards past end_timestamp for this run + start_timestamp as reward_start_exclusive, + MIN(global_end_inclusive, end_timestamp) as reward_end_inclusive, + tokens_per_day, + token, + multiplier, + strategy, + reward_hash, + reward_type, + global_end_inclusive, + block_date as reward_submission_date + FROM active_rewards_modified +), +-- For each reward hash, find the latest snapshot +active_rewards_updated_start_timestamps as ( + SELECT + ap.avs, + ap.reward_start_exclusive as reward_start_exclusive, + ap.reward_end_inclusive, + ap.token, + post_nile_tokens_per_day(ap.tokens_per_day) as tokens_per_day_decimal, + pre_nile_tokens_per_day(ap.tokens_per_day) as tokens_per_day, + ap.multiplier, + ap.strategy, + ap.reward_hash, + ap.reward_type, + ap.global_end_inclusive, + ap.reward_submission_date + FROM active_rewards_updated_end_timestamps ap + LEFT JOIN gold_table g ON g.reward_hash = ap.reward_hash + GROUP BY ap.avs, ap.reward_end_inclusive, ap.token, ap.tokens_per_day, ap.multiplier, ap.strategy, ap.reward_hash, ap.global_end_inclusive, ap.reward_start_exclusive, ap.reward_type, ap.reward_submission_date +), +-- Parse out invalid ranges +active_reward_ranges AS ( + SELECT * from active_rewards_updated_start_timestamps + -- Take out (reward_start_exclusive, reward_end_inclusive) windows where + -- 1. reward_start_exclusive >= reward_end_inclusive: The reward period is done or we will handle on a subsequent run + WHERE reward_start_exclusive < reward_end_inclusive +), + date_bounds as ( + select + min(reward_start_exclusive) as min_start, + max(reward_end_inclusive) as max_end + from active_reward_ranges + ), + day_series AS ( + with RECURSIVE day_series_inner AS ( + SELECT DATE(min_start) AS day + FROM date_bounds + UNION ALL + SELECT DATE(day, '+1 day') + FROM day_series_inner + WHERE day < (SELECT max_end FROM date_bounds) + ) + select * from day_series_inner + ), +-- Explode out the ranges for a day per inclusive date + exploded_active_range_rewards AS ( + SELECT + arr.*, + day_series.day as day + FROM active_reward_ranges as arr + cross join day_series + where DATE(day_series.day) between DATE(reward_start_exclusive) and DATE(reward_end_inclusive) + ), + active_rewards_final AS ( + SELECT + avs, + DATE(day) as snapshot, + token, + tokens_per_day, + tokens_per_day_decimal, + multiplier, + strategy, + reward_hash, + reward_type, + reward_submission_date + FROM exploded_active_range_rewards + -- Remove snapshots on the start day + WHERE day != reward_start_exclusive + ) +select + avs, + snapshot, + token, + tokens_per_day, + tokens_per_day_decimal, + multiplier, + strategy, + reward_hash, + reward_type, + reward_submission_date +from active_rewards_final +` + +type ResultRow struct { + Avs string + Snapshot string + Token string + TokensPerDay string + TokensPerDayDecimal string +} + +// GenerateActiveRewards generates active rewards for the gold_1_active_rewards table +// +// @param snapshotDate: The upper bound of when to calculate rewards to +// @param startDate: The lower bound of when to calculate rewards from. If we're running rewards for the first time, +// this will be "1970-01-01". If this is a subsequent run, this will be the last snapshot date. +func (r *RewardsCalculator) GenerateActiveRewards(cutoffDate string, startDate string) error { + r.logger.Sugar().Infow("Generating active rewards", "cutoffDate", cutoffDate, "startDate", startDate) + rows := make([]ResultRow, 0) + res := r.grm.Raw(_1_goldActiveRewardsQuery, + sql.Named("cutoffDate", cutoffDate), + sql.Named("rewardsStart", startDate), + ).Scan(&rows) + fmt.Printf("Rows: %+v\n", rows) + r.logger.Sugar().Infof("Rows affected: %+v", res.RowsAffected) + r.logger.Sugar().Infow("Executed active rewards query") + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to generate active rewards", "error", res.Error) + return res.Error + } + return nil +} + +func (r *RewardsCalculator) CreateGold1ActiveRewardsTable() error { + query := ` + create table if not exists gold_1_active_rewards ( + avs TEXT NOT NULL, + snapshot DATE NOT NULL, + token TEXT NOT NULL, + tokens_per_day TEXT NOT NULL, + tokens_per_day_decimal TEXT NOT NULL, + multiplier TEXT NOT NULL, + strategy TEXT NOT NULL, + reward_hash TEXT NOT NULL, + reward_type TEXT NOT NULL, + reward_submission_date DATE NOT NULL + ) + ` + res := r.grm.Exec(query) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to create gold_1_active_rewards table", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/2_goldStakerRewardAmounts.go b/pkg/rewards/2_goldStakerRewardAmounts.go new file mode 100644 index 00000000..9f6b2ab9 --- /dev/null +++ b/pkg/rewards/2_goldStakerRewardAmounts.go @@ -0,0 +1,180 @@ +package rewards + +import ( + "database/sql" + "github.com/Layr-Labs/go-sidecar/internal/config" +) + +const _2_goldStakerRewardAmountsQuery = ` +insert into gold_2_staker_reward_amounts +WITH reward_snapshot_operators as ( + SELECT + ap.reward_hash, + ap.snapshot, + ap.token, + ap.tokens_per_day, + ap.tokens_per_day_decimal, + ap.avs, + ap.strategy, + ap.multiplier, + ap.reward_type, + ap.reward_submission_date, + oar.operator + FROM gold_1_active_rewards ap + JOIN operator_avs_registration_snapshots oar + ON ap.avs = oar.avs and ap.snapshot = oar.snapshot + WHERE ap.reward_type = 'avs' +), +_operator_restaked_strategies AS ( + SELECT + rso.* + FROM reward_snapshot_operators rso + JOIN operator_avs_strategy_snapshots oas + ON + rso.operator = oas.operator AND + rso.avs = oas.avs AND + rso.strategy = oas.strategy AND + rso.snapshot = oas.snapshot +), +-- Get the stakers that were delegated to the operator for the snapshot +staker_delegated_operators AS ( + SELECT + ors.*, + sds.staker + FROM _operator_restaked_strategies ors + JOIN staker_delegation_snapshots sds + ON + ors.operator = sds.operator AND + ors.snapshot = sds.snapshot +), +-- Get the shares for staker delegated to the operator +staker_avs_strategy_shares AS ( + SELECT + sdo.*, + sss.shares + FROM staker_delegated_operators sdo + JOIN staker_share_snapshots sss + ON + sdo.staker = sss.staker AND + sdo.snapshot = sss.snapshot AND + sdo.strategy = sss.strategy + -- Parse out negative shares and zero multiplier so there is no division by zero case + WHERE sss.shares > 0 and sdo.multiplier != 0 +), +-- Calculate the weight of a staker +staker_weights AS ( + SELECT *, + sum_big(numeric_multiply(multiplier, shares)) OVER (PARTITION BY staker, reward_hash, snapshot) AS staker_weight + FROM staker_avs_strategy_shares +), +-- Get distinct stakers since their weights are already calculated +distinct_stakers AS ( + SELECT * + FROM ( + SELECT *, + -- We can use an arbitrary order here since the staker_weight is the same for each (staker, strategy, hash, snapshot) + -- We use strategy ASC for better debuggability + ROW_NUMBER() OVER (PARTITION BY reward_hash, snapshot, staker ORDER BY strategy ASC) as rn + FROM staker_weights + ) t + WHERE rn = 1 + ORDER BY reward_hash, snapshot, staker +), +-- Calculate sum of all staker weights for each reward and snapshot +staker_weight_sum AS ( + SELECT *, + sum_big(staker_weight) OVER (PARTITION BY reward_hash, snapshot) as total_weight + FROM distinct_stakers +), +-- Calculate staker proportion of tokens for each reward and snapshot +staker_proportion AS ( + SELECT *, + calc_staker_proportion(staker_weight, total_weight) as staker_proportion + FROM staker_weight_sum +), +-- Calculate total tokens to the (staker, operator) pair +staker_operator_total_tokens AS ( + SELECT *, + CASE + -- For snapshots that are before the hard fork AND submitted before the hard fork, we use the old calc method + WHEN snapshot < DATE(@amazonHardforkDate) AND reward_submission_date < DATE(@amazonHardforkDate) THEN + amazon_token_rewards(staker_proportion, tokens_per_day) + -- cast(staker_proportion * tokens_per_day AS DECIMAL(38,0)) + WHEN snapshot < DATE(@nileHardforkDate) AND reward_submission_date < DATE(@nileHardforkDate) THEN + nile_token_rewards(staker_proportion, tokens_per_day) + -- (staker_proportion * tokens_per_day)::text::decimal(38,0) + ELSE + -- FLOOR(staker_proportion * tokens_per_day_decimal) + post_nile_token_rewards(staker_proportion, tokens_per_day) + END as total_staker_operator_payout + FROM staker_proportion +), +operator_tokens as ( + select * + CASE + WHEN snapshot < DATE(@amazonHardforkDate) AND reward_submission_date < DATE(@amazonHardforkDate) THEN + amazon_operator_tokens(total_staker_operator_payout) + --cast(total_staker_operator_payout * 0.10 AS DECIMAL(38,0)) + WHEN snapshot < DATE(@nileHardforkDate) AND reward_submission_date < DATE(@nileHardforkDate) THEN + nile_operator_tokens(total_staker_operator_payout) + -- (total_staker_operator_payout * 0.10)::text::decimal(38,0) + ELSE + post_nile_operator_tokens(total_staker_operator_payout) + -- floor(total_staker_operator_payout * 0.10) + END as operator_tokens + from staker_operator_total_tokens +), +-- Calculate the token breakdown for each (staker, operator) pair +token_breakdowns AS ( + SELECT *, + subtract_big(total_staker_operator_payout, operator_tokens) as staker_tokens + FROM operator_tokens +) +SELECT * from token_breakdowns +ORDER BY reward_hash, snapshot, staker, operator +` + +func (rc *RewardsCalculator) GenerateGoldStakerRewardAmountsTable(forks config.ForkMap) error { + res := rc.grm.Exec(_2_goldStakerRewardAmountsQuery, + sql.Named("amazonHardforkDate", forks[config.Fork_Amazon]), + sql.Named("nileHardforkDate", forks[config.Fork_Nile]), + ) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_staker_reward_amounts", "error", res.Error) + return res.Error + } + return nil +} + +func (rc *RewardsCalculator) CreateGold2RewardAmountsTable() error { + query := ` + create table if not exists gold_2_staker_reward_amounts ( + reward_hash TEXT NOT NULL, + snapshot DATE NOT NULL, + token TEXT NOT NULL, + tokens_per_day TEXT NOT NULL, + tokens_per_day_decimal TEXT NOT NULL, + avs TEXT NOT NULL, + strategy TEXT NOT NULL, + multiplier TEXT NOT NULL, + reward_type TEXT NOT NULL, + reward_submission_date DATE NOT NULL, + operator TEXT NOT NULL, + staker TEXT NOT NULL, + shares TEXT NOT NULL, + staker_weight TEXT NOT NULL, + rn INTEGER NOT NULL, + total_weight TEXT NOT NULL, + staker_proportion TEXT NOT NULL, + total_staker_operator_payout TEXT NOT NULL, + operator_tokens TEXT NOT NULL, + staker_tokens TEXT NOT NULL + ) + ` + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_2_staker_reward_amounts table", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/3_goldOperatorRewardAmounts.go b/pkg/rewards/3_goldOperatorRewardAmounts.go new file mode 100644 index 00000000..8204834f --- /dev/null +++ b/pkg/rewards/3_goldOperatorRewardAmounts.go @@ -0,0 +1,65 @@ +package rewards + +const _3_goldOperatorRewardAmountsQuery = ` +insert into gold_3_operator_reward_amounts +WITH operator_token_sums AS ( + SELECT + reward_hash, + snapshot, + token, + tokens_per_day, + avs, + strategy, + multiplier, + reward_type, + operator, + sum_big(operator_tokens) OVER (PARTITION BY operator, reward_hash, snapshot) AS operator_tokens + FROM gold_2_staker_reward_amounts +), +-- Dedupe the operator tokens across strategies for each operator, reward hash, and snapshot +distinct_operators AS ( + SELECT * + FROM ( + SELECT *, + -- We can use an arbitrary order here since the staker_weight is the same for each (operator, strategy, hash, snapshot) + -- We use strategy ASC for better debuggability + ROW_NUMBER() OVER (PARTITION BY reward_hash, snapshot, operator ORDER BY strategy ASC) as rn + FROM operator_token_sums + ) t + WHERE rn = 1 +) +SELECT * FROM distinct_operators +` + +func (rc *RewardsCalculator) GenerateGoldOperatorRewardAmountsTable() error { + res := rc.grm.Exec(_3_goldOperatorRewardAmountsQuery) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_operator_reward_amounts", "error", res.Error) + return res.Error + } + return nil +} + +func (rc *RewardsCalculator) CreateGold3OperatorRewardsTable() error { + query := ` + create table if not exists gold_3_operator_reward_amounts ( + reward_hash TEXT NOT NULL, + snapshot DATE NOT NULL, + token TEXT NOT NULL, + tokens_per_day TEXT NOT NULL, + avs TEXT NOT NULL, + strategy TEXT NOT NULL, + multiplier TEXT NOT NULL, + reward_type TEXT NOT NULL, + operator TEXT NOT NULL, + operator_tokens TEXT NOT NULL, + rn INTEGER NOT NULL + ) + ` + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_3_operator_reward_amounts table", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/4_goldRewardsForAll.go b/pkg/rewards/4_goldRewardsForAll.go new file mode 100644 index 00000000..02c00acb --- /dev/null +++ b/pkg/rewards/4_goldRewardsForAll.go @@ -0,0 +1,101 @@ +package rewards + +const _4_goldRewardsForAllQuery = ` +insert into gold_4_rewards_for_all +WITH reward_snapshot_stakers AS ( + SELECT + ap.reward_hash, + ap.snapshot, + ap.token, + ap.tokens_per_day, + ap.avs, + ap.strategy, + ap.multiplier, + ap.reward_type, + sss.staker, + sss.shares + FROM gold_1_active_rewards ap + JOIN staker_share_snapshots as sss + ON ap.strategy = sss.strategy and ap.snapshot = sss.snapshot + WHERE ap.reward_type = 'all_stakers' + -- Parse out negative shares and zero multiplier so there is no division by zero case + AND big_gt(sss.shares, "0") and ap.multiplier != "0" +), +-- Calculate the weight of a staker +staker_weights AS ( + SELECT *, + sum_big(numeric_multiply(multiplier, shares)) OVER (PARTITION BY staker, reward_hash, snapshot) AS staker_weight + FROM reward_snapshot_stakers +), +-- Get distinct stakers since their weights are already calculated +distinct_stakers AS ( + SELECT * + FROM ( + SELECT *, + -- We can use an arbitrary order here since the staker_weight is the same for each (staker, strategy, hash, snapshot) + -- We use strategy ASC for better debuggability + ROW_NUMBER() OVER (PARTITION BY reward_hash, snapshot, staker ORDER BY strategy ASC) as rn + FROM staker_weights + ) t + WHERE rn = 1 + ORDER BY reward_hash, snapshot, staker +), +-- Calculate sum of all staker weights +staker_weight_sum AS ( + SELECT *, + sum_big(staker_weight) OVER (PARTITION BY reward_hash, snapshot) as total_staker_weight + FROM distinct_stakers +), +-- Calculate staker token proportion +staker_proportion AS ( + SELECT *, + calc_staker_proportion(staker_weight, total_staker_weight) as staker_proportion + FROM staker_weight_sum +), +-- Calculate total tokens to staker +staker_tokens AS ( + SELECT *, + -- TODO: update to using floor when we reactivate this + nile_token_rewards(staker_proportion, tokens_per_day) as staker_tokens + -- (tokens_per_day * staker_proportion)::text::decimal(38,0) as staker_tokens + FROM staker_proportion +) +SELECT * from staker_tokens +` + +func (rc *RewardsCalculator) GenerateGoldRewardsForAllTable() error { + res := rc.grm.Exec(_4_goldRewardsForAllQuery) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_rewards_for_all", "error", res.Error) + return res.Error + } + return nil +} + +func (rc *RewardsCalculator) CreateGold4RewardsForAllTable() error { + query := ` + create table if not exists gold_4_rewards_for_all ( + reward_hash TEXT NOT NULL, + snapshot DATE NOT NULL, + token TEXT NOT NULL, + tokens_per_day TEXT NOT NULL, + avs TEXT NOT NULL, + strategy TEXT NOT NULL, + multiplier TEXT NOT NULL, + reward_type TEXT NOT NULL, + staker TEXT NOT NULL, + shares TEXT NOT NULL, + staker_weight TEXT NOT NULL, + rn INTEGER NOT NULL, + total_staker_weight TEXT NOT NULL, + staker_proportion TEXT NOT NULL, + staker_tokens TEXT NOT NULL + ) + ` + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_4_rewards_for_all table", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/5_goldRfaeStakers.go b/pkg/rewards/5_goldRfaeStakers.go new file mode 100644 index 00000000..b81da8fe --- /dev/null +++ b/pkg/rewards/5_goldRfaeStakers.go @@ -0,0 +1,141 @@ +package rewards + +const _5_goldRfaeStakersQuery = ` +insert into gold_5_rfae_stakers +WITH avs_opted_operators AS ( + SELECT DISTINCT + snapshot, + operator + FROM operator_avs_registration_snapshots +), +-- Get the operators who will earn rewards for the reward submission at the given snapshot +reward_snapshot_operators as ( + SELECT + ap.reward_hash, + ap.snapshot, + ap.token, + ap.tokens_per_day_decimal, + ap.avs, + ap.strategy, + ap.multiplier, + ap.reward_type, + ap.reward_submission_date, + aoo.operator + FROM gold_1_active_rewards ap + JOIN avs_opted_operators aoo + ON ap.snapshot = aoo.snapshot + WHERE ap.reward_type = 'all_earners' +), +-- Get the stakers that were delegated to the operator for the snapshot +staker_delegated_operators AS ( + SELECT + rso.*, + sds.staker + FROM reward_snapshot_operators rso + JOIN staker_delegation_snapshots sds + ON + rso.operator = sds.operator AND + rso.snapshot = sds.snapshot +), +-- Get the shares of each strategy the staker has delegated to the operator +staker_strategy_shares AS ( + SELECT + sdo.*, + sss.shares + FROM staker_delegated_operators sdo + JOIN staker_share_snapshots sss + ON + sdo.staker = sss.staker AND + sdo.snapshot = sss.snapshot AND + sdo.strategy = sss.strategy + -- Parse out negative shares and zero multiplier so there is no division by zero case + WHERE big_gt(sss.shares, "0") and sdo.multiplier != "0" +), +-- Calculate the weight of a staker +staker_weights AS ( + SELECT *, + big_sum(numeric_multiply(multiplier, shares)) OVER (PARTITION BY staker, reward_hash, snapshot) AS staker_weight + FROM staker_strategy_shares +), +-- Get distinct stakers since their weights are already calculated +distinct_stakers AS ( + SELECT * + FROM ( + SELECT *, + -- We can use an arbitrary order here since the staker_weight is the same for each (staker, strategy, hash, snapshot) + -- We use strategy ASC for better debuggability + ROW_NUMBER() OVER (PARTITION BY reward_hash, snapshot, staker ORDER BY strategy ASC) as rn + FROM staker_weights + ) t + WHERE rn = 1 + ORDER BY reward_hash, snapshot, staker +), +-- Calculate sum of all staker weights for each reward and snapshot +staker_weight_sum AS ( + SELECT *, + big_sum(staker_weight) OVER (PARTITION BY reward_hash, snapshot) as total_weight + FROM distinct_stakers +), +-- Calculate staker proportion of tokens for each reward and snapshot +staker_proportion AS ( + SELECT *, + calc_staker_proportion(staker_weight, total_weight) as staker_proportion + FROM staker_weight_sum +), +-- Calculate total tokens to the (staker, operator) pair +staker_operator_total_tokens AS ( + SELECT *, + post_nile_token_rewards(staker_proportion, tokens_per_day_decimal) as total_staker_operator_payout + FROM staker_proportion +), +-- Calculate the token breakdown for each (staker, operator) pair +token_breakdowns AS ( + SELECT *, + floor(total_staker_operator_payout * 0.10) as operator_tokens, + subtract_big(total_staker_operator_payout, post_nile_operator_tokens(total_staker_operator_payout)) as staker_tokens + FROM staker_operator_total_tokens +) +SELECT * from token_breakdowns +ORDER BY reward_hash, snapshot, staker, operator +` + +func (rc *RewardsCalculator) GenerateGoldRfaeStakersTable() error { + res := rc.grm.Exec(_5_goldRfaeStakersQuery) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_rfae_stakers", "error", res.Error) + return res.Error + } + return nil +} + +func (rc *RewardsCalculator) CreateGold5RfaeStakersTable() error { + query := ` + create table if not exists gold_5_rfae_stakers ( + reward_hash TEXT NOT NULL, + snapshot DATE NOT NULL, + token TEXT NOT NULL, + tokens_per_day TEXT NOT NULL, + avs TEXT NOT NULL, + strategy TEXT NOT NULL, + multiplier TEXT NOT NULL, + reward_type TEXT NOT NULL, + reward_submission_date DATE NOT NULL, + operator TEXT NOT NULL, + staker TEXT NOT NULL, + shares TEXT NOT NULL, + staker_weight TEXT NOT NULL, + rn INTEGER NOT NULL, + total_weight TEXT NOT NULL, + staker_proportion TEXT NOT NULL, + total_staker_operator_payout TEXT NOT NULL, + operator_tokens TEXT NOT NULL, + staker_tokens TEXT NOT NULL + ) + ` + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_5_rfae_stakers table", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/6_goldRfaeOperators.go b/pkg/rewards/6_goldRfaeOperators.go new file mode 100644 index 00000000..d7b25f54 --- /dev/null +++ b/pkg/rewards/6_goldRfaeOperators.go @@ -0,0 +1,65 @@ +package rewards + +const _6_goldRfaeOperatorsQuery = ` +insert into gold_6_rfae_operators as +WITH operator_token_sums AS ( + SELECT + reward_hash, + snapshot, + token, + tokens_per_day_decimal, + avs, + strategy, + multiplier, + reward_type, + operator, + big_sum(operator_tokens) OVER (PARTITION BY operator, reward_hash, snapshot) AS operator_tokens + FROM gold_5_rfae_stakers +), +-- Dedupe the operator tokens across strategies for each operator, reward hash, and snapshot +distinct_operators AS ( + SELECT * + FROM ( + SELECT *, + -- We can use an arbitrary order here since the staker_weight is the same for each (operator, strategy, hash, snapshot) + -- We use strategy ASC for better debuggability + ROW_NUMBER() OVER (PARTITION BY reward_hash, snapshot, operator ORDER BY strategy ASC) as rn + FROM operator_token_sums + ) t + WHERE rn = 1 +) +SELECT * FROM distinct_operators +` + +func (rc *RewardsCalculator) GenerateGoldRfaeOperatorsTable() error { + res := rc.grm.Exec(_6_goldRfaeOperatorsQuery) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_rfae_operators", "error", res.Error) + return res.Error + } + return nil +} + +func (rc *RewardsCalculator) CreateGold6RfaeOperatorsTable() error { + query := ` + create table if not exists gold_6_rfae_operators ( + reward_hash TEXT NOT NULL, + snapshot DATE NOT NULL, + token TEXT NOT NULL, + tokens_per_day_decimal TEXT NOT NULL, + avs TEXT NOT NULL, + strategy TEXT NOT NULL, + multiplier TEXT NOT NULL, + reward_type TEXT NOT NULL, + operator TEXT NOT NULL, + operator_tokens TEXT NOT NULL, + rn INTEGER NOT NULL + ) + ` + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_6_rfae_operators", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/7_goldStaging.go b/pkg/rewards/7_goldStaging.go new file mode 100644 index 00000000..ddb93301 --- /dev/null +++ b/pkg/rewards/7_goldStaging.go @@ -0,0 +1,107 @@ +package rewards + +const _7_goldStagingQuery = ` +insert into gold_7_staging as +WITH staker_rewards AS ( + -- We can select DISTINCT here because the staker's tokens are the same for each strategy in the reward hash + SELECT DISTINCT + staker as earner, + snapshot, + reward_hash, + token, + staker_tokens as amount + FROM gold_2_staker_reward_amounts +), +operator_rewards AS ( + SELECT DISTINCT + -- We can select DISTINCT here because the operator's tokens are the same for each strategy in the reward hash + operator as earner, + snapshot, + reward_hash, + token, + operator_tokens as amount + FROM gold_3_operator_reward_amounts +), +rewards_for_all AS ( + SELECT DISTINCT + staker as earner, + snapshot, + reward_hash, + token, + staker_tokens as amount + FROM gold_4_rewards_for_all +), +rewards_for_all_earners_stakers AS ( + SELECT DISTINCT + staker as earner, + snapshot, + reward_hash, + token, + staker_tokens as amounts + FROM gold_5_rfae_stakers +), +rewards_for_all_earners_operators AS ( + SELECT DISTINCT + operator as earner, + snapshot, + reward_hash, + token, + operator_tokens as amount + FROM gold_6_rfae_operators +), +combined_rewards AS ( + SELECT * FROM operator_rewards + UNION ALL + SELECT * FROM staker_rewards + UNION ALL + SELECT * FROM rewards_for_all + UNION ALL + SELECT * FROM rewards_for_all_earners_stakers + UNION ALL + SELECT * FROM rewards_for_all_earners_operators +), +-- Dedupe earners, primarily operators who are also their own staker. +deduped_earners AS ( + SELECT + earner, + snapshot, + reward_hash, + token, + big_sum(amount) as amount + FROM combined_rewards + GROUP BY + earner, + snapshot, + reward_hash, + token +) +SELECT * +FROM deduped_earners +` + +func (rc *RewardsCalculator) GenerateGoldStagingTable() error { + res := rc.grm.Exec(_7_goldStagingQuery) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_staging", "error", res.Error) + return res.Error + } + return nil +} + +func (rc *RewardsCalculator) CreateGold7StagingTable() error { + query := ` + create table if not exists gold_7_staging ( + earner TEXT NOT NULL, + snapshot DATE NOT NULL, + reward_hash TEXT NOT NULL, + token TEXT NOT NULL, + amount TEXT NOT NULL + ) + ` + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_7_staging", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/8_goldFinal.go b/pkg/rewards/8_goldFinal.go new file mode 100644 index 00000000..7746faa7 --- /dev/null +++ b/pkg/rewards/8_goldFinal.go @@ -0,0 +1,39 @@ +package rewards + +const _8_goldFinalQuery = ` +insert into gold_final +SELECT + earner, + snapshot, + reward_hash, + token, + amount +FROM gold_7_staging +` + +func (rc *RewardsCalculator) GenerateGoldFinalTable() error { + res := rc.grm.Exec(_8_goldFinalQuery) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_final", "error", res.Error) + return res.Error + } + return nil +} + +func (rc *RewardsCalculator) Create8GoldTable() error { + query := ` + create table if not exists gold_table ( + earner TEXT NOT NULL, + snapshot DATE NOT NULL, + reward_hash TEXT NOT NULL, + token TEXT NOT NULL, + amount TEXT NOT NULL + ) + ` + res := rc.grm.Exec(query) + if res.Error != nil { + rc.logger.Sugar().Errorw("Failed to create gold_table", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/combinedRewards.go b/pkg/rewards/combinedRewards.go new file mode 100644 index 00000000..6b779c13 --- /dev/null +++ b/pkg/rewards/combinedRewards.go @@ -0,0 +1,100 @@ +package rewards + +import ( + "time" +) + +const rewardsCombinedQuery = ` + with combined_rewards as ( + select + rs.avs, + rs.reward_hash, + rs.token, + rs.amount, + rs.strategy, + rs.strategy_index, + rs.multiplier, + rs.start_timestamp, + rs.end_timestamp, + rs.duration, + rs.block_number, + b.block_time, + DATE(b.block_time) as block_date, + rs.reward_type, + ROW_NUMBER() OVER (PARTITION BY reward_hash, strategy_index ORDER BY block_number asc) as rn + from reward_submissions as rs + left join blocks as b on (b.number = rs.block_number) + ) + select * from combined_rewards + where rn = 1 +` + +type CombinedRewards struct { + Avs string + RewardHash string + Token string + Amount string + Strategy string + StrategyIndex uint64 + Multiplier string + StartTimestamp *time.Time `gorm:"type:DATETIME"` + EndTimestamp *time.Time `gorm:"type:DATETIME"` + Duration uint64 + BlockNumber uint64 + BlockDate string + BlockTime *time.Time `gorm:"type:DATETIME"` + RewardType string // avs, all_stakers, all_earners +} + +func (r *RewardsCalculator) GenerateCombinedRewards() ([]*CombinedRewards, error) { + combinedRewards := make([]*CombinedRewards, 0) + + res := r.grm.Raw(rewardsCombinedQuery).Scan(&combinedRewards) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to generate combined rewards", "error", res.Error) + return nil, res.Error + } + return combinedRewards, nil +} + +func (r *RewardsCalculator) GenerateAndInsertCombinedRewards() error { + combinedRewards, err := r.GenerateCombinedRewards() + if err != nil { + r.logger.Sugar().Errorw("Failed to generate combined rewards", "error", err) + return err + } + + r.logger.Sugar().Infow("Inserting combined rewards", "count", len(combinedRewards)) + res := r.grm.Model(&CombinedRewards{}).CreateInBatches(combinedRewards, 500) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to insert combined rewards", "error", res.Error) + return res.Error + } + return nil +} + +func (r *RewardsCalculator) CreateCombinedRewardsTable() error { + res := r.grm.Exec(` + CREATE TABLE IF NOT EXISTS combined_rewards ( + avs TEXT, + reward_hash TEXT, + token TEXT, + amount TEXT, + strategy TEXT, + strategy_index INTEGER, + multiplier TEXT, + start_timestamp DATETIME, + end_timestamp DATETIME, + duration INTEGER, + block_number INTEGER, + block_time DATETIME, + block_date DATE, + reward_type string + )`, + ) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to create combined_rewards table", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/combinedRewards_test.go b/pkg/rewards/combinedRewards_test.go new file mode 100644 index 00000000..f468a0f9 --- /dev/null +++ b/pkg/rewards/combinedRewards_test.go @@ -0,0 +1,114 @@ +package rewards + +import ( + "fmt" + "github.com/Layr-Labs/go-sidecar/internal/config" + "github.com/Layr-Labs/go-sidecar/internal/logger" + "github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations" + "github.com/Layr-Labs/go-sidecar/internal/tests" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "gorm.io/gorm" + "testing" +) + +func setupCombinedRewards() ( + string, + *config.Config, + *gorm.DB, + *zap.Logger, + error, +) { + cfg := tests.GetConfig() + l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) + + dbName, db, err := tests.GetFileBasedSqliteDatabaseConnection(l) + if err != nil { + panic(err) + } + sqliteMigrator := migrations.NewSqliteMigrator(db, l) + if err := sqliteMigrator.MigrateAll(); err != nil { + l.Sugar().Fatalw("Failed to migrate", "error", err) + } + + return dbName, cfg, db, l, err +} + +func teardownCombinedRewards(grm *gorm.DB) { + queries := []string{ + `delete from reward_submissions`, + `delete from blocks`, + } + for _, query := range queries { + if res := grm.Exec(query); res.Error != nil { + fmt.Printf("Failed to run query: %v\n", res.Error) + } + } +} + +func hydrateRewardSubmissionsTable(grm *gorm.DB, l *zap.Logger) error { + projectRoot := getProjectRootPath() + contents, err := tests.GetCombinedRewardsSqlFile(projectRoot) + + if err != nil { + return err + } + + res := grm.Exec(contents) + if res.Error != nil { + l.Sugar().Errorw("Failed to execute sql", "error", zap.Error(res.Error)) + return res.Error + } + return nil +} + +func Test_CombinedRewards(t *testing.T) { + if !rewardsTestsEnabled() { + t.Skipf("Skipping %s", t.Name()) + return + } + + dbFileName, cfg, grm, l, err := setupCombinedRewards() + + if err != nil { + t.Fatal(err) + } + + t.Run("Should hydrate blocks and reward_submissions tables", func(t *testing.T) { + err := hydrateAllBlocksTable(grm, l) + if err != nil { + t.Fatal(err) + } + + query := "select count(*) from blocks" + var count int + res := grm.Raw(query).Scan(&count) + assert.Nil(t, res.Error) + assert.Equal(t, TOTAL_BLOCK_COUNT, count) + + err = hydrateRewardSubmissionsTable(grm, l) + if err != nil { + t.Fatal(err) + } + + query = "select count(*) from reward_submissions" + res = grm.Raw(query).Scan(&count) + assert.Nil(t, res.Error) + assert.Equal(t, 192, count) + }) + t.Run("Should generate the proper combinedRewards", func(t *testing.T) { + rewards, _ := NewRewardsCalculator(l, grm, cfg) + + combinedRewards, err := rewards.GenerateCombinedRewards() + assert.Nil(t, err) + assert.NotNil(t, combinedRewards) + + t.Logf("Generated %d combinedRewards", len(combinedRewards)) + + assert.Equal(t, 192, len(combinedRewards)) + }) + t.Cleanup(func() { + tests.DeleteTestSqliteDB(dbFileName) + teardownCombinedRewards(grm) + }) +} diff --git a/pkg/rewards/operatorAvsRegistrationSnaphots.go b/pkg/rewards/operatorAvsRegistrationSnaphots.go new file mode 100644 index 00000000..9367cb99 --- /dev/null +++ b/pkg/rewards/operatorAvsRegistrationSnaphots.go @@ -0,0 +1,172 @@ +package rewards + +import ( + "database/sql" +) + +// Operator AVS Registration Windows: Ranges at which an operator has registered for an AVS +// 0. Ranked: Rank the operator state changes by block_time and log_index since sqlite lacks LEAD/LAG functions +// 1. Marked_statuses: Denote which registration status comes after one another +// 2. Removed_same_day_deregistrations: Remove a pairs of (registration, deregistration) that happen on the same day +// 3. Registration_periods: Combine registration together, only select registrations with: +// a. (Registered, Unregistered) +// b. (Registered, Null). If null, the end time is the current timestamp +// 4. Registration_snapshots: Round up each start_time to 0 UTC on NEXT DAY and round down each end_time to 0 UTC on CURRENT DAY +// 5. Operator_avs_registration_windows: Ranges that start and end on same day are invalid +// Note: We cannot assume that the operator is registered for avs at end_time because it is +// Payments calculations should only be done on snapshots from the PREVIOUS DAY. For example say we have the following: +// <-----0-------1-------2------3------> +// ^ ^ +// Entry Exit +// Since exits (deregistrations) are rounded down, we must only look at the day 2 snapshot on a pipeline run on day 3. +const operatorAvsRegistrationSnapshotsQuery = ` +WITH state_changes as ( + select + aosc.*, + b.block_time as block_time, + DATE(b.block_time) as block_date + from avs_operator_state_changes as aosc + left join blocks as b on (b.number = aosc.block_number) +), +marked_statuses as ( + SELECT + operator, + avs, + registered, + block_time, + DATE(block_time) as block_date, + -- Mark the next action as next_block_time + LEAD(block_time) OVER (PARTITION BY operator, avs ORDER BY block_time ASC, log_index ASC) AS next_block_time, + -- The below lead/lag combinations are only used in the next CTE + -- Get the next row's registered status and block_date + LEAD(registered) OVER (PARTITION BY operator, avs ORDER BY block_time ASC, log_index ASC) AS next_registration_status, + LEAD(block_date) OVER (PARTITION BY operator, avs ORDER BY block_time ASC, log_index ASC) AS next_block_date, + -- Get the previous row's registered status and block_date + LAG(registered) OVER (PARTITION BY operator, avs ORDER BY block_time ASC, log_index ASC) AS prev_registered, + LAG(block_date) OVER (PARTITION BY operator, avs ORDER BY block_time ASC, log_index ASC) AS prev_block_date + FROM state_changes +), +-- Ignore a (registration,deregistration) pairs that happen on the exact same date +removed_same_day_deregistrations as ( + SELECT * from marked_statuses + WHERE NOT ( + -- Remove the registration part + (registered = TRUE AND + COALESCE(next_registration_status = FALSE, false) AND -- default to false if null + COALESCE(block_date = next_block_date, false)) OR + -- Remove the deregistration part + (registered = FALSE AND + COALESCE(prev_registered = TRUE, false) and + COALESCE(block_date = prev_block_date, false) + ) + ) +), +-- Combine corresponding registrations into a single record +-- start_time is the beginning of the record +registration_periods AS ( + SELECT + operator, + avs, + block_time AS start_time, + -- Mark the next_block_time as the end_time for the range + -- Use coalesce because if the next_block_time for a registration is not closed, then we use cutoff_date + COALESCE(next_block_time, DATETIME(@cutoffDate)) AS end_time, + registered + FROM removed_same_day_deregistrations + WHERE registered = TRUE +), +-- Round UP each start_time and round DOWN each end_time +registration_windows_extra as ( + SELECT + operator, + avs, + DATE(start_time, '+1 day') as start_time, + -- End time is end time non inclusive because the operator is not registered on the AVS at the end time OR it is current timestamp rounded up + DATE(end_time) as end_time + FROM registration_periods +), +-- Ignore start_time and end_time that last less than a day +operator_avs_registration_windows as ( + SELECT * from registration_windows_extra + WHERE start_time != end_time +), +cleaned_records AS ( + SELECT * FROM operator_avs_registration_windows + WHERE start_time < end_time +), +date_bounds as ( + select + min(start_time) as min_start, + max(end_time) as max_end + from operator_avs_registration_windows +), +day_series AS ( + with RECURSIVE day_series_inner AS ( + SELECT DATE(min_start) AS day + FROM date_bounds + UNION ALL + SELECT DATE(day, '+1 day') + FROM day_series_inner + WHERE day < (SELECT max_end FROM date_bounds) + ) + select * from day_series_inner +), +final_results as ( + select + operator, + avs, + day as snapshot + from cleaned_records + cross join day_series +where DATE(day) between DATE(start_time) and DATE(end_time, '-1 day') +) +select * from final_results +` + +type OperatorAvsRegistrationSnapshots struct { + Avs string + Operator string + Snapshot string +} + +// GenerateOperatorAvsRegistrationSnapshots returns a list of OperatorAvsRegistrationSnapshots objects +func (r *RewardsCalculator) GenerateOperatorAvsRegistrationSnapshots(snapshotDate string) ([]*OperatorAvsRegistrationSnapshots, error) { + results := make([]*OperatorAvsRegistrationSnapshots, 0) + + res := r.grm.Raw(operatorAvsRegistrationSnapshotsQuery, sql.Named("cutoffDate", snapshotDate)).Scan(&results) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to generate operator AVS registration windows", "error", res.Error) + return nil, res.Error + } + return results, nil +} + +func (r *RewardsCalculator) GenerateAndInsertOperatorAvsRegistrationSnapshots(snapshotDate string) error { + snapshots, err := r.GenerateOperatorAvsRegistrationSnapshots(snapshotDate) + if err != nil { + r.logger.Sugar().Errorw("Failed to generate operator AVS registration snapshots", "error", err) + return err + } + + r.logger.Sugar().Infow("Inserting operator AVS registration snapshots", "count", len(snapshots)) + + res := r.grm.Model(&OperatorAvsRegistrationSnapshots{}).CreateInBatches(snapshots, 100) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to insert operator AVS registration snapshots", "error", res.Error) + return err + } + return nil +} + +func (r *RewardsCalculator) CreateOperatorAvsRegistrationSnapshotsTable() error { + res := r.grm.Exec(`CREATE TABLE IF NOT EXISTS operator_avs_registration_snapshots ( + avs TEXT, + operator TEXT, + snapshot TEXT + )`) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to create operator_avs_registration_snapshots table", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/operatorAvsRegistrationSnaphots_test.go b/pkg/rewards/operatorAvsRegistrationSnaphots_test.go new file mode 100644 index 00000000..4725ca12 --- /dev/null +++ b/pkg/rewards/operatorAvsRegistrationSnaphots_test.go @@ -0,0 +1,159 @@ +package rewards + +import ( + "fmt" + "github.com/Layr-Labs/go-sidecar/internal/config" + "github.com/Layr-Labs/go-sidecar/internal/logger" + "github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations" + "github.com/Layr-Labs/go-sidecar/internal/tests" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "gorm.io/gorm" + "slices" + "testing" +) + +func setupOperatorAvsRegistrationSnapshot() ( + string, + *config.Config, + *gorm.DB, + *zap.Logger, + error, +) { + cfg := tests.GetConfig() + l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) + + dbFileName, db, err := tests.GetFileBasedSqliteDatabaseConnection(l) + if err != nil { + panic(err) + } + sqliteMigrator := migrations.NewSqliteMigrator(db, l) + if err := sqliteMigrator.MigrateAll(); err != nil { + l.Sugar().Fatalw("Failed to migrate", "error", err) + } + + return dbFileName, cfg, db, l, err +} + +func teardownOperatorAvsRegistrationSnapshot(grm *gorm.DB) { + queries := []string{ + `delete from avs_operator_state_changes`, + `delete from blocks`, + } + for _, query := range queries { + if res := grm.Exec(query); res.Error != nil { + fmt.Printf("Failed to run query: %v\n", res.Error) + } + } +} + +func hydrateOperatorAvsStateChangesTable(grm *gorm.DB, l *zap.Logger) error { + projectRoot := getProjectRootPath() + contents, err := tests.GetOperatorAvsRegistrationsSqlFile(projectRoot) + + if err != nil { + return err + } + + res := grm.Exec(contents) + if res.Error != nil { + l.Sugar().Errorw("Failed to execute sql", "error", zap.Error(res.Error)) + return res.Error + } + return nil +} + +func Test_OperatorAvsRegistrationSnapshots(t *testing.T) { + if !rewardsTestsEnabled() { + t.Skipf("Skipping %s", t.Name()) + return + } + + projectRoot := getProjectRootPath() + dbFileName, cfg, grm, l, err := setupOperatorAvsRegistrationSnapshot() + + if err != nil { + t.Fatal(err) + } + + snapshotDate := "2024-09-01" + + t.Run("Should hydrate blocks and operatorAvsStateChanges tables", func(t *testing.T) { + err := hydrateAllBlocksTable(grm, l) + if err != nil { + t.Fatal(err) + } + + query := "select count(*) from blocks" + var count int + res := grm.Raw(query).Scan(&count) + assert.Nil(t, res.Error) + assert.Equal(t, TOTAL_BLOCK_COUNT, count) + + err = hydrateOperatorAvsStateChangesTable(grm, l) + if err != nil { + t.Fatal(err) + } + + query = "select count(*) from avs_operator_state_changes" + res = grm.Raw(query).Scan(&count) + assert.Nil(t, res.Error) + assert.Equal(t, 20442, count) + }) + t.Run("Should generate the proper operatorAvsRegistrationWindows", func(t *testing.T) { + rewards, _ := NewRewardsCalculator(l, grm, cfg) + + snapshots, err := rewards.GenerateOperatorAvsRegistrationSnapshots(snapshotDate) + assert.Nil(t, err) + assert.NotNil(t, snapshots) + + t.Logf("Generated %d snapshots", len(snapshots)) + + expectedResults, err := tests.GetExpectedOperatorAvsSnapshotResults(projectRoot) + assert.Nil(t, err) + + t.Logf("Expected %d snapshots", len(expectedResults)) + assert.Equal(t, len(expectedResults), len(snapshots)) + + lacksExpectedResult := make([]*OperatorAvsRegistrationSnapshots, 0) + + mappedExpectedResults := make(map[string][]string) + for _, expectedResult := range expectedResults { + slotId := fmt.Sprintf("%s_%s", expectedResult.Operator, expectedResult.Avs) + if _, ok := mappedExpectedResults[slotId]; !ok { + mappedExpectedResults[slotId] = make([]string, 0) + } + mappedExpectedResults[slotId] = append(mappedExpectedResults[slotId], expectedResult.Snapshot) + } + + // If the two result sets are different lengths, we need to find out why. + if len(expectedResults) != len(snapshots) { + // Go line-by-line in the window results and find the corresponding line in the expected results. + // If one doesnt exist, add it to the missing list. + for _, window := range snapshots { + slotId := fmt.Sprintf("%s_%s", window.Operator, window.Avs) + found, ok := mappedExpectedResults[slotId] + if !ok { + t.Logf("Operator/AVS not found in results: %+v\n", window) + lacksExpectedResult = append(lacksExpectedResult, window) + } else { + if !slices.Contains(found, window.Snapshot) { + t.Logf("Found operator/AVS, but no snapshot: %+v - %+v\n", window, found) + lacksExpectedResult = append(lacksExpectedResult, window) + } + } + } + assert.Equal(t, 0, len(lacksExpectedResult)) + + if len(lacksExpectedResult) > 0 { + for i, window := range lacksExpectedResult { + fmt.Printf("%d - Window: %+v\n", i, window) + } + } + } + }) + t.Cleanup(func() { + tests.DeleteTestSqliteDB(dbFileName) + teardownOperatorAvsRegistrationSnapshot(grm) + }) +} diff --git a/pkg/rewards/operatorAvsStrategySnapshots.go b/pkg/rewards/operatorAvsStrategySnapshots.go new file mode 100644 index 00000000..cfc8ce47 --- /dev/null +++ b/pkg/rewards/operatorAvsStrategySnapshots.go @@ -0,0 +1,231 @@ +package rewards + +import ( + "database/sql" +) + +// Operator AVS Strategy Windows: Ranges for which an Operator, Strategy is restaked on an AVS +// 1. Ranked_records: Order all records. Round up block_time to 0 UTC +// 2. Latest_records: Get latest records for each (operator, avs, strategy, day) combination +// 3. Grouped_records: Find the next start_time for each (operator, avs, strategy) combination +// 4. Parsed_ranges: Complicated step. Here, we set end_time = start_time in three cases: +// Case 1: end_time is null because there are no more RPC calls made. For example, if today is 4/28 and +// the start_time is 4/27 (rounded from 4/26), there is nothing we can do on the (4/27, 4/28) +// range since it has not ended. +// Case 2: end_time is null because the (operator, strategy) combo is no longer registered +// Case 3: end_time is more than 1 day greater than start_time. In this case, if there is a new range, +// it will be accounted for. Say we have a range (4/21, 4/22, 4/23), (4/23, 4/25), (4/25, 4/26). +// The second range will be discarded since its not contiguous. We will keep (4/21-4/23) and (4/25-4/26) +// 5. Active_windows: Parse out all rows whose start_time == end_time (see above conditions) +// 6. Gaps_and_islands: Mark the previous end time for each row. If null, then start of a range +// 7. Island_detection: Mark islands if the previous end time is equal to the start time +// 8. Island_groups: Group islands by summing up ids +// 9. Operator_avs_strategy_windows: Combine ranges with same id +const operatorAvsStrategyWindowsQuery = ` +with ranked_records AS ( + SELECT + lower(operator) as operator, + lower(avs) as avs, + lower(strategy) as strategy, + block_time, + DATE(block_time, '+1 day') as start_time, + ROW_NUMBER() OVER ( + PARTITION BY operator, avs, strategy, DATE(block_time, '+1 day') + ORDER BY block_time DESC -- want latest records to be ranked highest + ) AS rn + FROM operator_restaked_strategies + WHERE avs_directory_address = lower(@avsDirectoryAddress) +), +-- Get the latest records for each (operator, avs, strategy, day) combination +latest_records AS ( + SELECT + operator, + avs, + strategy, + start_time, + block_time, + rn + FROM ranked_records + WHERE rn = 1 +), +-- Find the next entry for each (operator,avs,strategy) grouping +grouped_records AS ( + SELECT + operator, + avs, + strategy, + start_time, + LEAD(start_time) OVER ( + PARTITION BY operator, avs, strategy + ORDER BY start_time ASC + ) AS next_start_time + FROM latest_records +), +-- Parse out any holes (ie. any next_start_times that are not exactly one day after the current start_time) +parsed_ranges AS ( + SELECT + operator, + avs, + strategy, + start_time, + -- If the next_start_time is not on the consecutive day, close off the end_time + CASE + WHEN next_start_time IS NULL OR next_start_time > DATE(start_time, '+1 day') THEN start_time + ELSE next_start_time + END AS end_time + FROM grouped_records +), +-- Remove the (operator,avs,strategy) combos where start_time == end_time +active_windows as ( + SELECT * + FROM parsed_ranges + WHERE start_time != end_time +), +partitioned_gaps_and_islands as ( + select + operator, + avs, + strategy, + start_time, + end_time, + ROW_NUMBER() OVER (PARTITION BY operator, avs, strategy ORDER BY start_time) as rn + from active_windows +), +-- Mark the prev_end_time for each row. If new window, then gap is empty +gaps_and_islands AS ( + SELECT + operator, + avs, + strategy, + start_time, + end_time, + LAG(end_time) OVER(PARTITION BY operator, avs, strategy ORDER BY start_time) as prev_end_time + FROM active_windows +), +-- Detect islands +island_detection AS ( + SELECT operator, avs, strategy, start_time, end_time, prev_end_time, + CASE + -- If the previous end time is equal to the start time, then mark as part of the island, else create a new island + WHEN prev_end_time = start_time THEN 0 + ELSE 1 + END as new_island + FROM gaps_and_islands +), +-- Group each based on their ID +island_groups AS ( + SELECT + t1.operator, + t1.avs, + t1.strategy, + t1.start_time, + t1.end_time, + ( + SELECT SUM(t2.new_island) + FROM island_detection t2 + WHERE t2.operator = t1.operator + AND t2.avs = t1.avs + AND t2.strategy = t1.strategy + AND t2.start_time <= t1.start_time + ) AS island_id + FROM island_detection t1 + ORDER BY t1.operator, t1.avs, t1.strategy, t1.start_time +), +operator_avs_strategy_windows AS ( + SELECT + operator, + avs, + strategy, + MIN(start_time) AS start_time, + MAX(end_time) AS end_time + FROM island_groups + GROUP BY operator, avs, strategy, island_id + ORDER BY operator, avs, strategy, start_time +), +cleaned_records AS ( + SELECT * + FROM operator_avs_strategy_windows + WHERE start_time < end_time +), +date_bounds as ( + select + min(start_time) as min_start, + max(end_time) as max_end + from cleaned_records +), +day_series AS ( + with RECURSIVE day_series_inner AS ( + SELECT DATE(min_start) AS day + FROM date_bounds + UNION ALL + SELECT DATE(day, '+1 day') + FROM day_series_inner + WHERE day < (SELECT max_end FROM date_bounds) + ) + select * from day_series_inner +), +final_results as ( + select + operator, + avs, + strategy, + day as snapshot + from cleaned_records + cross join day_series + where DATE(day) between DATE(start_time) and DATE(end_time, '-1 day') +) +select * from final_results; +` + +type OperatorAvsStrategySnapshot struct { + Operator string + Avs string + Strategy string + Snapshot string +} + +func (r *RewardsCalculator) GenerateOperatorAvsStrategySnapshots(snapshotDate string) ([]*OperatorAvsStrategySnapshot, error) { + results := make([]*OperatorAvsStrategySnapshot, 0) + + contractAddresses := r.globalConfig.GetContractsMapForChain() + + res := r.grm.Raw(operatorAvsStrategyWindowsQuery, sql.Named("avsDirectoryAddress", contractAddresses.AvsDirectory)).Scan(&results) + + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to generate operator AVS strategy windows", "error", res.Error) + return nil, res.Error + } + return results, nil +} + +func (r *RewardsCalculator) GenerateAndInsertOperatorAvsStrategySnapshots(snapshotDate string) error { + snapshots, err := r.GenerateOperatorAvsStrategySnapshots(snapshotDate) + if err != nil { + r.logger.Sugar().Errorw("Failed to generate operator AVS strategy snapshots", "error", err) + return err + } + + r.logger.Sugar().Infow("Inserting operator AVS strategy snapshots", "count", len(snapshots)) + res := r.grm.Model(&OperatorAvsStrategySnapshot{}).CreateInBatches(snapshots, 100) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to insert operator AVS strategy snapshots", "error", res.Error) + return res.Error + } + return nil +} + +func (r *RewardsCalculator) CreateOperatorAvsStrategySnapshotsTable() error { + res := r.grm.Exec(` + CREATE TABLE IF NOT EXISTS operator_avs_strategy_snapshots ( + operator VARCHAR(42) NOT NULL, + avs VARCHAR(42) NOT NULL, + strategy VARCHAR(42) NOT NULL, + snapshot DATE NOT NULL + ); + `) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to create operator_avs_strategy_snapshots table", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/operatorAvsStrategySnapshots_test.go b/pkg/rewards/operatorAvsStrategySnapshots_test.go new file mode 100644 index 00000000..5c1932e1 --- /dev/null +++ b/pkg/rewards/operatorAvsStrategySnapshots_test.go @@ -0,0 +1,148 @@ +package rewards + +import ( + "fmt" + "github.com/Layr-Labs/go-sidecar/internal/config" + "github.com/Layr-Labs/go-sidecar/internal/logger" + "github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations" + "github.com/Layr-Labs/go-sidecar/internal/tests" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "gorm.io/gorm" + "slices" + "strings" + "testing" +) + +func setupOperatorAvsStrategyWindows() ( + string, + *config.Config, + *gorm.DB, + *zap.Logger, + error, +) { + cfg := tests.GetConfig() + cfg.Chain = config.Chain_Holesky + + l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) + + dbFileName, db, err := tests.GetFileBasedSqliteDatabaseConnection(l) + if err != nil { + panic(err) + } + sqliteMigrator := migrations.NewSqliteMigrator(db, l) + if err := sqliteMigrator.MigrateAll(); err != nil { + l.Sugar().Fatalw("Failed to migrate", "error", err) + } + + return dbFileName, cfg, db, l, err +} + +func teardownOperatorAvsStrategyWindows(grm *gorm.DB) { + queries := []string{ + `delete from operator_avs_strategy_snapshots`, + } + for _, query := range queries { + grm.Exec(query) + } +} + +func hydrateOperatorAvsRestakedStrategies(grm *gorm.DB, l *zap.Logger) error { + projectRoot := getProjectRootPath() + contents, err := tests.GetOperatorAvsRestakedStrategiesSqlFile(projectRoot) + + if err != nil { + return err + } + + res := grm.Exec(contents) + if res.Error != nil { + l.Sugar().Errorw("Failed to execute sql", "error", zap.Error(res.Error)) + return res.Error + } + return nil +} + +func Test_OperatorAvsStrategySnapshots(t *testing.T) { + if !rewardsTestsEnabled() { + t.Skipf("Skipping %s", t.Name()) + return + } + + projectRoot := getProjectRootPath() + dbFileName, cfg, grm, l, err := setupOperatorAvsStrategyWindows() + + if err != nil { + t.Fatal(err) + } + + t.Run("Should hydrate dependency tables", func(t *testing.T) { + t.Log("Hydrating restaked strategies") + err := hydrateOperatorAvsRestakedStrategies(grm, l) + if err != nil { + t.Fatal(err) + } + + query := `select count(*) from operator_restaked_strategies` + var count int + res := grm.Raw(query).Scan(&count) + + assert.Nil(t, res.Error) + assert.Equal(t, 3144978, count) + }) + + t.Run("Should calculate correct operatorAvsStrategy windows", func(t *testing.T) { + rewards, _ := NewRewardsCalculator(l, grm, cfg) + + t.Log("Generating snapshots") + windows, err := rewards.GenerateOperatorAvsStrategySnapshots("2024-09-01") + assert.Nil(t, err) + + t.Log("Getting expected results") + expectedResults, err := tests.GetExpectedOperatorAvsSnapshots(projectRoot) + assert.Nil(t, err) + t.Logf("Loaded %d expected results", len(expectedResults)) + + assert.Equal(t, len(expectedResults), len(windows)) + + // Memoize to make lookups faster rather than n^2 + mappedExpectedResults := make(map[string][]string) + for _, r := range expectedResults { + slotId := strings.ToLower(fmt.Sprintf("%s_%s_%s", r.Operator, r.Avs, r.Strategy)) + val, ok := mappedExpectedResults[slotId] + if !ok { + mappedExpectedResults[slotId] = make([]string, 0) + } + mappedExpectedResults[slotId] = append(val, r.Snapshot) + } + + lacksExpectedResult := make([]*OperatorAvsStrategySnapshot, 0) + // Go line-by-line in the window results and find the corresponding line in the expected results. + // If one doesnt exist, add it to the missing list. + for _, window := range windows { + slotId := strings.ToLower(fmt.Sprintf("%s_%s_%s", window.Operator, window.Avs, window.Strategy)) + + found, ok := mappedExpectedResults[slotId] + if !ok { + lacksExpectedResult = append(lacksExpectedResult, window) + t.Logf("Could not find expected result for %+v", window) + continue + } + if !slices.Contains(found, window.Snapshot) { + t.Logf("Found result, but snapshot doesnt match: %+v - %v", window, found) + lacksExpectedResult = append(lacksExpectedResult, window) + } + } + assert.Equal(t, 0, len(lacksExpectedResult)) + + //if len(lacksExpectedResult) > 0 { + // for i, window := range lacksExpectedResult { + // fmt.Printf("%d - Snapshot: %+v\n", i, window) + // } + //} + }) + t.Cleanup(func() { + teardownOperatorAvsStrategyWindows(grm) + tests.DeleteTestSqliteDB(dbFileName) + }) +} diff --git a/pkg/rewards/operatorShareSnapshots.go b/pkg/rewards/operatorShareSnapshots.go new file mode 100644 index 00000000..ecc18525 --- /dev/null +++ b/pkg/rewards/operatorShareSnapshots.go @@ -0,0 +1,132 @@ +package rewards + +import ( + "database/sql" + "go.uber.org/zap" +) + +const operatorShareSnapshotsQuery = ` +with operator_shares_with_block_info as ( + select + os.operator, + os.strategy, + os.shares, + os.block_number, + b.block_time, + DATE(b.block_time) as block_date + from operator_shares as os + left join blocks as b on (b.number = os.block_number) +), +ranked_operator_records as ( + select + *, + ROW_NUMBER() OVER (PARTITION BY operator, strategy, block_date ORDER BY block_time DESC) as rn + from operator_shares_with_block_info as os +), +-- Get the latest record for each day & round up to the snapshot day +snapshotted_records as ( + SELECT + operator, + strategy, + shares, + block_time, + DATE(block_date, '+1 day') as snapshot_time + from ranked_operator_records + where rn = 1 +), +-- Get the range for each operator, strategy pairing +operator_share_windows as ( + SELECT + operator, strategy, shares, snapshot_time as start_time, + CASE + -- If the range does not have the end, use the current timestamp truncated to 0 UTC + WHEN LEAD(snapshot_time) OVER (PARTITION BY operator, strategy ORDER BY snapshot_time) is null THEN DATE(@cutoffDate) + ELSE LEAD(snapshot_time) OVER (PARTITION BY operator, strategy ORDER BY snapshot_time) + END AS end_time + FROM snapshotted_records +), +cleaned_records as ( + SELECT * FROM operator_share_windows + WHERE start_time < end_time +), +date_bounds as ( + select + min(start_time) as min_start, + max(end_time) as max_end + from cleaned_records +), +day_series AS ( + with RECURSIVE day_series_inner AS ( + SELECT DATE(min_start) AS day + FROM date_bounds + UNION ALL + SELECT DATE(day, '+1 day') + FROM day_series_inner + WHERE day < (SELECT max_end FROM date_bounds) + ) + select * from day_series_inner +), +final_results as ( + SELECT + operator, + strategy, + shares, + day as snapshot + FROM cleaned_records + cross join day_series + where DATE(day) between DATE(start_time) and DATE(end_time, '-1 day') +) +select * from final_results +` + +type OperatorShareSnapshots struct { + Operator string + Strategy string + Shares string + Snapshot string +} + +func (r *RewardsCalculator) GenerateOperatorShareSnapshots(snapshotDate string) ([]*OperatorShareSnapshots, error) { + results := make([]*OperatorShareSnapshots, 0) + + res := r.grm.Raw(operatorShareSnapshotsQuery, sql.Named("cutoffDate", snapshotDate)).Scan(&results) + + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to generate operator share snapshots", "error", res.Error) + return nil, res.Error + } + return results, nil +} + +func (r *RewardsCalculator) GenerateAndInsertOperatorShareSnapshots(snapshotDate string) error { + snapshots, err := r.GenerateOperatorShareSnapshots(snapshotDate) + if err != nil { + r.logger.Sugar().Errorw("Failed to generate operator share snapshots", "error", err) + return err + } + + r.logger.Sugar().Infow("Inserting operator share snapshots", "count", len(snapshots)) + res := r.grm.Model(&OperatorShareSnapshots{}).CreateInBatches(snapshots, 100) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to insert operator share snapshots", "error", res.Error) + return res.Error + } + + return nil +} + +func (r *RewardsCalculator) CreateOperatorSharesSnapshotsTable() error { + res := r.grm.Exec(` + CREATE TABLE IF NOT EXISTS operator_share_snapshots ( + operator TEXT, + strategy TEXT, + shares TEXT, + snapshot TEXT + ) + `) + if res.Error != nil { + r.logger.Error("Failed to create operator share snapshots table", zap.Error(res.Error)) + return res.Error + } + return nil +} diff --git a/pkg/rewards/operatorShareSnapshots_test.go b/pkg/rewards/operatorShareSnapshots_test.go new file mode 100644 index 00000000..89cd01ad --- /dev/null +++ b/pkg/rewards/operatorShareSnapshots_test.go @@ -0,0 +1,142 @@ +package rewards + +import ( + "fmt" + "github.com/Layr-Labs/go-sidecar/internal/config" + "github.com/Layr-Labs/go-sidecar/internal/logger" + "github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations" + "github.com/Layr-Labs/go-sidecar/internal/tests" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "gorm.io/gorm" + "testing" +) + +func setupOperatorShareSnapshot() ( + string, + *config.Config, + *gorm.DB, + *zap.Logger, + error, +) { + cfg := tests.GetConfig() + l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) + + dbFileName, db, err := tests.GetFileBasedSqliteDatabaseConnection(l) + if err != nil { + panic(err) + } + sqliteMigrator := migrations.NewSqliteMigrator(db, l) + if err := sqliteMigrator.MigrateAll(); err != nil { + l.Sugar().Fatalw("Failed to migrate", "error", err) + } + + return dbFileName, cfg, db, l, err +} + +func teardownOperatorShareSnapshot(grm *gorm.DB) { + queries := []string{ + `delete from operator_shares`, + `delete from blocks`, + } + for _, query := range queries { + if res := grm.Exec(query); res.Error != nil { + fmt.Printf("Failed to run query: %v\n", res.Error) + } + } +} + +func hydrateOperatorShares(grm *gorm.DB, l *zap.Logger) error { + projectRoot := getProjectRootPath() + contents, err := tests.GetOperatorSharesSqlFile(projectRoot) + + if err != nil { + return err + } + + res := grm.Exec(contents) + if res.Error != nil { + l.Sugar().Errorw("Failed to execute sql", "error", zap.Error(res.Error)) + return res.Error + } + return nil +} + +func Test_OperatorShareSnapshots(t *testing.T) { + if !rewardsTestsEnabled() { + t.Skipf("Skipping %s", t.Name()) + return + } + + projectRoot := getProjectRootPath() + dbFileName, cfg, grm, l, err := setupOperatorShareSnapshot() + + if err != nil { + t.Fatal(err) + } + + snapshotDate := "2024-09-01" + + t.Run("Should hydrate dependency tables", func(t *testing.T) { + if err = hydrateAllBlocksTable(grm, l); err != nil { + t.Error(err) + } + if err = hydrateOperatorShares(grm, l); err != nil { + t.Error(err) + } + }) + t.Run("Should generate operator share snapshots", func(t *testing.T) { + rewards, _ := NewRewardsCalculator(l, grm, cfg) + + t.Log("Generating operator share snapshots") + snapshots, err := rewards.GenerateOperatorShareSnapshots(snapshotDate) + assert.Nil(t, err) + + t.Log("Loading expected results") + expectedResults, err := tests.GetOperatorSharesExpectedResults(projectRoot) + assert.Nil(t, err) + + assert.Equal(t, len(expectedResults), len(snapshots)) + + mappedExpectedResults := make(map[string]string) + + for _, expectedResult := range expectedResults { + slotId := fmt.Sprintf("%s_%s_%s", expectedResult.Operator, expectedResult.Strategy, expectedResult.Snapshot) + mappedExpectedResults[slotId] = expectedResult.Shares + } + + if len(expectedResults) != len(snapshots) { + t.Errorf("Expected %d snapshots, got %d", len(expectedResults), len(snapshots)) + + lacksExpectedResult := make([]*OperatorShareSnapshots, 0) + // Go line-by-line in the snapshot results and find the corresponding line in the expected results. + // If one doesnt exist, add it to the missing list. + for _, snapshot := range snapshots { + + slotId := fmt.Sprintf("%s_%s_%s", snapshot.Operator, snapshot.Strategy, snapshot.Snapshot) + + found, ok := mappedExpectedResults[slotId] + if !ok { + t.Logf("Record not found %+v", snapshot) + lacksExpectedResult = append(lacksExpectedResult, snapshot) + continue + } + if found != snapshot.Shares { + t.Logf("Expected: %s, Got: %s for %+v", found, snapshot.Shares, snapshot) + lacksExpectedResult = append(lacksExpectedResult, snapshot) + } + } + assert.Equal(t, 0, len(lacksExpectedResult)) + + if len(lacksExpectedResult) > 0 { + for i, window := range lacksExpectedResult { + fmt.Printf("%d - Snapshot: %+v\n", i, window) + } + } + } + }) + t.Cleanup(func() { + teardownOperatorShareSnapshot(grm) + tests.DeleteTestSqliteDB(dbFileName) + }) +} diff --git a/pkg/rewards/rewards.go b/pkg/rewards/rewards.go new file mode 100644 index 00000000..f50b3b06 --- /dev/null +++ b/pkg/rewards/rewards.go @@ -0,0 +1,173 @@ +package rewards + +import ( + "fmt" + "github.com/Layr-Labs/go-sidecar/internal/config" + "github.com/Layr-Labs/go-sidecar/internal/eigenState/submittedDistributionRoots" + "go.uber.org/zap" + "gorm.io/gorm" + "time" +) + +type RewardsCalculator struct { + logger *zap.Logger + grm *gorm.DB + globalConfig *config.Config +} + +func NewRewardsCalculator( + l *zap.Logger, + grm *gorm.DB, + cfg *config.Config, +) (*RewardsCalculator, error) { + rc := &RewardsCalculator{ + logger: l, + grm: grm, + globalConfig: cfg, + } + + if err := rc.initializeRewardsSchema(); err != nil { + l.Sugar().Errorw("Failed to initialize rewards schema", zap.Error(err)) + return nil, err + } + return rc, nil +} + +// CalculateRewardsForSnapshot calculates the rewards for a given snapshot date. +// +// Rewards are calculated for the period between the last snapshot published on-chain +// via the DistributionRootSubmitted event and the desired snapshot date (exclusive). +// +// If there is no previous DistributionRoot, the rewards are calculated from EigenLayer Genesis. +func (rc *RewardsCalculator) CalculateRewardsForSnapshot(desiredSnapshotDate uint64) error { + // First make sure that the snapshot date is valid as provided. + // The time should be at 00:00:00 UTC. and should be in the past. + snapshotDate := time.Unix(int64(desiredSnapshotDate), 0).UTC() + + if !rc.isValidSnapshotDate(snapshotDate) { + return fmt.Errorf("invalid snapshot date '%s'", snapshotDate.String()) + } + + // Get the last snapshot date published on-chain. + distributionRoot, err := rc.getMostRecentDistributionRoot() + if err != nil { + rc.logger.Error("Failed to get the most recent distribution root", zap.Error(err)) + return err + } + + var lowerBoundBlockNumber uint64 + if distributionRoot != nil { + lowerBoundBlockNumber = distributionRoot.BlockNumber + } else { + lowerBoundBlockNumber = rc.globalConfig.GetGenesisBlockNumber() + } + + rc.logger.Sugar().Infow("Calculating rewards for snapshot date", + zap.String("snapshot_date", snapshotDate.String()), + zap.Uint64("lowerBoundBlockNumber", lowerBoundBlockNumber), + ) + + // Calculate the rewards for the period. + // TODO(seanmcgary): lower bound should be either 0 (i.e. 1970-01-01) or the snapshot of the previously calculated rewards. + return rc.calculateRewards("", snapshotDate) +} + +func (rc *RewardsCalculator) isValidSnapshotDate(snapshotDate time.Time) bool { + // Check if the snapshot date is in the past. + // The snapshot date should be at 00:00:00 UTC. + if snapshotDate.After(time.Now().UTC()) { + rc.logger.Error("Snapshot date is in the future") + return false + } + + if snapshotDate.Hour() != 0 || snapshotDate.Minute() != 0 || snapshotDate.Second() != 0 { + rc.logger.Error("Snapshot date is not at 00:00:00 UTC") + return false + } + + return true +} + +func (rc *RewardsCalculator) getMostRecentDistributionRoot() (*submittedDistributionRoots.SubmittedDistributionRoots, error) { + var distributionRoot *submittedDistributionRoots.SubmittedDistributionRoots + res := rc.grm.Model(&submittedDistributionRoots.SubmittedDistributionRoots{}).Order("block_number desc").First(&distributionRoot) + if res != nil { + return nil, res.Error + } + return distributionRoot, nil +} + +func (rc *RewardsCalculator) initializeRewardsSchema() error { + funcs := []func() error{ + rc.CreateOperatorAvsRegistrationSnapshotsTable, + rc.CreateOperatorAvsStrategySnapshotsTable, + rc.CreateOperatorSharesSnapshotsTable, + rc.CreateStakerShareSnapshotsTable, + rc.CreateStakerDelegationSnapshotsTable, + rc.CreateCombinedRewardsTable, + + // Gold tables + rc.CreateGold1ActiveRewardsTable, + rc.CreateGold2RewardAmountsTable, + rc.CreateGold3OperatorRewardsTable, + rc.CreateGold4RewardsForAllTable, + rc.CreateGold5RfaeStakersTable, + rc.CreateGold6RfaeOperatorsTable, + rc.CreateGold7StagingTable, + rc.Create8GoldTable, + } + for _, f := range funcs { + err := f() + if err != nil { + return err + } + } + return nil +} + +func (rc *RewardsCalculator) generateSnapshotData(snapshotDate string) error { + var err error + + if err = rc.GenerateAndInsertCombinedRewards(); err != nil { + rc.logger.Sugar().Errorw("Failed to generate combined rewards", "error", err) + return err + } + rc.logger.Sugar().Debugw("Generated combined rewards") + + if err = rc.GenerateAndInsertOperatorAvsRegistrationSnapshots(snapshotDate); err != nil { + rc.logger.Sugar().Errorw("Failed to generate operator AVS registration snapshots", "error", err) + return err + } + rc.logger.Sugar().Debugw("Generated operator AVS registration snapshots") + + if err = rc.GenerateAndInsertOperatorAvsStrategySnapshots(snapshotDate); err != nil { + rc.logger.Sugar().Errorw("Failed to generate operator AVS strategy snapshots", "error", err) + return err + } + rc.logger.Sugar().Debugw("Generated operator AVS strategy snapshots") + + if err = rc.GenerateAndInsertOperatorShareSnapshots(snapshotDate); err != nil { + rc.logger.Sugar().Errorw("Failed to generate operator share snapshots", "error", err) + return err + } + rc.logger.Sugar().Debugw("Generated operator share snapshots") + + if err = rc.GenerateAndInsertStakerShareSnapshots(snapshotDate); err != nil { + rc.logger.Sugar().Errorw("Failed to generate staker share snapshots", "error", err) + return err + } + rc.logger.Sugar().Debugw("Generated staker share snapshots") + + if err = rc.GenerateAndInsertStakerDelegationSnapshots(snapshotDate); err != nil { + rc.logger.Sugar().Errorw("Failed to generate staker delegation snapshots", "error", err) + return err + } + rc.logger.Sugar().Debugw("Generated staker delegation snapshots") + + return nil +} + +func (rc *RewardsCalculator) calculateRewards(previousSnapshotDate string, snapshotDate time.Time) error { + + return nil +} diff --git a/pkg/rewards/rewards_test.go b/pkg/rewards/rewards_test.go new file mode 100644 index 00000000..e9c497a3 --- /dev/null +++ b/pkg/rewards/rewards_test.go @@ -0,0 +1,191 @@ +package rewards + +import ( + "fmt" + "github.com/Layr-Labs/go-sidecar/internal/config" + "github.com/Layr-Labs/go-sidecar/internal/logger" + "github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations" + "github.com/Layr-Labs/go-sidecar/internal/tests" + "github.com/Layr-Labs/go-sidecar/internal/types/numbers" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "gorm.io/gorm" + "os" + "path/filepath" + "slices" + "testing" +) + +const TOTAL_BLOCK_COUNT = 1229187 + +func rewardsTestsEnabled() bool { + return os.Getenv("TEST_REWARDS") == "true" +} + +func getProjectRootPath() string { + wd, err := os.Getwd() + if err != nil { + panic(err) + } + p, err := filepath.Abs(fmt.Sprintf("%s/../..", wd)) + if err != nil { + panic(err) + } + return p +} + +func hydrateAllBlocksTable(grm *gorm.DB, l *zap.Logger) error { + projectRoot := getProjectRootPath() + contents, err := tests.GetAllBlocksSqlFile(projectRoot) + + if err != nil { + return err + } + + res := grm.Exec(contents) + if res.Error != nil { + l.Sugar().Errorw("Failed to execute sql", "error", zap.Error(res.Error)) + return res.Error + } + return nil +} + +func setupRewards() ( + string, + *config.Config, + *gorm.DB, + *zap.Logger, + error, +) { + cfg := tests.GetConfig() + cfg.Debug = true + l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) + + dbFileName, db, err := tests.GetFileBasedSqliteDatabaseConnection(l) + if err != nil { + panic(err) + } + sqliteMigrator := migrations.NewSqliteMigrator(db, l) + if err := sqliteMigrator.MigrateAll(); err != nil { + l.Sugar().Fatalw("Failed to migrate", "error", err) + } + + return dbFileName, cfg, db, l, err +} + +func teardownRewards(grm *gorm.DB) { + teardownOperatorAvsRegistrationSnapshot(grm) + teardownOperatorAvsStrategyWindows(grm) + teardownOperatorShareSnapshot(grm) + teardownStakerDelegationSnapshot(grm) + teardownStakerShareSnapshot(grm) +} + +func Test_Rewards(t *testing.T) { + if !rewardsTestsEnabled() { + t.Skipf("Skipping %s", t.Name()) + return + } + + if err := numbers.InitPython(); err != nil { + t.Fatal(err) + } + + dbFileName, cfg, grm, l, err := setupRewards() + fmt.Printf("Using db file: %+v\n", dbFileName) + + if err != nil { + t.Fatal(err) + } + + snapshotDate := "2024-09-01" + + t.Run("Should initialize the rewards calculator", func(t *testing.T) { + rc, err := NewRewardsCalculator(l, grm, cfg) + assert.Nil(t, err) + if err != nil { + t.Fatal(err) + } + assert.NotNil(t, rc) + + fmt.Printf("DB Path: %+v", dbFileName) + + query := `select name from main.sqlite_master where type = 'table' order by name asc` + type row struct{ Name string } + var tables []row + res := rc.grm.Raw(query).Scan(&tables) + assert.Nil(t, res.Error) + + expectedTables := []string{ + "combined_rewards", + "gold_1_active_rewards", + "gold_2_staker_reward_amounts", + "gold_3_operator_reward_amounts", + "gold_4_rewards_for_all", + "gold_5_rfae_stakers", + "gold_6_rfae_operators", + "gold_7_staging", + "gold_table", + "operator_avs_registration_snapshots", + "operator_avs_strategy_snapshots", + "operator_share_snapshots", + "staker_delegation_snapshots", + "staker_share_snapshots", + } + tablesList := make([]string, 0) + for i, table := range tables { + fmt.Printf("[%v]: %+v\n", i, table.Name) + tablesList = append(tablesList, table.Name) + } + + for _, table := range expectedTables { + assert.True(t, slices.Contains(tablesList, table)) + } + + // Setup all tables and source data + err = hydrateAllBlocksTable(grm, l) + assert.Nil(t, err) + + err = hydrateOperatorAvsStateChangesTable(grm, l) + assert.Nil(t, err) + + err = hydrateOperatorAvsRestakedStrategies(grm, l) + assert.Nil(t, err) + + err = hydrateOperatorShares(grm, l) + assert.Nil(t, err) + + err = hydrateStakerDelegations(grm, l) + assert.Nil(t, err) + + err = hydrateStakerShares(grm, l) + assert.Nil(t, err) + + err = hydrateRewardSubmissionsTable(grm, l) + assert.Nil(t, err) + + t.Log("Hydrated tables") + + // Generate snapshots + err = rc.generateSnapshotData(snapshotDate) + assert.Nil(t, err) + + t.Log("Generated and inserted snapshots") + + startDate := "1970-01-01" + err = rc.GenerateActiveRewards(snapshotDate, startDate) + assert.Nil(t, err) + + query = `select count(*) from gold_1_active_rewards` + var count int + res = rc.grm.Raw(query).Scan(&count) + assert.Nil(t, res.Error) + fmt.Printf("Count: %v\n", count) + + fmt.Printf("Done!\n\n") + t.Cleanup(func() { + teardownRewards(grm) + // tests.DeleteTestSqliteDB(dbFileName) + }) + }) +} diff --git a/pkg/rewards/stakerDelegationSnapshots.go b/pkg/rewards/stakerDelegationSnapshots.go new file mode 100644 index 00000000..6e7e56b4 --- /dev/null +++ b/pkg/rewards/stakerDelegationSnapshots.go @@ -0,0 +1,124 @@ +package rewards + +import "database/sql" + +const stakerDelegationSnapshotsQuery = ` +with staker_delegations_with_block_info as ( + select + sdc.staker, + case when sdc.delegated = false then '0x0000000000000000000000000000000000000000' else sdc.operator end as operator, + sdc.log_index, + sdc.block_number, + b.block_time, + DATE(b.block_time) as block_date + from staker_delegation_changes as sdc + left join blocks as b on (b.number = sdc.block_number) +), +ranked_staker_records as ( +SELECT *, + ROW_NUMBER() OVER (PARTITION BY staker, block_date ORDER BY block_time DESC, log_index desc) AS rn +FROM staker_delegations_with_block_info +), +-- Get the latest record for each day & round up to the snapshot day +snapshotted_records as ( + SELECT + staker, + operator, + block_time, + DATE(block_date, '+1 day') as snapshot_time + from ranked_staker_records + where rn = 1 +), +-- Get the range for each operator, strategy pairing +staker_share_windows as ( + SELECT + staker, operator, snapshot_time as start_time, + CASE + -- If the range does not have the end, use the current timestamp truncated to 0 UTC + WHEN LEAD(snapshot_time) OVER (PARTITION BY staker ORDER BY snapshot_time) is null THEN DATE(@cutoffDate) + ELSE LEAD(snapshot_time) OVER (PARTITION BY staker ORDER BY snapshot_time) + END AS end_time + FROM snapshotted_records +), +cleaned_records as ( + SELECT * FROM staker_share_windows + WHERE start_time < end_time +), +date_bounds as ( + select + min(start_time) as min_start, + max(end_time) as max_end + from cleaned_records +), +day_series AS ( + with RECURSIVE day_series_inner AS ( + SELECT DATE(min_start) AS day + FROM date_bounds + UNION ALL + SELECT DATE(day, '+1 day') + FROM day_series_inner + WHERE day < (SELECT max_end FROM date_bounds) + ) + select * from day_series_inner +), +final_results as ( + SELECT + staker, + operator, + day as snapshot + FROM cleaned_records + cross join day_series + where DATE(day) between DATE(start_time) and DATE(end_time, '-1 day') +) +select * from final_results +` + +type StakerDelegationSnapshot struct { + Staker string + Operator string + Snapshot string +} + +func (r *RewardsCalculator) GenerateStakerDelegationSnapshots(snapshotDate string) ([]*StakerDelegationSnapshot, error) { + results := make([]*StakerDelegationSnapshot, 0) + + res := r.grm.Raw(stakerDelegationSnapshotsQuery, sql.Named("cutoffDate", snapshotDate)).Scan(&results) + + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to generate staker delegation snapshots", "error", res.Error) + return nil, res.Error + } + return results, nil +} + +func (r *RewardsCalculator) GenerateAndInsertStakerDelegationSnapshots(snapshotDate string) error { + snapshots, err := r.GenerateStakerDelegationSnapshots(snapshotDate) + if err != nil { + r.logger.Sugar().Errorw("Failed to generate staker delegation snapshots", "error", err) + return err + } + + r.logger.Sugar().Infow("Inserting staker delegation snapshots", "count", len(snapshots)) + res := r.grm.Model(&StakerDelegationSnapshot{}).CreateInBatches(snapshots, 100) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to insert staker delegation snapshots", "error", res.Error) + return res.Error + } + + return nil +} + +func (r *RewardsCalculator) CreateStakerDelegationSnapshotsTable() error { + res := r.grm.Exec(` + CREATE TABLE IF NOT EXISTS staker_delegation_snapshots ( + staker TEXT, + operator TEXT, + snapshot TEXT + ) + `) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to create staker delegation snapshots table", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/stakerDelegationSnapshots_test.go b/pkg/rewards/stakerDelegationSnapshots_test.go new file mode 100644 index 00000000..10850c13 --- /dev/null +++ b/pkg/rewards/stakerDelegationSnapshots_test.go @@ -0,0 +1,143 @@ +package rewards + +import ( + "fmt" + "github.com/Layr-Labs/go-sidecar/internal/config" + "github.com/Layr-Labs/go-sidecar/internal/logger" + "github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations" + "github.com/Layr-Labs/go-sidecar/internal/tests" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "gorm.io/gorm" + "slices" + "testing" +) + +func setupStakerDelegationSnapshot() ( + string, + *config.Config, + *gorm.DB, + *zap.Logger, + error, +) { + cfg := tests.GetConfig() + l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) + + dbFileName, db, err := tests.GetFileBasedSqliteDatabaseConnection(l) + if err != nil { + panic(err) + } + sqliteMigrator := migrations.NewSqliteMigrator(db, l) + if err := sqliteMigrator.MigrateAll(); err != nil { + l.Sugar().Fatalw("Failed to migrate", "error", err) + } + + return dbFileName, cfg, db, l, err +} + +func teardownStakerDelegationSnapshot(grm *gorm.DB) { + queries := []string{ + `delete from staker_delegation_changes`, + `delete from blocks`, + } + for _, query := range queries { + if res := grm.Exec(query); res.Error != nil { + fmt.Printf("Failed to run query: %v\n", res.Error) + } + } +} + +func hydrateStakerDelegations(grm *gorm.DB, l *zap.Logger) error { + projectRoot := getProjectRootPath() + contents, err := tests.GetStakerDelegationsSqlFile(projectRoot) + + if err != nil { + return err + } + + res := grm.Exec(contents) + if res.Error != nil { + l.Sugar().Errorw("Failed to execute sql", "error", zap.Error(res.Error)) + return res.Error + } + return nil +} + +func Test_StakerDelegationSnapshots(t *testing.T) { + if !rewardsTestsEnabled() { + t.Skipf("Skipping %s", t.Name()) + return + } + + projectRoot := getProjectRootPath() + dbFileName, cfg, grm, l, err := setupStakerDelegationSnapshot() + + if err != nil { + t.Fatal(err) + } + + snapshotDate := "2024-09-01" + + t.Run("Should hydrate dependency tables", func(t *testing.T) { + if err := hydrateAllBlocksTable(grm, l); err != nil { + t.Error(err) + } + if err := hydrateStakerDelegations(grm, l); err != nil { + t.Error(err) + } + }) + t.Run("Should generate staker share snapshots", func(t *testing.T) { + rewards, _ := NewRewardsCalculator(l, grm, cfg) + + t.Log("Generating staker delegation snapshots") + snapshots, err := rewards.GenerateStakerDelegationSnapshots(snapshotDate) + assert.Nil(t, err) + + t.Log("Getting expected results") + expectedResults, err := tests.GetStakerDelegationExpectedResults(projectRoot) + assert.Nil(t, err) + + assert.Equal(t, len(expectedResults), len(snapshots)) + + mappedExpectedResults := make(map[string][]string) + for _, expectedResult := range expectedResults { + slotId := fmt.Sprintf("%s_%s", expectedResult.Staker, expectedResult.Operator) + if _, ok := mappedExpectedResults[slotId]; !ok { + mappedExpectedResults[slotId] = make([]string, 0) + } + mappedExpectedResults[slotId] = append(mappedExpectedResults[slotId], expectedResult.Snapshot) + } + + if len(expectedResults) != len(snapshots) { + t.Errorf("Expected %d snapshots, got %d", len(expectedResults), len(snapshots)) + + lacksExpectedResult := make([]*StakerDelegationSnapshot, 0) + // Go line-by-line in the snapshot results and find the corresponding line in the expected results. + // If one doesnt exist, add it to the missing list. + for _, snapshot := range snapshots { + slotId := fmt.Sprintf("%s_%s", snapshot.Staker, snapshot.Operator) + found, ok := mappedExpectedResults[slotId] + if !ok { + t.Logf("Staker/operator not found in results: %+v\n", snapshot) + lacksExpectedResult = append(lacksExpectedResult, snapshot) + } else { + if !slices.Contains(found, snapshot.Snapshot) { + t.Logf("Found staker operator, but no snapshot: %+v - %+v\n", snapshot, found) + lacksExpectedResult = append(lacksExpectedResult, snapshot) + } + } + } + assert.Equal(t, 0, len(lacksExpectedResult)) + + if len(lacksExpectedResult) > 0 { + for i, window := range lacksExpectedResult { + fmt.Printf("%d - Snapshot: %+v\n", i, window) + } + } + } + }) + t.Cleanup(func() { + teardownStakerDelegationSnapshot(grm) + tests.DeleteTestSqliteDB(dbFileName) + }) +} diff --git a/pkg/rewards/stakerShareSnapshots.go b/pkg/rewards/stakerShareSnapshots.go new file mode 100644 index 00000000..6b92846c --- /dev/null +++ b/pkg/rewards/stakerShareSnapshots.go @@ -0,0 +1,127 @@ +package rewards + +import "database/sql" + +const stakerShareSnapshotsQuery = ` +with staker_shares_with_block_info as ( + select + ss.staker, + ss.strategy, + ss.shares, + ss.block_number, + b.block_time, + DATE(b.block_time) as block_date + from staker_shares as ss + left join blocks as b on (b.number = ss.block_number) +), +ranked_staker_records as ( +SELECT *, + ROW_NUMBER() OVER (PARTITION BY staker, strategy, block_date ORDER BY block_time DESC) AS rn +FROM staker_shares_with_block_info +), +-- Get the latest record for each day & round up to the snapshot day +snapshotted_records as ( + SELECT + staker, + strategy, + shares, + block_time, + DATE(block_date, '+1 day') as snapshot_time + from ranked_staker_records + where rn = 1 +), +-- Get the range for each operator, strategy pairing +staker_share_windows as ( + SELECT + staker, strategy, shares, snapshot_time as start_time, + CASE + -- If the range does not have the end, use the current timestamp truncated to 0 UTC + WHEN LEAD(snapshot_time) OVER (PARTITION BY staker, strategy ORDER BY snapshot_time) is null THEN DATE(@cutoffDate) + ELSE LEAD(snapshot_time) OVER (PARTITION BY staker, strategy ORDER BY snapshot_time) + END AS end_time + FROM snapshotted_records +), +cleaned_records as ( + SELECT * FROM staker_share_windows + WHERE start_time < end_time +), +date_bounds as ( + select + min(start_time) as min_start, + max(end_time) as max_end + from cleaned_records +), +day_series AS ( + with RECURSIVE day_series_inner AS ( + SELECT DATE(min_start) AS day + FROM date_bounds + UNION ALL + SELECT DATE(day, '+1 day') + FROM day_series_inner + WHERE day < (SELECT max_end FROM date_bounds) + ) + select * from day_series_inner +), +final_results as ( + SELECT + staker, + strategy, + shares, + day as snapshot + FROM cleaned_records + cross join day_series + where DATE(day) between DATE(start_time) and DATE(end_time, '-1 day') +) +select * from final_results +` + +type StakerShareSnapshot struct { + Staker string + Strategy string + Snapshot string + Shares string +} + +func (r *RewardsCalculator) GenerateStakerShareSnapshots(snapshotDate string) ([]*StakerShareSnapshot, error) { + results := make([]*StakerShareSnapshot, 0) + + res := r.grm.Raw(stakerShareSnapshotsQuery, sql.Named("cutoffDate", snapshotDate)).Scan(&results) + + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to generate staker share snapshots", "error", res.Error) + return nil, res.Error + } + return results, nil +} + +func (r *RewardsCalculator) GenerateAndInsertStakerShareSnapshots(snapshotDate string) error { + snapshots, err := r.GenerateStakerShareSnapshots(snapshotDate) + if err != nil { + r.logger.Sugar().Errorw("Failed to generate staker share snapshots", "error", err) + return err + } + + r.logger.Sugar().Infow("Inserting staker share snapshots", "count", len(snapshots)) + res := r.grm.Model(&StakerShareSnapshot{}).CreateInBatches(snapshots, 100) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to insert staker share snapshots", "error", res.Error) + return res.Error + } + return nil +} + +func (r *RewardsCalculator) CreateStakerShareSnapshotsTable() error { + res := r.grm.Exec(` + CREATE TABLE IF NOT EXISTS staker_share_snapshots ( + staker TEXT, + strategy TEXT, + shares TEXT, + snapshot TEXT + ) + `) + if res.Error != nil { + r.logger.Sugar().Errorw("Failed to create staker share snapshots table", "error", res.Error) + return res.Error + } + return nil +} diff --git a/pkg/rewards/stakerShareSnapshots_test.go b/pkg/rewards/stakerShareSnapshots_test.go new file mode 100644 index 00000000..0395d226 --- /dev/null +++ b/pkg/rewards/stakerShareSnapshots_test.go @@ -0,0 +1,140 @@ +package rewards + +import ( + "fmt" + "github.com/Layr-Labs/go-sidecar/internal/config" + "github.com/Layr-Labs/go-sidecar/internal/logger" + "github.com/Layr-Labs/go-sidecar/internal/sqlite/migrations" + "github.com/Layr-Labs/go-sidecar/internal/tests" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" + "gorm.io/gorm" + "testing" +) + +func setupStakerShareSnapshot() ( + string, + *config.Config, + *gorm.DB, + *zap.Logger, + error, +) { + cfg := tests.GetConfig() + l, _ := logger.NewLogger(&logger.LoggerConfig{Debug: cfg.Debug}) + + dbFileName, db, err := tests.GetFileBasedSqliteDatabaseConnection(l) + if err != nil { + panic(err) + } + sqliteMigrator := migrations.NewSqliteMigrator(db, l) + if err := sqliteMigrator.MigrateAll(); err != nil { + l.Sugar().Fatalw("Failed to migrate", "error", err) + } + + return dbFileName, cfg, db, l, err +} + +func teardownStakerShareSnapshot(grm *gorm.DB) { + queries := []string{ + `delete from staker_shares`, + `delete from blocks`, + } + for _, query := range queries { + if res := grm.Exec(query); res.Error != nil { + fmt.Printf("Failed to run query: %v\n", res.Error) + } + } +} + +func hydrateStakerShares(grm *gorm.DB, l *zap.Logger) error { + projectRoot := getProjectRootPath() + contents, err := tests.GetStakerSharesSqlFile(projectRoot) + + if err != nil { + return err + } + + res := grm.Exec(contents) + if res.Error != nil { + l.Sugar().Errorw("Failed to execute sql", "error", zap.Error(res.Error), zap.String("query", contents)) + return res.Error + } + return nil +} + +func Test_StakerShareSnapshots(t *testing.T) { + if !rewardsTestsEnabled() { + t.Skipf("Skipping %s", t.Name()) + return + } + + projectRoot := getProjectRootPath() + dbFileName, cfg, grm, l, err := setupStakerShareSnapshot() + + if err != nil { + t.Fatal(err) + } + + snapshotDate := "2024-09-01" + + t.Run("Should hydrate dependency tables", func(t *testing.T) { + if err = hydrateAllBlocksTable(grm, l); err != nil { + t.Error(err) + } + if err = hydrateStakerShares(grm, l); err != nil { + t.Error(err) + } + }) + t.Run("Should generate staker share snapshots", func(t *testing.T) { + rewards, _ := NewRewardsCalculator(l, grm, cfg) + + t.Log("Generating staker share snapshots") + snapshots, err := rewards.GenerateStakerShareSnapshots(snapshotDate) + assert.Nil(t, err) + + t.Log("Getting expected results") + expectedResults, err := tests.GetStakerSharesExpectedResults(projectRoot) + assert.Nil(t, err) + + assert.Equal(t, len(expectedResults), len(snapshots)) + + t.Log("Comparing results") + mappedExpectedResults := make(map[string]string) + for _, expectedResult := range expectedResults { + slotId := fmt.Sprintf("%s_%s_%s", expectedResult.Staker, expectedResult.Strategy, expectedResult.Snapshot) + mappedExpectedResults[slotId] = expectedResult.Shares + } + + if len(expectedResults) != len(snapshots) { + t.Errorf("Expected %d snapshots, got %d", len(expectedResults), len(snapshots)) + + lacksExpectedResult := make([]*StakerShareSnapshot, 0) + // Go line-by-line in the snapshot results and find the corresponding line in the expected results. + // If one doesnt exist, add it to the missing list. + for _, snapshot := range snapshots { + slotId := fmt.Sprintf("%s_%s_%s", snapshot.Staker, snapshot.Strategy, snapshot.Snapshot) + + found, ok := mappedExpectedResults[slotId] + if !ok { + lacksExpectedResult = append(lacksExpectedResult, snapshot) + continue + } + if found != snapshot.Shares { + t.Logf("Record found, but shares dont match. Expected %s, got %+v", found, snapshot) + lacksExpectedResult = append(lacksExpectedResult, snapshot) + } + } + assert.Equal(t, 0, len(lacksExpectedResult)) + + if len(lacksExpectedResult) > 0 { + for i, window := range lacksExpectedResult { + fmt.Printf("%d - Snapshot: %+v\n", i, window) + } + } + } + }) + t.Cleanup(func() { + teardownStakerShareSnapshot(grm) + tests.DeleteTestSqliteDB(dbFileName) + }) +} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 5e9ffea9..abb779f8 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -19,6 +19,15 @@ func Filter[A any](coll []A, criteria func(i A) bool) []A { return out } +func Find[A any](coll []*A, criteria func(i *A) bool) *A { + for _, item := range coll { + if criteria(item) { + return item + } + } + return nil +} + func Reduce[A any, B any](coll []A, processor func(accum B, next A) B, initialState B) B { val := initialState for _, item := range coll { diff --git a/scripts/downloadTestData.sh b/scripts/downloadTestData.sh new file mode 100755 index 00000000..39f07f1f --- /dev/null +++ b/scripts/downloadTestData.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +version=$(cat .testdataVersion) +bucketName="eigenlayer-sidecar-testdata" + +dataUrl="https://${bucketName}.s3.amazonaws.com/${version}.tar" + +if [[ -z $version ]]; then + echo "No version found in .testdataVersion" + exit 1 +fi +echo "Downloading testdata version $dataUrl" +curl -L $dataUrl | tar xvz -C ./ diff --git a/scripts/updateTestData.sh b/scripts/updateTestData.sh new file mode 100755 index 00000000..7daacbbd --- /dev/null +++ b/scripts/updateTestData.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +bucketName="eigenlayer-sidecar-testdata" +testdataVersionFile="./.testdataVersion" + +if git status --porcelain | grep -q .; +then + echo "You have uncommitted changes. Please commit or stash them before running this script." + # exit 1 +fi + +newVersion=$(git rev-parse HEAD) + +currentVersion=$(cat $testdataVersionFile) +if [[ -z $currentVersion ]]; then + echo "No current version found" +else + echo "Current version: $currentVersion" +fi + +if [[ $currentVersion == $newVersion ]]; then + echo "Current version is the same as the new version. Exiting." + exit 0 +fi + +echo "New version: $newVersion" + +tar -cvf "${newVersion}.tar" internal/tests/testdata + +aws s3 cp "${newVersion}.tar" "s3://${bucketName}/" + +rm "${newVersion}.tar" + +echo -n $newVersion > $testdataVersionFile + +git add . +git commit -m "Updated testdata version to $newVersion" + diff --git a/sql/schema.sql b/sql/schema.sql deleted file mode 100644 index 8c391d37..00000000 --- a/sql/schema.sql +++ /dev/null @@ -1,737 +0,0 @@ --- --- PostgreSQL database dump --- - --- Dumped from database version 14.11 (Homebrew) --- Dumped by pg_dump version 15.8 (Homebrew) - -SET statement_timeout = 0; -SET lock_timeout = 0; -SET idle_in_transaction_session_timeout = 0; -SET client_encoding = 'UTF8'; -SET standard_conforming_strings = on; -SELECT pg_catalog.set_config('search_path', '', false); -SET check_function_bodies = false; -SET xmloption = content; -SET client_min_messages = warning; -SET row_security = off; - --- --- Name: public; Type: SCHEMA; Schema: -; Owner: - --- - --- *not* creating schema, since initdb creates it - - -SET default_tablespace = ''; - -SET default_table_access_method = heap; - --- --- Name: avs_operator_changes; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.avs_operator_changes ( - id integer NOT NULL, - operator character varying, - avs character varying, - registered boolean, - transaction_hash character varying, - transaction_index bigint, - log_index bigint, - block_number bigint, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP -); - - --- --- Name: avs_operator_changes_id_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE public.avs_operator_changes_id_seq - AS integer - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: avs_operator_changes_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - --- - -ALTER SEQUENCE public.avs_operator_changes_id_seq OWNED BY public.avs_operator_changes.id; - - --- --- Name: blocks; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.blocks ( - number bigint NOT NULL, - hash character varying(255) NOT NULL, - blob_path text NOT NULL, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP, - updated_at timestamp with time zone, - deleted_at timestamp with time zone, - block_time timestamp with time zone NOT NULL -); - - --- --- Name: contracts; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.contracts ( - contract_address character varying(255) NOT NULL, - contract_abi text, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP, - updated_at timestamp with time zone, - deleted_at timestamp with time zone, - bytecode_hash character varying(64) DEFAULT NULL::character varying, - verified boolean DEFAULT false, - matching_contract_address character varying(255) DEFAULT NULL::character varying, - checked_for_proxy boolean DEFAULT false NOT NULL, - id integer NOT NULL, - checked_for_abi boolean -); - - --- --- Name: contracts_id_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE public.contracts_id_seq - AS integer - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: contracts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - --- - -ALTER SEQUENCE public.contracts_id_seq OWNED BY public.contracts.id; - - --- --- Name: delegated_stakers; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.delegated_stakers ( - staker character varying, - operator character varying, - block_number bigint, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP -); - - --- --- Name: migrations; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.migrations ( - name text NOT NULL, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP, - updated_at timestamp with time zone -); - - --- --- Name: operator_restaked_strategies; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.operator_restaked_strategies ( - id integer NOT NULL, - block_number bigint NOT NULL, - operator character varying NOT NULL, - avs character varying NOT NULL, - strategy character varying NOT NULL, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP, - updated_at timestamp with time zone, - deleted_at timestamp with time zone, - block_time timestamp with time zone NOT NULL, - avs_directory_address character varying -); - - --- --- Name: operator_restaked_strategies_id_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE public.operator_restaked_strategies_id_seq - AS integer - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: operator_restaked_strategies_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - --- - -ALTER SEQUENCE public.operator_restaked_strategies_id_seq OWNED BY public.operator_restaked_strategies.id; - - --- --- Name: operator_share_changes; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.operator_share_changes ( - id integer NOT NULL, - operator character varying, - strategy character varying, - shares numeric, - transaction_hash character varying, - transaction_index bigint, - log_index bigint, - block_number bigint, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP -); - - --- --- Name: operator_share_changes_id_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE public.operator_share_changes_id_seq - AS integer - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: operator_share_changes_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - --- - -ALTER SEQUENCE public.operator_share_changes_id_seq OWNED BY public.operator_share_changes.id; - - --- --- Name: operator_shares; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.operator_shares ( - operator character varying, - strategy character varying, - shares numeric, - block_number bigint, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP -); - - --- --- Name: proxy_contracts; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.proxy_contracts ( - block_number bigint NOT NULL, - contract_address character varying(255) NOT NULL, - proxy_contract_address character varying(255) NOT NULL, - created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, - updated_at timestamp with time zone, - deleted_at timestamp with time zone -); - - --- --- Name: registered_avs_operators; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.registered_avs_operators ( - operator character varying, - avs character varying, - block_number bigint, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP -); - - --- --- Name: staker_delegation_changes; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.staker_delegation_changes ( - staker character varying, - operator character varying, - delegated boolean, - transaction_hash character varying, - log_index bigint, - transaction_index bigint, - block_number bigint, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP -); - - --- --- Name: transaction_logs; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.transaction_logs ( - transaction_hash character varying(255) NOT NULL, - address character varying(255) NOT NULL, - arguments jsonb, - event_name character varying(255) NOT NULL, - log_index bigint NOT NULL, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP, - updated_at timestamp with time zone, - deleted_at timestamp with time zone, - block_number bigint NOT NULL, - transaction_index integer NOT NULL, - output_data jsonb -); - - --- --- Name: transactions; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.transactions ( - block_number bigint NOT NULL, - transaction_hash character varying(255) NOT NULL, - transaction_index bigint NOT NULL, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP, - updated_at timestamp with time zone, - deleted_at timestamp with time zone, - from_address character varying(255) NOT NULL, - to_address character varying(255) DEFAULT NULL::character varying, - contract_address character varying(255) DEFAULT NULL::character varying, - bytecode_hash character varying(64) DEFAULT NULL::character varying, - gas_used numeric, - cumulative_gas_used numeric, - effective_gas_price numeric -); - - --- --- Name: unverified_contracts; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.unverified_contracts ( - contract_address character varying(255) NOT NULL, - created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP, - updated_at timestamp with time zone, - deleted_at timestamp with time zone -); - - --- --- Name: avs_operator_changes id; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.avs_operator_changes ALTER COLUMN id SET DEFAULT nextval('public.avs_operator_changes_id_seq'::regclass); - - --- --- Name: contracts id; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.contracts ALTER COLUMN id SET DEFAULT nextval('public.contracts_id_seq'::regclass); - - --- --- Name: operator_restaked_strategies id; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.operator_restaked_strategies ALTER COLUMN id SET DEFAULT nextval('public.operator_restaked_strategies_id_seq'::regclass); - - --- --- Name: operator_share_changes id; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.operator_share_changes ALTER COLUMN id SET DEFAULT nextval('public.operator_share_changes_id_seq'::regclass); - - --- --- Name: avs_operator_changes avs_operator_changes_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.avs_operator_changes - ADD CONSTRAINT avs_operator_changes_pkey PRIMARY KEY (id); - - --- --- Name: blocks block_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.blocks - ADD CONSTRAINT block_pkey PRIMARY KEY (number); - - --- --- Name: blocks blocks_unique_block_number_hash; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.blocks - ADD CONSTRAINT blocks_unique_block_number_hash UNIQUE (number, hash); - - --- --- Name: contracts contracts_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.contracts - ADD CONSTRAINT contracts_pkey PRIMARY KEY (contract_address); - - --- --- Name: delegated_stakers delegated_stakers_staker_operator_block_number_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.delegated_stakers - ADD CONSTRAINT delegated_stakers_staker_operator_block_number_key UNIQUE (staker, operator, block_number); - - --- --- Name: migrations migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.migrations - ADD CONSTRAINT migrations_pkey PRIMARY KEY (name); - - --- --- Name: operator_restaked_strategies operator_restaked_strategies_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.operator_restaked_strategies - ADD CONSTRAINT operator_restaked_strategies_pkey PRIMARY KEY (id); - - --- --- Name: operator_share_changes operator_share_changes_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.operator_share_changes - ADD CONSTRAINT operator_share_changes_pkey PRIMARY KEY (id); - - --- --- Name: operator_shares operator_shares_operator_strategy_block_number_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.operator_shares - ADD CONSTRAINT operator_shares_operator_strategy_block_number_key UNIQUE (operator, strategy, block_number); - - --- --- Name: registered_avs_operators registered_avs_operators_operator_avs_block_number_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.registered_avs_operators - ADD CONSTRAINT registered_avs_operators_operator_avs_block_number_key UNIQUE (operator, avs, block_number); - - --- --- Name: staker_delegation_changes staker_delegation_changes_staker_operator_log_index_block_n_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.staker_delegation_changes - ADD CONSTRAINT staker_delegation_changes_staker_operator_log_index_block_n_key UNIQUE (staker, operator, log_index, block_number); - - --- --- Name: transactions transactions_transaction_hash_block_number_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.transactions - ADD CONSTRAINT transactions_transaction_hash_block_number_key UNIQUE (transaction_hash, block_number); - - --- --- Name: blocks unique_blocks; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.blocks - ADD CONSTRAINT unique_blocks UNIQUE (number); - - --- --- Name: unverified_contracts unverified_contracts_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.unverified_contracts - ADD CONSTRAINT unverified_contracts_pkey PRIMARY KEY (contract_address); - - --- --- Name: idx_avs_operator_changes_avs_operator; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_avs_operator_changes_avs_operator ON public.avs_operator_changes USING btree (avs, operator); - - --- --- Name: idx_avs_operator_changes_block; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_avs_operator_changes_block ON public.avs_operator_changes USING btree (block_number); - - --- --- Name: idx_bytecode_hash; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_bytecode_hash ON public.contracts USING btree (bytecode_hash); - - --- --- Name: idx_delegated_stakers_block; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_delegated_stakers_block ON public.delegated_stakers USING btree (block_number); - - --- --- Name: idx_delegated_stakers_staker_operator; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_delegated_stakers_staker_operator ON public.delegated_stakers USING btree (staker, operator); - - --- --- Name: idx_operator_share_changes_block; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_operator_share_changes_block ON public.operator_share_changes USING btree (block_number); - - --- --- Name: idx_operator_share_changes_operator_strat; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_operator_share_changes_operator_strat ON public.operator_share_changes USING btree (operator, strategy); - - --- --- Name: idx_operator_shares_block; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_operator_shares_block ON public.operator_shares USING btree (block_number); - - --- --- Name: idx_operator_shares_operator_strategy; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_operator_shares_operator_strategy ON public.operator_shares USING btree (operator, strategy); - - --- --- Name: idx_proxy_contract_contract_address; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_proxy_contract_contract_address ON public.proxy_contracts USING btree (contract_address); - - --- --- Name: idx_proxy_contract_proxy_contract_address; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_proxy_contract_proxy_contract_address ON public.proxy_contracts USING btree (proxy_contract_address); - - --- --- Name: idx_registered_avs_operators_avs_operator; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_registered_avs_operators_avs_operator ON public.registered_avs_operators USING btree (avs, operator); - - --- --- Name: idx_registered_avs_operators_block; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_registered_avs_operators_block ON public.registered_avs_operators USING btree (block_number); - - --- --- Name: idx_staker_delegation_changes_block; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_staker_delegation_changes_block ON public.staker_delegation_changes USING btree (block_number); - - --- --- Name: idx_staker_delegation_changes_staker_operator; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_staker_delegation_changes_staker_operator ON public.staker_delegation_changes USING btree (staker, operator); - - --- --- Name: idx_transaciton_logs_block_number; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_transaciton_logs_block_number ON public.transaction_logs USING btree (block_number); - - --- --- Name: idx_transaction_hash; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX idx_transaction_hash ON public.transaction_logs USING btree (transaction_hash, log_index); - - --- --- Name: idx_transaction_logs_address; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_transaction_logs_address ON public.transaction_logs USING btree (address); - - --- --- Name: idx_transaction_logs_transaction_hash; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_transaction_logs_transaction_hash ON public.transaction_logs USING btree (transaction_hash); - - --- --- Name: idx_transactions_block_number; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_transactions_block_number ON public.transactions USING btree (block_number); - - --- --- Name: idx_transactions_bytecode_hash; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_transactions_bytecode_hash ON public.transactions USING btree (bytecode_hash); - - --- --- Name: idx_transactions_from_address; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_transactions_from_address ON public.transactions USING btree (from_address); - - --- --- Name: idx_transactions_to_address; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX idx_transactions_to_address ON public.transactions USING btree (to_address); - - --- --- Name: idx_uniq_proxy_contract; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX idx_uniq_proxy_contract ON public.proxy_contracts USING btree (block_number, contract_address); - - --- --- Name: idx_unique_operator_restaked_strategies; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX idx_unique_operator_restaked_strategies ON public.operator_restaked_strategies USING btree (block_number, operator, avs, strategy); - - --- --- Name: transactions_contract_address; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX transactions_contract_address ON public.transactions USING btree (contract_address); - - --- --- Name: transaction_logs fk_transaction_hash_block_number_key; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.transaction_logs - ADD CONSTRAINT fk_transaction_hash_block_number_key FOREIGN KEY (transaction_hash, block_number) REFERENCES public.transactions(transaction_hash, block_number) ON DELETE CASCADE; - - --- --- Name: transaction_logs transaction_logs_block_number_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.transaction_logs - ADD CONSTRAINT transaction_logs_block_number_fkey FOREIGN KEY (block_number) REFERENCES public.blocks(number) ON DELETE CASCADE; - - --- --- Name: transactions transactions_block_number_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.transactions - ADD CONSTRAINT transactions_block_number_fkey FOREIGN KEY (block_number) REFERENCES public.blocks(number) ON DELETE CASCADE; - - --- --- Name: SCHEMA public; Type: ACL; Schema: -; Owner: - --- - -REVOKE USAGE ON SCHEMA public FROM PUBLIC; -GRANT ALL ON SCHEMA public TO PUBLIC; - - --- --- PostgreSQL database dump complete --- - --- --- PostgreSQL database dump --- - --- Dumped from database version 14.11 (Homebrew) --- Dumped by pg_dump version 15.8 (Homebrew) - -SET statement_timeout = 0; -SET lock_timeout = 0; -SET idle_in_transaction_session_timeout = 0; -SET client_encoding = 'UTF8'; -SET standard_conforming_strings = on; -SELECT pg_catalog.set_config('search_path', '', false); -SET check_function_bodies = false; -SET xmloption = content; -SET client_min_messages = warning; -SET row_security = off; - --- --- Data for Name: migrations; Type: TABLE DATA; Schema: public; Owner: seanmcgary --- - -INSERT INTO public.migrations VALUES ('202405150900_bootstrapDb', '2024-05-17 09:13:15.57261-05', NULL); -INSERT INTO public.migrations VALUES ('202405150917_insertContractAbi', '2024-05-17 09:13:15.575844-05', NULL); -INSERT INTO public.migrations VALUES ('202405151523_addTransactionToFrom', '2024-05-17 09:13:15.578099-05', NULL); -INSERT INTO public.migrations VALUES ('202405170842_addBlockInfoToTransactionLog', '2024-05-17 09:13:15.580354-05', NULL); -INSERT INTO public.migrations VALUES ('202405171056_unverifiedContracts', '2024-05-17 11:00:17.149086-05', NULL); -INSERT INTO public.migrations VALUES ('202405171345_addUpdatedPaymentCoordinatorAbi', '2024-05-17 13:51:24.584807-05', NULL); -INSERT INTO public.migrations VALUES ('202405201503_fixTransactionHashConstraint', '2024-05-20 15:10:45.476856-05', NULL); -INSERT INTO public.migrations VALUES ('202405300925_addUniqueBlockConstraint', '2024-05-30 09:33:13.115195-05', NULL); -INSERT INTO public.migrations VALUES ('202405312008_indexTransactionContractAddress', '2024-05-31 21:13:37.099393-05', NULL); -INSERT INTO public.migrations VALUES ('202405312134_handleProxyContracts', '2024-05-31 22:21:46.84577-05', NULL); -INSERT INTO public.migrations VALUES ('202406030920_addCheckedForProxyFlag', '2024-06-03 10:09:02.176827-05', NULL); -INSERT INTO public.migrations VALUES ('202406031946_addSerialIdToContracts', '2024-06-04 08:54:09.723152-05', NULL); -INSERT INTO public.migrations VALUES ('202406051937_addBytecodeIndex', '2024-06-05 19:54:36.665099-05', NULL); -INSERT INTO public.migrations VALUES ('202406071318_indexTransactionLogBlockNumber', '2024-06-07 13:20:19.291429-05', NULL); -INSERT INTO public.migrations VALUES ('202406110848_transactionLogsContractIndex', '2024-06-11 11:26:43.316616-05', NULL); -INSERT INTO public.migrations VALUES ('202406141007_addCheckedForAbiFlag', '2024-06-14 10:13:35.067238-05', NULL); -INSERT INTO public.migrations VALUES ('202406251424_addTransactionLogsOutputDataColumn', '2024-06-25 14:29:47.494612-05', NULL); -INSERT INTO public.migrations VALUES ('202406251426_addTransactionIndexes', '2024-06-25 14:29:47.500543-05', NULL); -INSERT INTO public.migrations VALUES ('202407101440_addOperatorRestakedStrategiesTable', '2024-07-11 09:48:48.933519-05', NULL); -INSERT INTO public.migrations VALUES ('202407110946_addBlockTimeToRestakedStrategies', '2024-07-11 09:49:17.325774-05', NULL); -INSERT INTO public.migrations VALUES ('202407111116_addAvsDirectoryAddress', '2024-07-24 09:13:18.235218-05', NULL); -INSERT INTO public.migrations VALUES ('202407121407_updateProxyContractIndex', '2024-07-24 09:13:18.240594-05', NULL); -INSERT INTO public.migrations VALUES ('202408200934_eigenlayerStateTables', '2024-09-05 16:16:40.950631-05', NULL); -INSERT INTO public.migrations VALUES ('202409051720_operatorShareChanges', '2024-09-05 19:14:07.595987-05', NULL); -INSERT INTO public.migrations VALUES ('202409052151_stakerDelegations', '2024-09-05 22:24:34.016039-05', NULL); -INSERT INTO public.migrations VALUES ('202409061121_removeSequenceId', '2024-09-06 11:42:21.585605-05', NULL); - - --- --- PostgreSQL database dump complete --- - diff --git a/sqlite-extensions/README.md b/sqlite-extensions/README.md new file mode 100644 index 00000000..4ab1ba2e --- /dev/null +++ b/sqlite-extensions/README.md @@ -0,0 +1,5 @@ +### Running + +```bash +PYTHONPATH="/Users/seanmcgary/Code/sidecar/sqlite-extensions:$PYTHONPATH" lldb -- /opt/homebrew/opt/sqlite/bin/sqlite3 +``` diff --git a/sqlite-extensions/calculations.c b/sqlite-extensions/calculations.c new file mode 100644 index 00000000..0e0cc28b --- /dev/null +++ b/sqlite-extensions/calculations.c @@ -0,0 +1,310 @@ +#define PY_SSIZE_T_CLEAN +#include +#include +#include +#include "calculations.h" +SQLITE_EXTENSION_INIT1 + +static PyObject *pModule = NULL; +static PyGILState_STATE gilstate; + +int init_python() { + Py_Initialize(); + PyRun_SimpleString("import sys; sys.path.append('.')"); + pModule = PyImport_ImportModule("calculations"); + if (pModule == NULL) { + PyErr_Print(); + return 0; + } + return 1; +} + +void finalize_python() { + Py_XDECREF(pModule); + Py_Finalize(); +} + +char* call_python_func(const char* func_name, const char* arg1, const char* arg2) { + char *result = NULL; + gilstate = PyGILState_Ensure(); + + PyObject *pFunc = PyObject_GetAttrString(pModule, func_name); + if (pFunc && PyCallable_Check(pFunc)) { + PyObject *pArgs; + if (arg2 == NULL) { + pArgs = Py_BuildValue("(s)", arg1); + } else { + pArgs = Py_BuildValue("(ss)", arg1, arg2); + } + PyObject *pValue = PyObject_CallObject(pFunc, pArgs); + Py_DECREF(pArgs); + if (pValue != NULL) { + PyObject *str_obj = PyObject_Str(pValue); + const char *str_result = PyUnicode_AsUTF8(str_obj); + result = strdup(str_result); + Py_DECREF(str_obj); + Py_DECREF(pValue); + } else { + PyErr_Print(); + } + } else { + PyErr_Print(); + } + Py_XDECREF(pFunc); + + PyGILState_Release(gilstate); + return result; +} + +// _pre_nile_tokens_per_day is a non-sqlite3 function that is called by pre_nile_tokens_per_day +char* _pre_nile_tokens_per_day(const char* tokens) { + return call_python_func("preNileTokensPerDay", tokens, NULL); +} + +// Your custom function +void pre_nile_tokens_per_day(sqlite3_context *context, int argc, sqlite3_value **argv) { + if (argc != 1) { + sqlite3_result_error(context, "pre_nile_tokens_per_day() requires exactly one argument", -1); + return; + } + const char* input = (const char*)sqlite3_value_text(argv[0]); + if (!input) { + sqlite3_result_null(context); + return; + } + + int ready = init_python(); + if (ready == 0) { + sqlite3_result_error(context, "Failed to initialize Python", -1); + return; + } + + char* tokens = _pre_nile_tokens_per_day(input); + if (!tokens) { + sqlite3_result_null(context); + return; + } + + sqlite3_result_text(context, tokens, -1, SQLITE_STATIC); + // TODO(seanmcgary): figure out why this breaks things... + //finalize_python(); +} + +char* _amazon_staker_token_rewards(const char* sp, const char* tpd) { + return call_python_func("amazonStakerTokenRewards", sp, tpd); +} + +void amazon_staker_token_rewards(sqlite3_context *context, int argc, sqlite3_value **argv) { + if (argc != 2) { + sqlite3_result_error(context, "amazon_staker_token_rewards() requires two arguments", -1); + return; + } + const char* sp = (const char*)sqlite3_value_text(argv[0]); + if (!sp) { + sqlite3_result_null(context); + return; + } + + const char* tpd = (const char*)sqlite3_value_text(argv[1]); + if (!tpd) { + sqlite3_result_null(context); + return; + } + + int ready = init_python(); + if (ready == 0) { + sqlite3_result_error(context, "Failed to initialize Python", -1); + return; + } + + char* tokens = _amazon_staker_token_rewards(sp, tpd); + if (!tokens) { + sqlite3_result_null(context); + return; + } + + sqlite3_result_text(context, tokens, -1, SQLITE_STATIC); + // TODO(seanmcgary): figure out why this breaks things... + //finalize_python(); +} + +char* _nile_staker_token_rewards(const char* sp, const char* tpd) { + return call_python_func("nileStakerTokenRewards", sp, tpd); +} + +void nile_staker_token_rewards(sqlite3_context *context, int argc, sqlite3_value **argv) { + if (argc != 2) { + sqlite3_result_error(context, "nile_staker_token_rewards() requires two arguments", -1); + return; + } + const char* sp = (const char*)sqlite3_value_text(argv[0]); + if (!sp) { + sqlite3_result_null(context); + return; + } + + const char* tpd = (const char*)sqlite3_value_text(argv[1]); + if (!tpd) { + sqlite3_result_null(context); + return; + } + + int ready = init_python(); + if (ready == 0) { + sqlite3_result_error(context, "Failed to initialize Python", -1); + return; + } + + char* tokens = _nile_staker_token_rewards(sp, tpd); + if (!tokens) { + sqlite3_result_null(context); + return; + } + + sqlite3_result_text(context, tokens, -1, SQLITE_STATIC); + // TODO(seanmcgary): figure out why this breaks things... + //finalize_python(); +} + +char* _staker_token_rewards(const char* sp, const char* tpd) { + return call_python_func("stakerTokenRewards", sp, tpd); +} + +void staker_token_rewards(sqlite3_context *context, int argc, sqlite3_value **argv) { + if (argc != 2) { + sqlite3_result_error(context, "staker_token_rewards() requires two arguments", -1); + return; + } + const char* sp = (const char*)sqlite3_value_text(argv[0]); + if (!sp) { + sqlite3_result_null(context); + return; + } + + const char* tpd = (const char*)sqlite3_value_text(argv[1]); + if (!tpd) { + sqlite3_result_null(context); + return; + } + + int ready = init_python(); + if (ready == 0) { + sqlite3_result_error(context, "Failed to initialize Python", -1); + return; + } + + char* tokens = _staker_token_rewards(sp, tpd); + if (!tokens) { + sqlite3_result_null(context); + return; + } + + sqlite3_result_text(context, tokens, -1, SQLITE_STATIC); + // TODO(seanmcgary): figure out why this breaks things... + //finalize_python(); +} + +char* _amazon_operator_token_rewards(const char* totalStakerOperatorTokens) { + return call_python_func("amazonOperatorTokenRewards", totalStakerOperatorTokens, NULL); +} + +void amazon_operator_token_rewards(sqlite3_context *context, int argc, sqlite3_value **argv) { + if (argc != 1) { + sqlite3_result_error(context, "amazon_operator_token_rewards() requires exactly one argument", -1); + return; + } + const char* input = (const char*)sqlite3_value_text(argv[0]); + if (!input) { + sqlite3_result_null(context); + return; + } + + int ready = init_python(); + if (ready == 0) { + sqlite3_result_error(context, "Failed to initialize Python", -1); + return; + } + + char* tokens = _amazon_operator_token_rewards(input); + if (!tokens) { + sqlite3_result_null(context); + return; + } + + sqlite3_result_text(context, tokens, -1, SQLITE_STATIC); + // TODO(seanmcgary): figure out why this breaks things... + //finalize_python(); +} + + +char* _nile_operator_token_rewards(const char* totalStakerOperatorTokens) { + return call_python_func("nileOperatorTokenRewards", totalStakerOperatorTokens, NULL); +} +void nile_operator_token_rewards(sqlite3_context *context, int argc, sqlite3_value **argv) { + if (argc != 1) { + sqlite3_result_error(context, "nile_operator_token_rewards() requires exactly one argument", -1); + return; + } + const char* input = (const char*)sqlite3_value_text(argv[0]); + if (!input) { + sqlite3_result_null(context); + return; + } + + int ready = init_python(); + if (ready == 0) { + sqlite3_result_error(context, "Failed to initialize Python", -1); + return; + } + + char* tokens = _nile_operator_token_rewards(input); + if (!tokens) { + sqlite3_result_null(context); + return; + } + + sqlite3_result_text(context, tokens, -1, SQLITE_STATIC); + // TODO(seanmcgary): figure out why this breaks things... + //finalize_python(); +} + +int sqlite3_calculations_init(sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi) { + SQLITE_EXTENSION_INIT2(pApi); + int rc; + rc = sqlite3_create_function(db, "pre_nile_tokens_per_day", 1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, 0, pre_nile_tokens_per_day, 0, 0); + if (rc != SQLITE_OK) { + fprintf(stderr, "Failed to create function: %s\n", sqlite3_errmsg(db)); + return rc; + } + + rc = sqlite3_create_function(db, "amazon_staker_token_rewards", 2, SQLITE_UTF8 | SQLITE_DETERMINISTIC, 0, amazon_staker_token_rewards, 0, 0); + if (rc != SQLITE_OK) { + fprintf(stderr, "Failed to create function: %s\n", sqlite3_errmsg(db)); + return rc; + } + + rc = sqlite3_create_function(db, "nile_staker_token_rewards", 2, SQLITE_UTF8 | SQLITE_DETERMINISTIC, 0, nile_staker_token_rewards, 0, 0); + if (rc != SQLITE_OK) { + fprintf(stderr, "Failed to create function: %s\n", sqlite3_errmsg(db)); + return rc; + } + + rc = sqlite3_create_function(db, "staker_token_rewards", 2, SQLITE_UTF8 | SQLITE_DETERMINISTIC, 0, staker_token_rewards, 0, 0); + if (rc != SQLITE_OK) { + fprintf(stderr, "Failed to create function: %s\n", sqlite3_errmsg(db)); + return rc; + } + + rc = sqlite3_create_function(db, "amazon_operator_token_rewards", 1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, 0, amazon_operator_token_rewards, 0, 0); + if (rc != SQLITE_OK) { + fprintf(stderr, "Failed to create function: %s\n", sqlite3_errmsg(db)); + return rc; + } + + rc = sqlite3_create_function(db, "nile_operator_token_rewards", 1, SQLITE_UTF8 | SQLITE_DETERMINISTIC, 0, nile_operator_token_rewards, 0, 0); + if (rc != SQLITE_OK) { + fprintf(stderr, "Failed to create function: %s\n", sqlite3_errmsg(db)); + return rc; + } + return SQLITE_OK; +} diff --git a/sqlite-extensions/calculations.h b/sqlite-extensions/calculations.h new file mode 100644 index 00000000..5034fc47 --- /dev/null +++ b/sqlite-extensions/calculations.h @@ -0,0 +1,28 @@ +#ifndef CALCULATIONS_H +#define CALCULATIONS_H + +#include + +int init_python(); +void finalize_python(); +char* call_python_func(const char* func_name, const char* arg1, const char* arg2); + +char* _pre_nile_tokens_per_day(const char* tokens); +void pre_nile_tokens_per_day(sqlite3_context *context, int argc, sqlite3_value **argv); + +char* _amazon_staker_token_rewards(const char* sp, const char* tpd); +void amazon_staker_token_rewards(sqlite3_context *context, int argc, sqlite3_value **argv); + +char* _nile_staker_token_rewards(const char* sp, const char* tpd); +void nile_staker_token_rewards(sqlite3_context *context, int argc, sqlite3_value **argv); + +char* _staker_token_rewards(const char* sp, const char* tpd); +void staker_token_rewards(sqlite3_context *context, int argc, sqlite3_value **argv); + +char* _amazon_operator_token_rewards(const char* totalStakerOperatorTokens); +void amazon_operator_token_rewards(sqlite3_context *context, int argc, sqlite3_value **argv); + +char* _nile_operator_token_rewards(const char* totalStakerOperatorTokens); +void nile_operator_token_rewards(sqlite3_context *context, int argc, sqlite3_value **argv); + +#endif // CALCULATIONS_H diff --git a/sqlite-extensions/calculations.py b/sqlite-extensions/calculations.py new file mode 100644 index 00000000..674e3282 --- /dev/null +++ b/sqlite-extensions/calculations.py @@ -0,0 +1,74 @@ +from decimal import Decimal, getcontext, ROUND_HALF_UP, ROUND_UP, ROUND_DOWN + +def preNileTokensPerDay(tokens: str) -> str: + big_amount = float(tokens) + div = 0.999999999999999 + res = big_amount * div + + res_str = "{}".format(res) + return "{}".format(int(Decimal(res_str))) + +def amazonStakerTokenRewards(sp:str, tpd:str) -> str: + getcontext().prec = 15 + stakerProportion = Decimal(sp) + tokensPerDay = Decimal(tpd) + + decimal_res = Decimal(stakerProportion * tokensPerDay) + + getcontext().prec = 20 + rounded = decimal_res.quantize(Decimal('1'), rounding=ROUND_HALF_UP) + + return "{}".format(rounded) + + +def nileStakerTokenRewards(sp:str, tpd:str) -> str: + stakerProportion = Decimal(sp) + tokensPerDay = Decimal(tpd) + + decimal_res = Decimal(stakerProportion * tokensPerDay) + # Truncate to 0.x decimals + truncated = decimal_res.quantize(Decimal('0.1'), rounding=ROUND_UP) + + # Bankers rounding to match postgres + rounded = truncated.quantize(Decimal('1'), rounding=ROUND_HALF_UP) + + return "{}".format(rounded) + +def stakerTokenRewards(sp:str, tpd:str) -> str: + stakerProportion = float(sp) + tokensPerDay = int(tpd) + + decimal_res = stakerProportion * tokensPerDay + + parts = str(decimal_res).split("+") + # Need more precision + if len(parts) == 2 and int(parts[1]) > 16: + stakerProportion = Decimal(sp) + tokensPerDay = Decimal(tpd) + + getcontext().prec = 17 + getcontext().rounding = ROUND_DOWN + decimal_res = stakerProportion * tokensPerDay + + return "{}".format(int(decimal_res)) + + return "{}".format(int(decimal_res)) + + +def amazonOperatorTokenRewards(totalStakerOperatorTokens:str) -> str: + totalStakerOperatorTokens = Decimal(totalStakerOperatorTokens) + + operatorTokens = totalStakerOperatorTokens * Decimal(0.1) + + rounded = operatorTokens.quantize(Decimal('1'), rounding=ROUND_HALF_UP) + + return "{}".format(rounded) + +def nileOperatorTokenRewards(totalStakerOperatorTokens:str) -> str: + if totalStakerOperatorTokens[-1] == "0": + return "{}".format(int(totalStakerOperatorTokens) // 10) + totalStakerOperatorTokens = Decimal(totalStakerOperatorTokens) + operatorTokens = Decimal(str(totalStakerOperatorTokens)) * Decimal(0.1) + rounded = operatorTokens.quantize(Decimal('1'), rounding=ROUND_HALF_UP) + return "{}".format(rounded) +