diff --git a/Dockerfile.indexerd b/Dockerfile similarity index 100% rename from Dockerfile.indexerd rename to Dockerfile diff --git a/Dockerfile.addressValidator b/Dockerfile.addressValidator deleted file mode 100644 index 030ad32..0000000 --- a/Dockerfile.addressValidator +++ /dev/null @@ -1,28 +0,0 @@ -FROM golang:alpine AS builder - -RUN apk update && apk add --no-cache git gcc g++ make libc-dev pkgconfig zeromq-dev curl libunwind-dev - -RUN adduser -D -u 1001 -g '' appuser -WORKDIR $GOPATH/src/mypackage/myapp/ -COPY . . - -RUN go env CGO_ENABLED -RUN go mod download -x -RUN go mod verify - -RUN GOOS=linux GOARCH=amd64 go build -ldflags="-w -s" -o /go/bin/addressValidator ./cmd/address-validator - -RUN chmod u+x /go/bin/* - -FROM alpine:latest - -RUN apk update && apk add --no-cache zeromq - -WORKDIR /app - -COPY --from=builder /etc/passwd /etc/passwd -COPY --from=builder /go/bin/addressValidator /app/validator - -COPY .env.dist /app/.env - -ENTRYPOINT ["/app/validator"] \ No newline at end of file diff --git a/Dockerfile.cfundValidator b/Dockerfile.cfundValidator deleted file mode 100644 index 643b3f4..0000000 --- a/Dockerfile.cfundValidator +++ /dev/null @@ -1,28 +0,0 @@ -FROM golang:alpine AS builder - -RUN apk update && apk add --no-cache git gcc g++ make libc-dev pkgconfig zeromq-dev curl libunwind-dev - -RUN adduser -D -u 1001 -g '' appuser -WORKDIR $GOPATH/src/mypackage/myapp/ -COPY . . - -RUN go env CGO_ENABLED -RUN go mod download -x -RUN go mod verify - -RUN GOOS=linux GOARCH=amd64 go build -ldflags="-w -s" -o /go/bin/cfundValidator ./cmd/cfund-validator - -RUN chmod u+x /go/bin/* - -FROM alpine:latest - -RUN apk update && apk add --no-cache zeromq - -WORKDIR /app - -COPY --from=builder /etc/passwd /etc/passwd -COPY --from=builder /go/bin/cfundValidator /app/validator - -COPY .env.dist /app/.env - -ENTRYPOINT ["/app/validator"] \ No newline at end of file diff --git a/Dockerfile.indexerCli b/Dockerfile.indexerCli deleted file mode 100644 index 8fe4f0b..0000000 --- a/Dockerfile.indexerCli +++ /dev/null @@ -1,30 +0,0 @@ -FROM golang:alpine AS builder - -RUN apk update && apk add --no-cache git gcc g++ make libc-dev pkgconfig zeromq-dev curl libunwind-dev - -RUN adduser -D -u 1001 -g '' appuser -WORKDIR $GOPATH/src/mypackage/myapp/ -COPY . . - -RUN go env CGO_ENABLED -RUN go mod download -x -RUN go mod verify - -RUN GOOS=linux GOARCH=amd64 go build -ldflags="-w -s" -o /go/bin/indexer-cli ./cmd/indexer-cli - -RUN chmod u+x /go/bin/* - -FROM alpine:latest - -RUN apk update && apk add --no-cache zeromq - -WORKDIR /app - -COPY --from=builder /etc/passwd /etc/passwd -COPY --from=builder /go/bin/indexer-cli /app/indexer-cli - -COPY .env.dist /app/.env - -COPY ./config/mappings /app/mappings - -ENTRYPOINT ["/app/indexer-cli"] \ No newline at end of file diff --git a/bin/build b/bin/build index 93901ff..39ee61b 100755 --- a/bin/build +++ b/bin/build @@ -14,14 +14,5 @@ dingo -src=./internal/config/di -dest=./generated export DOCKER_BUILDKIT=1 -docker build -f ./Dockerfile.indexerd . -t navexplorer/indexer:$tag +docker build -f ./Dockerfile . -t navexplorer/indexer:$tag docker push navexplorer/indexer:$tag - -#docker build -f ./Dockerfile.indexerCli . -t navexplorer/indexer-cli:$tag -#docker push navexplorer/indexer-cli:$tag - -docker build -f ./Dockerfile.addressValidator . -t navexplorer/address-validator:$tag -docker push navexplorer/address-validator:$tag - -#docker build -f ./Dockerfile.cfundValidator . -t navexplorer/cfund-validator:$tag -#docker push navexplorer/cfund-validator:$tag \ No newline at end of file diff --git a/cmd/address-validator/main.go b/cmd/address-validator/main.go index e653913..b346068 100644 --- a/cmd/address-validator/main.go +++ b/cmd/address-validator/main.go @@ -1,9 +1,23 @@ package main import ( - "github.com/NavExplorer/navexplorer-indexer-go/internal/validator" + "github.com/NavExplorer/navexplorer-indexer-go/generated/dic" + "github.com/NavExplorer/navexplorer-indexer-go/internal/config" + "github.com/sarulabs/dingo/v3" + "os" ) func main() { - new(validator.AddressValidator).Execute() + config.Init() + + container, _ := dic.NewContainer(dingo.App) + container.GetElastic().InstallMappings() + + addressRepo := container.GetAddressRepo() + rewinder := container.GetAddressRewinder() + + address, err := addressRepo.GetAddress(os.Args[1]) + if err == nil { + rewinder.ResetAddress(address) + } } diff --git a/cmd/cfund-validator/main.go b/cmd/cfund-validator/main.go deleted file mode 100644 index 51e2078..0000000 --- a/cmd/cfund-validator/main.go +++ /dev/null @@ -1,9 +0,0 @@ -package main - -import ( - "github.com/NavExplorer/navexplorer-indexer-go/internal/validator" -) - -func main() { - new(validator.CfundValidator).Execute() -} diff --git a/cmd/soft-fork/main.go b/cmd/soft-fork/main.go new file mode 100644 index 0000000..5e739f6 --- /dev/null +++ b/cmd/soft-fork/main.go @@ -0,0 +1,39 @@ +package main + +import ( + "github.com/NavExplorer/navexplorer-indexer-go/generated/dic" + "github.com/NavExplorer/navexplorer-indexer-go/internal/config" + "github.com/sarulabs/dingo/v3" + log "github.com/sirupsen/logrus" +) + +var container *dic.Container + +func main() { + config.Init() + + container, _ = dic.NewContainer(dingo.App) + container.GetElastic().InstallMappings() + container.GetSoftforkService().InitSoftForks() + bestBlock, err := container.GetBlockRepo().GetBestBlock() + if err != nil { + log.Fatal(err) + } + + for i := 1; i <= int(bestBlock.Height); i += 2016 { + blocks, err := container.GetBlockRepo().GetBlocksBetweenHeight(uint64(i), uint64(i+2016)) + if err != nil { + log.Fatal(err) + } + for _, block := range blocks { + container.GetSoftforkIndexer().Index(block) + } + container.GetElastic().Persist() + } + container.GetElastic().Persist() + //block, err := container.GetBlockRepo().GetBlockByHeight() + //if err == nil { + // log.Info("Rewind SoftForks to height: ", block.Height) + //// container.GetSoftforkRewinder().Rewind(block.Height - 10) + //} +} diff --git a/config/mappings/address.json b/config/mappings/address.json index 7713375..9af7fa5 100644 --- a/config/mappings/address.json +++ b/config/mappings/address.json @@ -1,5 +1,8 @@ { "mappings": { + "_source": { + "excludes": ["rich_list"] + }, "properties": { "hash": { "type": "text", @@ -11,47 +14,15 @@ } }, "height": {"type": "long"}, - "balance": {"type": "long"}, - "received": { - "type": "long" - }, - "receivedCount": { - "type": "long" - }, - "sent": { - "type": "long" - }, - "sentCount": { - "type": "long" - }, - "staked": { - "type": "long" - }, - "stakedCount": { - "type": "long" - }, - "coldReceived": { - "type": "long" - }, - "coldReceivedCount": { - "type": "long" - }, - "coldSent": { - "type": "long" - }, - "coldSentCount": { - "type": "long" - }, - "coldStaked": { - "type": "long" + "spendable": {"type": "long"}, + "stakabout": {"type": "long"}, + "voting_weight": {"type": "long"}, + "created_time": { + "type": "date" }, - "coldStakedCount": { + "created_block": { "type": "long" - }, - "coldBalance": { - "type": "long" - }, - "position": {"type": "long"} + } } }, "settings": { diff --git a/config/mappings/addresstransaction.json b/config/mappings/addresshistory.json similarity index 53% rename from config/mappings/addresstransaction.json rename to config/mappings/addresshistory.json index 8a7d195..4818e2b 100644 --- a/config/mappings/addresstransaction.json +++ b/config/mappings/addresshistory.json @@ -1,14 +1,14 @@ { "mappings": { "properties": { - "hash": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 - } - } + "height": { + "type": "long" + }, + "txindex": { + "type": "integer" + }, + "time": { + "type": "date" }, "txid": { "type": "text", @@ -19,16 +19,7 @@ } } }, - "height": { - "type": "long" - }, - "index": { - "type": "integer" - }, - "time": { - "type": "date" - }, - "type": { + "address": { "type": "text", "fields": { "keyword": { @@ -37,13 +28,31 @@ } } }, - "input": {"type": "long"}, - "output": {"type": "long"}, - "total": {"type": "long"}, + "changes": { + "type": "nested", + "properties": { + "spendable": {"type": "long"}, + "stakable": {"type": "long"}, + "voting_weight": {"type": "long"} + } + }, "balance": { - "type": "long" + "type": "nested", + "properties": { + "spendable": {"type": "long"}, + "stakable": {"type": "long"}, + "voting_weight": {"type": "long"} + } + }, + "is_stake": { + "type": "boolean" + }, + "is_cfund_payout": { + "type": "boolean" }, - "cold": {"type": "boolean"} + "is_stake_payout": { + "type": "boolean" + } } }, "settings":{ diff --git a/go.mod b/go.mod index 8ace47e..12599df 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,14 @@ module github.com/NavExplorer/navexplorer-indexer-go go 1.13 require ( - github.com/NavExplorer/navcoind-go v0.1.8-0.20200527214456-804845973a06 - github.com/NavExplorer/subscriber v0.0.0-20200601212953-3a6c48dd3d5d + github.com/NavExplorer/navcoind-go v0.1.8-0.20200709223613-3322962ba291 + github.com/NavExplorer/subscriber v0.0.0-20200923000930-1321b56838fc github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 // indirect github.com/dgraph-io/badger/v2 v2.0.3 // indirect github.com/getsentry/raven-go v0.2.0 github.com/gosimple/slug v1.9.0 github.com/joho/godotenv v1.3.0 + github.com/mr-tron/base58 v1.2.0 github.com/olivere/elastic/v7 v7.0.9 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/sarulabs/dingo/v3 v3.1.0 @@ -17,5 +18,6 @@ require ( github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 // indirect github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71 + golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4 gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) diff --git a/go.sum b/go.sum index 30094a9..b133d55 100644 --- a/go.sum +++ b/go.sum @@ -1,42 +1,13 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/NavExplorer/navcoind-go v0.1.1 h1:Qwc3O6aIl7Dg5CVb59QLSGRNt0PocJk4SB92pRy/N4g= -github.com/NavExplorer/navcoind-go v0.1.1/go.mod h1:SkiAvsIfYh0pWBox2yOOiDaPOucS/bayAVkMUNR0Sv0= -github.com/NavExplorer/navcoind-go v0.1.3 h1:gcxk46RYPw3VtT+uKdBnm7H3d9OtOVamqtc2mvVM9cs= -github.com/NavExplorer/navcoind-go v0.1.3/go.mod h1:SkiAvsIfYh0pWBox2yOOiDaPOucS/bayAVkMUNR0Sv0= -github.com/NavExplorer/navcoind-go v0.1.4 h1:DRbDt8nzPZ/sOnQ7TaXpO0A8n8qz04/CTCqFUd1MYrw= -github.com/NavExplorer/navcoind-go v0.1.4/go.mod h1:SkiAvsIfYh0pWBox2yOOiDaPOucS/bayAVkMUNR0Sv0= -github.com/NavExplorer/navcoind-go v0.1.5 h1:AFKRBOJjhnNB8+y28xUXzzdQqUKHIRRSZJtxw9t2x4o= -github.com/NavExplorer/navcoind-go v0.1.5/go.mod h1:SkiAvsIfYh0pWBox2yOOiDaPOucS/bayAVkMUNR0Sv0= -github.com/NavExplorer/navcoind-go v0.1.6 h1:2zLrs95lonSS/Z02ITVHdYi17ZoLv3GnZc86jxVu/dE= -github.com/NavExplorer/navcoind-go v0.1.6/go.mod h1:SkiAvsIfYh0pWBox2yOOiDaPOucS/bayAVkMUNR0Sv0= -github.com/NavExplorer/navcoind-go v0.1.7 h1:ws3DaDUK45SbcHOKkT6o6HOlRmZe226TuK+lt4ojm2E= -github.com/NavExplorer/navcoind-go v0.1.7/go.mod h1:SkiAvsIfYh0pWBox2yOOiDaPOucS/bayAVkMUNR0Sv0= -github.com/NavExplorer/navcoind-go v0.1.8-0.20200301231942-61d700d501b5 h1:IYoCcBDRETcR387/9hCia4iLZolbXb7p9qEY+N4C3d0= -github.com/NavExplorer/navcoind-go v0.1.8-0.20200301231942-61d700d501b5/go.mod h1:SkiAvsIfYh0pWBox2yOOiDaPOucS/bayAVkMUNR0Sv0= -github.com/NavExplorer/navcoind-go v0.1.8-0.20200302222257-8d68a4c3d090 h1:ErWZLa/xY1+PyHUAXjC5lPKvGYzv6OtaW1qe4qVJDlI= -github.com/NavExplorer/navcoind-go v0.1.8-0.20200302222257-8d68a4c3d090/go.mod h1:SkiAvsIfYh0pWBox2yOOiDaPOucS/bayAVkMUNR0Sv0= -github.com/NavExplorer/navcoind-go v0.1.8-0.20200312103358-e2b6fc88b794 h1:SESVNWOQHIT6BeMLqWNYxsR7xqnsAleD8Fvhurn0DC0= -github.com/NavExplorer/navcoind-go v0.1.8-0.20200312103358-e2b6fc88b794/go.mod h1:SkiAvsIfYh0pWBox2yOOiDaPOucS/bayAVkMUNR0Sv0= -github.com/NavExplorer/navcoind-go v0.1.8-0.20200312104854-27b68fb61fe2 h1:cNTUHRWsDwqyqw0c74/+f5X7MS+Z/wzvXPWkY2XXOv4= -github.com/NavExplorer/navcoind-go v0.1.8-0.20200312104854-27b68fb61fe2/go.mod h1:SkiAvsIfYh0pWBox2yOOiDaPOucS/bayAVkMUNR0Sv0= -github.com/NavExplorer/navcoind-go v0.1.8-0.20200513131954-44d6cb62cf16 h1:pcMwYic6OtRAcwaNJGaE7Avv5e7vj3o9gcFh/ah2xVw= -github.com/NavExplorer/navcoind-go v0.1.8-0.20200513131954-44d6cb62cf16/go.mod h1:SkiAvsIfYh0pWBox2yOOiDaPOucS/bayAVkMUNR0Sv0= -github.com/NavExplorer/navcoind-go v0.1.8-0.20200513221350-a4bbac36931f h1:gD0oZDGSfM07JAP29Uz7sroIhayRTKSBAD9RWDBImEQ= -github.com/NavExplorer/navcoind-go v0.1.8-0.20200513221350-a4bbac36931f/go.mod h1:SkiAvsIfYh0pWBox2yOOiDaPOucS/bayAVkMUNR0Sv0= -github.com/NavExplorer/navcoind-go v0.1.8-0.20200516202154-f528c63e3a1d h1:Pi/mpQyKOe1n/52kvkXfAzzVUf9Dfaz3t9S3ZaHRd0g= -github.com/NavExplorer/navcoind-go v0.1.8-0.20200516202154-f528c63e3a1d/go.mod h1:SkiAvsIfYh0pWBox2yOOiDaPOucS/bayAVkMUNR0Sv0= -github.com/NavExplorer/navcoind-go v0.1.8-0.20200518235421-aef73039a473 h1:ajTMeFd0lPuY/x35BvZBq2SJX1/L5agsiboNlJos8e8= -github.com/NavExplorer/navcoind-go v0.1.8-0.20200518235421-aef73039a473/go.mod h1:SkiAvsIfYh0pWBox2yOOiDaPOucS/bayAVkMUNR0Sv0= -github.com/NavExplorer/navcoind-go v0.1.8-0.20200527214456-804845973a06 h1:ezTriPNd8773iTdIchMNieriXMqa/5d3fmOlJnZscIc= -github.com/NavExplorer/navcoind-go v0.1.8-0.20200527214456-804845973a06/go.mod h1:SkiAvsIfYh0pWBox2yOOiDaPOucS/bayAVkMUNR0Sv0= -github.com/NavExplorer/subscriber v0.0.0-20200601205031-4135a91491af h1:KWrfS1eDXRPSpY6pyBMGYiqLSIJ08TargSgbFeUTaMA= -github.com/NavExplorer/subscriber v0.0.0-20200601205031-4135a91491af/go.mod h1:b9UVwdUV1OVGDsOMqOJeu5Rk5SPyOgfHfvM0HH4fvdU= +github.com/NavExplorer/navcoind-go v0.1.8-0.20200709223613-3322962ba291 h1:kHIh0LP5ynNDNb0rzMHptcQwO5ZH/rw+vWp3S10uACY= +github.com/NavExplorer/navcoind-go v0.1.8-0.20200709223613-3322962ba291/go.mod h1:CEjLeqFkhdtqOUbpd5Aoo288nX1X1M/TXUpGXCkm1tk= github.com/NavExplorer/subscriber v0.0.0-20200601212953-3a6c48dd3d5d h1:4tlsQCkBbil7rexH6A0CGyRh5I7z1QP/ftiFLWP7ADo= github.com/NavExplorer/subscriber v0.0.0-20200601212953-3a6c48dd3d5d/go.mod h1:b9UVwdUV1OVGDsOMqOJeu5Rk5SPyOgfHfvM0HH4fvdU= +github.com/NavExplorer/subscriber v0.0.0-20200923000930-1321b56838fc h1:Qh/wA4qAubfyl0s4LsZER26iGqnajRBT3tXi4/Tmbpo= +github.com/NavExplorer/subscriber v0.0.0-20200923000930-1321b56838fc/go.mod h1:2j06trBo93OSjGJk9CrU1v+k9DSNebc/AA3tFQ1/BqE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= @@ -49,7 +20,6 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 h1:JLaf/iINcLyjwbtTsCJjc6rtlASgHeIJPrB6QmwURnA= github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -62,16 +32,11 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/badger v1.6.1 h1:w9pSFNSdq/JPM1N12Fz/F/bzo993Is1W+Q7HjPzi7yg= -github.com/dgraph-io/badger/v2 v2.0.3 h1:inzdf6VF/NZ+tJ8RwwYMjJMvsOALTHYdozn0qSl6XJI= github.com/dgraph-io/badger/v2 v2.0.3/go.mod h1:3KY8+bsP8wI0OEnQJAKpd4wIJW/Mm32yw2j/9FUVnIM= -github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3 h1:MQLRM35Pp0yAyBYksjbj1nZI/w6eyRY/mWoM1sFf4kU= github.com/dgraph-io/ristretto v0.0.2-0.20200115201040-8f368f2f2ab3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= @@ -79,6 +44,7 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -100,21 +66,25 @@ github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4er github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gosimple/slug v1.9.0 h1:r5vDcYrFz9BmfIAMC829un9hq7hKM4cHUrsv36LbEqs= github.com/gosimple/slug v1.9.0/go.mod h1:AMZ+sOVe65uByN3kgEyf9WEBKBCSS+dJjMX9x4vDJbg= -github.com/gravitational/trace v1.1.9/go.mod h1:RvdOUHE4SHqR3oXlFFKnGzms8a5dugHygGw1bqDstYI= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= @@ -123,7 +93,6 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/interactive-solutions/go-logrus-elasticsearch v0.0.0-20190729081800-720ab42dc5d5/go.mod h1:8mDGCI5lL2wupP5T+FhbfrAdtgNrf8JhKa0GhhCqDgE= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= @@ -133,7 +102,6 @@ github.com/karrick/godirwalk v1.10.12/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0L github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -149,25 +117,28 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= +github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olivere/elastic v6.2.26+incompatible h1:3PjUHKyt8xKwbFQpRC5cgtEY7Qz6ejopBkukhI7UWvE= github.com/olivere/elastic/v7 v7.0.9 h1:+bTR1xJbfLYD8WnTBt9672mFlKxjfWRJpEQ1y8BMS3g= github.com/olivere/elastic/v7 v7.0.9/go.mod h1:2TeRd0vhLRTK9zqm5xP0uLiVeZ5yUoL7kZ+8SZA9r9Y= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.8.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= -github.com/patrickmn/go-cache v1.0.0 h1:3gD5McaYs9CxjyK5AXGcq8gdeCARtd/9gJDUvVeaZ0Y= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/pebbe/zmq4 v1.0.0 h1:D+MSmPpqkL5PSSmnh8g51ogirUCyemThuZzLW7Nrt78= -github.com/pebbe/zmq4 v1.0.0/go.mod h1:7N4y5R18zBiu3l0vajMUWQgZyjv464prE8RCyBcmnZM= github.com/pebbe/zmq4 v1.2.1 h1:jrXQW3mD8Si2mcSY/8VBs2nNkK/sKCOEM0rHAfxyc8c= github.com/pebbe/zmq4 v1.2.1/go.mod h1:7N4y5R18zBiu3l0vajMUWQgZyjv464prE8RCyBcmnZM= +github.com/pebbe/zmq4 v1.2.2-0.20200907174415-614833b9a108 h1:88kYFBacrhRXiglfEOZWylTHfzeH0ntoZbC9C1i7qk0= +github.com/pebbe/zmq4 v1.2.2-0.20200907174415-614833b9a108/go.mod h1:7N4y5R18zBiu3l0vajMUWQgZyjv464prE8RCyBcmnZM= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -195,14 +166,10 @@ github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sarulabs/dingo v2.0.0+incompatible h1:f9pmmm5cHdSOMQntbLqWh7rjmTXFR5oDOHMxbEurh1A= -github.com/sarulabs/dingo/v3 v3.0.0 h1:KVcsT4Jy/K9m9d42KlPu8ur/jZyrgBzLeJR1FFeXmJ8= -github.com/sarulabs/dingo/v3 v3.0.0/go.mod h1:vr6wVyi2X3o8mVffmajfLvY0xKLB9kgGwmGkRCCsrHE= github.com/sarulabs/dingo/v3 v3.1.0 h1:ElWFaP/wvI1NYl5THVYZl/fKxxikaAAmdQdyNBmHsh4= github.com/sarulabs/dingo/v3 v3.1.0/go.mod h1:vr6wVyi2X3o8mVffmajfLvY0xKLB9kgGwmGkRCCsrHE= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -212,12 +179,10 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -244,6 +209,7 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4 h1:ydJNl0ENAG67pFbB+9tfhiL2pYqLhfoaZFw/cjLhY4A= golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -261,8 +227,8 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -282,10 +248,14 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190515120540-06a5c4944438 h1:khxRGsvPk4n2y8I/mLLjp7e5dMTJmH75wvqS6nMwUtY= golang.org/x/sys v0.0.0-20190515120540-06a5c4944438/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -297,7 +267,7 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190624180213-70d37148ca0c/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -308,6 +278,12 @@ google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -322,5 +298,6 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/config/di/definitions.go b/internal/config/di/definitions.go index ea97cac..70204ef 100644 --- a/internal/config/di/definitions.go +++ b/internal/config/di/definitions.go @@ -114,8 +114,8 @@ var Definitions = []dingo.Def{ }, { Name: "address.indexer", - Build: func(elastic *elastic_cache.Index, repo *address.Repository) (*address.Indexer, error) { - return address.NewIndexer(elastic, repo), nil + Build: func(navcoin *navcoind.Navcoind, elastic *elastic_cache.Index, repo *address.Repository) (*address.Indexer, error) { + return address.NewIndexer(navcoin, elastic, repo), nil }, }, { diff --git a/internal/elastic_cache/Indices.go b/internal/elastic_cache/Indices.go index 00b50f0..a1b6bdf 100644 --- a/internal/elastic_cache/Indices.go +++ b/internal/elastic_cache/Indices.go @@ -8,17 +8,17 @@ import ( type Indices string var ( - AddressIndex Indices = "address" - AddressTransactionIndex Indices = "addresstransaction" - BlockIndex Indices = "block" - BlockTransactionIndex Indices = "blocktransaction" - ConsensusIndex Indices = "consensus" - ProposalIndex Indices = "proposal" - DaoVoteIndex Indices = "daovote" - DaoConsultationIndex Indices = "consultation" - PaymentRequestIndex Indices = "paymentrequest" - SignalIndex Indices = "signal" - SoftForkIndex Indices = "softfork" + AddressIndex Indices = "address" + AddressHistoryIndex Indices = "addresshistory" + BlockIndex Indices = "block" + BlockTransactionIndex Indices = "blocktransaction" + ConsensusIndex Indices = "consensus" + ProposalIndex Indices = "proposal" + DaoVoteIndex Indices = "daovote" + DaoConsultationIndex Indices = "consultation" + PaymentRequestIndex Indices = "paymentrequest" + SignalIndex Indices = "signal" + SoftForkIndex Indices = "softfork" ) // Sets the network and returns the full string @@ -29,7 +29,7 @@ func (i *Indices) Get() string { func All() []Indices { return []Indices{ AddressIndex, - AddressTransactionIndex, + AddressHistoryIndex, BlockIndex, BlockTransactionIndex, ConsensusIndex, diff --git a/internal/elastic_cache/elastic_cache.go b/internal/elastic_cache/elastic_cache.go index afca6e5..a90507a 100644 --- a/internal/elastic_cache/elastic_cache.go +++ b/internal/elastic_cache/elastic_cache.go @@ -192,7 +192,6 @@ func (i *Index) Persist() int { } func (i *Index) persist(bulk *elastic.BulkService) { - actions := bulk.NumberOfActions() response, err := bulk.Do(context.Background()) if err != nil { logrus.WithError(err).Fatal("Failed to persist requests") @@ -203,12 +202,10 @@ func (i *Index) persist(bulk *elastic.BulkService) { logrus.WithFields(logrus.Fields{"error": failed.Error}).Error("Failed to persist to ES") for { switch { - } } } } - logrus.Infof("Persisted %d actions", actions) } func (i *Index) DeleteHeightGT(height uint64, indices ...string) error { diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 8192fdc..7db69ee 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -13,6 +13,7 @@ import ( "github.com/getsentry/raven-go" log "github.com/sirupsen/logrus" "sync" + "time" ) type Indexer struct { @@ -87,6 +88,7 @@ func (i *Indexer) Index(option IndexOption.IndexOption) error { } func (i *Indexer) index(height uint64, option IndexOption.IndexOption) error { + start := time.Now() b, txs, header, err := i.blockIndexer.Index(height, option) if err != nil { if err.Error() != "-8: Block height out of range" { @@ -94,31 +96,40 @@ func (i *Indexer) index(height uint64, option IndexOption.IndexOption) error { } return err } - log.Infof("Indexed block at height %d", height) var wg sync.WaitGroup wg.Add(3) go func() { defer wg.Done() + start := time.Now() i.addressIndexer.Index(txs, b) - log.Infof("Indexed addresses at height %d", height) + elapsed := time.Since(start) + log.WithField("time", elapsed).Debugf("Indexed addresses at height %d", height) }() go func() { defer wg.Done() + start := time.Now() i.softForkIndexer.Index(b) - log.Infof("Indexed softforks at height %d", height) + elapsed := time.Since(start) + log.WithField("time", elapsed).Debugf("Indexed softforks at height %d", height) }() go func() { defer wg.Done() + start := time.Now() i.daoIndexer.Index(b, txs, header) - log.Infof("Indexed dao at height %d", height) + elapsed := time.Since(start) + log.WithField("time", elapsed).Debugf("Indexed dao at height %d", height) }() wg.Wait() + elapsed := time.Since(start) + log.WithField("time", elapsed).Debugf("Indexed block at height %d", height) + log.Debugf("") + LastBlockIndexed = height if option == IndexOption.BatchIndex { diff --git a/internal/service/address/factory.go b/internal/service/address/factory.go index 278daca..c6326b0 100644 --- a/internal/service/address/factory.go +++ b/internal/service/address/factory.go @@ -1,185 +1,62 @@ package address import ( - "encoding/json" + "github.com/NavExplorer/navcoind-go" "github.com/NavExplorer/navexplorer-indexer-go/pkg/explorer" - log "github.com/sirupsen/logrus" + "time" ) -func CreateAddress(hash string) *explorer.Address { - return &explorer.Address{Hash: hash} +func CreateAddress(hash string, height uint64, time time.Time) *explorer.Address { + return &explorer.Address{Hash: hash, CreatedBlock: height, CreatedTime: time} } -func ResetAddress(address *explorer.Address) { - address.ColdBalance = 0 - address.ColdReceived = 0 - address.ColdReceivedCount = 0 - address.ColdSent = 0 - address.ColdSentCount = 0 - address.ColdStaked = 0 - address.ColdStakedCount = 0 - address.Balance = 0 - address.Received = 0 - address.ReceivedCount = 0 - address.Sent = 0 - address.SentCount = 0 - address.Staked = 0 - address.StakedCount = 0 -} - -func ApplyTxToAddress(address *explorer.Address, tx *explorer.AddressTransaction) { - address.Height = tx.Height - - if tx.Cold == true { - if explorer.IsColdStake(tx.Type) { - address.ColdStaked += tx.Total - address.ColdStakedCount++ - } else if tx.Type == explorer.TransferSend { - address.ColdSent += tx.Total - address.ColdSentCount++ - } else if tx.Type == explorer.TransferReceive { - address.ColdReceived += tx.Total - address.ColdReceivedCount++ - } - } else { - if explorer.IsStake(tx.Type) || explorer.IsColdStake(tx.Type) { - address.Staked += tx.Total - address.StakedCount++ - } else if tx.Type == explorer.TransferSend { - address.Sent += tx.Total - address.SentCount++ - } else if tx.Type == explorer.TransferReceive { - address.Received += tx.Total - address.ReceivedCount++ - } else if tx.Type == explorer.TransferCommunityFundPayout { - address.Received += tx.Total - address.ReceivedCount++ - } else if tx.Type == explorer.TransferPoolFee { - address.Staked += tx.Total - address.StakedCount++ - } - } - - log.WithFields(log.Fields{"address": address.Hash, "staked": address.StakedCount, "sent": address.SentCount, "received": address.ReceivedCount}). - Debugf("Tx count at height %d", tx.Height) - - log.WithFields(log.Fields{"address": address.Hash, "tx": tx.Txid}). - Debugf("Balance at height %d: %d", tx.Height, tx.Balance) -} - -func CreateAddressTransaction(tx *explorer.BlockTransaction, block *explorer.Block) []*explorer.AddressTransaction { - addressTxs := make([]*explorer.AddressTransaction, 0) - for _, address := range tx.GetAllAddresses() { - if tx.HasColdInput(address) || tx.HasColdStakeStake(address) || tx.HasColdStakeReceive(address) { - if coldAddressTx := createColdTransaction(address, tx); coldAddressTx != nil { - addressTxs = append(addressTxs, coldAddressTx) +func CreateAddressHistory(history *navcoind.AddressHistory, tx *explorer.BlockTransaction, block *explorer.Block) *explorer.AddressHistory { + h := &explorer.AddressHistory{ + Height: history.Block, + TxIndex: history.TxIndex, + Time: time.Unix(history.Time, 0), + TxId: history.TxId, + Hash: history.Address, + Changes: explorer.AddressChanges{ + Spendable: history.Changes.Balance, + Stakable: history.Changes.Stakable, + VotingWeight: history.Changes.VotingWeight, + }, + Balance: explorer.AddressBalance{ + Spendable: history.Result.Balance, + Stakable: history.Result.Stakable, + VotingWeight: history.Result.VotingWeight, + }, + } + + hasPubKeyHashOutput := func() bool { + for _, v := range tx.Vout.WithAddress(h.Hash) { + if v.ScriptPubKey.Type == explorer.VoutPubkeyhash { + return true } } - if addressTx := createTransaction(address, tx, block); addressTx != nil { - addressTxs = append(addressTxs, addressTx) - } - } - - return addressTxs -} - -func createTransaction(address string, tx *explorer.BlockTransaction, block *explorer.Block) *explorer.AddressTransaction { - _, input := tx.Vin.GetAmountByAddress(address, false) - _, output := tx.Vout.GetAmountByAddress(address, false) - if input+output == 0 { - return nil + return false } + if history.Changes.Flags == 1 { + h.CfundPayout = tx.Type == explorer.TxCoinbase && tx.Version == 3 && hasPubKeyHashOutput() - addressTransaction := &explorer.AddressTransaction{ - Hash: address, - Txid: tx.Hash, - Height: tx.Height, - Index: tx.Index, - Time: tx.Time, - Cold: false, - Input: input, - Output: output, - Total: int64(output - input), - } - - if tx.IsStaking() { - addressTransaction.Type = explorer.TransferStake - } else if tx.IsCoinbase() { - if block.StakedBy == address { - // POW block_indexer - addressTransaction.Type = explorer.TransferStake - } else if tx.Version == 3 { - addressTransaction.Type = explorer.TransferCommunityFundPayout - } else { - bt, _ := json.Marshal(tx) - log.WithFields(log.Fields{"tx": string(bt)}).Fatal("Could not handle coinbase") - } - } else if tx.IsColdStaking() { - if addressTransaction.Input == addressTransaction.Output { - addressTransaction.Type = explorer.TransferStake - } else if tx.HasColdStakeSpend(address) { - if tx.Vin.HasAddress(address) { - addressTransaction.Type = explorer.TransferColdStake - } else { - addressTransaction.Type = explorer.TransferColdDelegateStake - } - } else if !tx.Vin.HasAddress(address) && tx.Vout.HasAddress(address) { - addressTransaction.Type = explorer.TransferPoolFee - } else if addressTransaction.Input == 0 { - addressTransaction.Type = explorer.TransferReceive - } - } else { - if addressTransaction.Input > addressTransaction.Output { - addressTransaction.Type = explorer.TransferSend + if tx.Vout.Count() > 1 && !tx.Vout[1].HasAddress(h.Hash) { + h.StakePayout = true } else { - addressTransaction.Type = explorer.TransferReceive + h.Stake = true } } - if addressTransaction.Type == "" { - bt, _ := json.Marshal(tx) - log.WithFields(log.Fields{"tx": string(bt)}).Fatal("addressTransaction.Type not identified: ", address) - return nil - } - - return addressTransaction -} - -func createColdTransaction(address string, tx *explorer.BlockTransaction) *explorer.AddressTransaction { - _, input := tx.Vin.GetAmountByAddress(address, true) - _, output := tx.Vout.GetAmountByAddress(address, true) - if input+output == 0 { - return nil - } - - addressTransaction := &explorer.AddressTransaction{ - Hash: address, - Txid: tx.Hash, - Height: tx.Height, - Index: tx.Index, - Time: tx.Time, - Cold: true, - Input: input, - Output: output, - Total: int64(output - input), - } - - if tx.IsSpend() { - if addressTransaction.Input > addressTransaction.Output { - addressTransaction.Type = explorer.TransferSend - } else { - addressTransaction.Type = explorer.TransferReceive + if h.IsSpend() { + switch tx.Version { + case 4: + h.Changes.Proposal = true + case 5: + h.Changes.PaymentRequest = true + case 6: + h.Changes.Consultation = true } - } else if tx.IsColdStaking() { - addressTransaction.Type = explorer.TransferColdStake - } else { - bt, _ := json.Marshal(tx) - log.WithFields(log.Fields{ - "address": address, - "tx": string(bt), - "type": tx.Type, - }).Fatal("WE FOUND SOMETHING ELSE IN COLD TX") } - return addressTransaction + return h } diff --git a/internal/service/address/indexer.go b/internal/service/address/indexer.go index d8e484c..3ba45bf 100644 --- a/internal/service/address/indexer.go +++ b/internal/service/address/indexer.go @@ -1,79 +1,92 @@ package address import ( + "github.com/NavExplorer/navcoind-go" "github.com/NavExplorer/navexplorer-indexer-go/internal/elastic_cache" "github.com/NavExplorer/navexplorer-indexer-go/pkg/explorer" - "github.com/getsentry/raven-go" log "github.com/sirupsen/logrus" ) type Indexer struct { + navcoin *navcoind.Navcoind elastic *elastic_cache.Index repo *Repository } -func NewIndexer(elastic *elastic_cache.Index, repo *Repository) *Indexer { - return &Indexer{elastic, repo} +func NewIndexer(navcoin *navcoind.Navcoind, elastic *elastic_cache.Index, repo *Repository) *Indexer { + return &Indexer{navcoin, elastic, repo} } -func (i *Indexer) GenerateAddressTransactions(address *explorer.Address, tx *explorer.BlockTransaction, block *explorer.Block) []*explorer.AddressTransaction { - txs := make([]*explorer.AddressTransaction, 0) - for _, tx := range CreateAddressTransaction(tx, block) { - if tx.Hash != address.Hash { - continue - } - if tx.Cold == true { - address.ColdBalance += tx.Total - tx.Balance = uint64(address.ColdBalance) - } else { - address.Balance += tx.Total - tx.Balance = uint64(address.Balance) +func (i *Indexer) Index(txs []*explorer.BlockTransaction, block *explorer.Block) { + if len(txs) == 0 { + return + } + + for _, addressHistory := range i.generateAddressHistory(block, txs) { + i.elastic.AddIndexRequest(elastic_cache.AddressHistoryIndex.Get(), addressHistory) + + err := i.updateAddress(addressHistory, block) + if err != nil { + log.WithError(err).Fatalf("Could not update address: %s", addressHistory.Hash) } - txs = append(txs, tx) } - return txs } -func (i *Indexer) Index(txs []*explorer.BlockTransaction, block *explorer.Block) { - if len(txs) == 0 { - return +func (i *Indexer) generateAddressHistory(block *explorer.Block, txs []*explorer.BlockTransaction) []*explorer.AddressHistory { + addressHistory := make([]*explorer.AddressHistory, 0) + + addresses := getAddressesForTxs(txs) + history, err := i.navcoin.GetAddressHistory(&block.Height, &block.Height, addresses...) + if err != nil { + log.WithError(err).Errorf("Could not get address history for height: %d", block.Height) + return addressHistory + } + + for _, h := range history { + addressHistory = append(addressHistory, CreateAddressHistory(h, getTxsById(h.TxId, txs), block)) } - hashes := make([]string, 0) + return addressHistory +} + +func (i *Indexer) updateAddress(history *explorer.AddressHistory, block *explorer.Block) error { + address := Addresses.GetByHash(history.Hash) + if address == nil { + var err error + address, err = i.repo.GetOrCreateAddress(history.Hash, block) + if err != nil { + return err + } + } + + address.Height = history.Height + address.Spendable = history.Balance.Spendable + address.Stakable = history.Balance.Stakable + address.VotingWeight = history.Balance.VotingWeight + + Addresses[address.Hash] = address + + i.elastic.AddUpdateRequest(elastic_cache.AddressIndex.Get(), address) + return nil +} + +func getAddressesForTxs(txs []*explorer.BlockTransaction) []string { + addresses := make([]string, 0) for _, tx := range txs { - for _, addressTx := range CreateAddressTransaction(tx, block) { - if Addresses.GetByHash(addressTx.Hash) == nil { - address, err := i.repo.GetOrCreateAddress(addressTx.Hash) - if err != nil { - raven.CaptureError(err, nil) - log.WithError(err).Fatalf("Could not get or create address: %s", addressTx.Hash) - } - Addresses[addressTx.Hash] = address - } - - if addressTx.Cold == true { - addressTx.Balance = uint64(Addresses[addressTx.Hash].ColdBalance + addressTx.Total) - } else { - addressTx.Balance = uint64(Addresses[addressTx.Hash].Balance + addressTx.Total) - } - - i.elastic.AddIndexRequest(elastic_cache.AddressTransactionIndex.Get(), addressTx) - - ApplyTxToAddress(Addresses[addressTx.Hash], addressTx) - if addressTx.Cold == true { - Addresses[addressTx.Hash].ColdBalance += addressTx.Total - } else { - Addresses[addressTx.Hash].Balance += addressTx.Total - } - - i.elastic.AddUpdateRequest(elastic_cache.AddressIndex.Get(), Addresses[addressTx.Hash]) + for _, address := range tx.GetAllAddresses() { + addresses = append(addresses, address) } } - if len(hashes) > 0 { - if _, err := i.repo.GetAddresses(hashes); err != nil { - raven.CaptureError(err, nil) - log.WithError(err).Fatalf("Could not get addresses for txs at height %d", txs[0].Height) + return addresses +} + +func getTxsById(txid string, txs []*explorer.BlockTransaction) *explorer.BlockTransaction { + for _, tx := range txs { + if tx.Txid == txid { + return tx } } + log.Fatal("Could not match tx") + return nil } diff --git a/internal/service/address/repository.go b/internal/service/address/repository.go index 9f39784..a5c893b 100644 --- a/internal/service/address/repository.go +++ b/internal/service/address/repository.go @@ -9,7 +9,6 @@ import ( "github.com/getsentry/raven-go" "github.com/olivere/elastic/v7" log "github.com/sirupsen/logrus" - "io" "strings" ) @@ -17,10 +16,38 @@ type Repository struct { Client *elastic.Client } +var ( + ErrLatestHistoryNotFound = errors.New("Latest history not found") +) + func NewRepo(Client *elastic.Client) *Repository { return &Repository{Client} } +func (r *Repository) GetAddress(hash string) (*explorer.Address, error) { + log.Debugf("GetAddress(hash:%s)", hash) + + results, err := r.Client.Search(elastic_cache.AddressIndex.Get()). + Query(elastic.NewTermQuery("hash.keyword", hash)). + Size(1). + Do(context.Background()) + if err != nil { + raven.CaptureError(err, nil) + return nil, err + } + + if results == nil || len(results.Hits.Hits) != 1 { + return nil, errors.New("Invalid result found") + } + + var address *explorer.Address + if err = json.Unmarshal(results.Hits.Hits[0].Source, &address); err != nil { + return nil, err + } + + return address, nil +} + func (r *Repository) GetAddresses(hashes []string) ([]*explorer.Address, error) { log.Debugf("GetAddresses([%s])", strings.Join(hashes, ",")) @@ -44,61 +71,31 @@ func (r *Repository) GetAddresses(hashes []string) ([]*explorer.Address, error) return addresses, nil } -func (r *Repository) GetAddressesHeightGt(height uint64) ([]string, error) { +func (r *Repository) GetAddressesHeightGt(height uint64) ([]*explorer.Address, error) { log.Debugf("GetAddressesHeightGt(height:%d)", height) - addresses := make(map[string]struct{}, 0) - getAddresses := func(addresses map[string]struct{}, results *elastic.SearchResult) map[string]struct{} { - if agg, found := results.Aggregations.Terms("hash"); found { - for _, bucket := range agg.Buckets { - addresses[bucket.Key.(string)] = struct{}{} - } - } - return addresses - } - getAddressesHeightGtByIndex := func(height uint64, index elastic_cache.Indices) (*elastic.SearchResult, error) { - return r.Client. - Search(index.Get()). - Query(elastic.NewRangeQuery("height").Gt(height)). - Aggregation("hash", elastic.NewTermsAggregation().Field("hash.keyword").Size(50000000)). - Size(0). - Do(context.Background()) - } - - results, err := getAddressesHeightGtByIndex(height, elastic_cache.AddressIndex) - if err != nil || results == nil { - raven.CaptureError(err, nil) - return nil, err - } - addresses = getAddresses(addresses, results) - - results, err = getAddressesHeightGtByIndex(height, elastic_cache.AddressTransactionIndex) + results, err := r.Client. + Search(elastic_cache.AddressIndex.Get()). + Query(elastic.NewRangeQuery("height").Gt(height)). + Size(50000). + Do(context.Background()) if err != nil || results == nil { - raven.CaptureError(err, nil) return nil, err } - addresses = getAddresses(addresses, results) - addressesSlice := make([]string, len(addresses)) - i := 0 - for f, _ := range addresses { - addressesSlice[i] = f - i++ + addresses := make([]*explorer.Address, 0) + for _, hit := range results.Hits.Hits { + var address *explorer.Address + if err = json.Unmarshal(hit.Source, &address); err != nil { + return nil, err + } + addresses = append(addresses, address) } - return addressesSlice, nil -} - -func (r *Repository) getAddressesHeightGtByIndex(height uint64, index elastic_cache.Indices) (*elastic.SearchResult, error) { - return r.Client. - Search(index.Get()). - Query(elastic.NewRangeQuery("height").Gt(height)). - Aggregation("hash", elastic.NewTermsAggregation().Field("hash.keyword").Size(50000000)). - Size(0). - Do(context.Background()) + return addresses, nil } -func (r *Repository) GetOrCreateAddress(hash string) (*explorer.Address, error) { +func (r *Repository) GetOrCreateAddress(hash string, block *explorer.Block) (*explorer.Address, error) { log.WithField("address", hash).Debug("GetOrCreateAddress") results, err := r.Client. @@ -111,8 +108,8 @@ func (r *Repository) GetOrCreateAddress(hash string) (*explorer.Address, error) } var address *explorer.Address - if len(results.Hits.Hits) == 0 { - address = CreateAddress(hash) + if results.TotalHits() == 0 { + address = CreateAddress(hash, block.Height, block.MedianTime) _, err := r.Client. Index(). Index(elastic_cache.AddressIndex.Get()). @@ -133,136 +130,27 @@ func (r *Repository) GetOrCreateAddress(hash string) (*explorer.Address, error) return address, nil } -func (r *Repository) GetAddress(hash string) (*explorer.Address, error) { - log.Debugf("GetAddress(hash:%s)", hash) +func (r *Repository) GetLatestHistoryByHash(hash string) (*explorer.AddressHistory, error) { + query := elastic.NewBoolQuery() + query = query.Must(elastic.NewTermQuery("hash.keyword", hash)) - results, err := r.Client. - Search(elastic_cache.AddressIndex.Get()). - Query(elastic.NewMatchQuery("hash", hash)). + results, err := r.Client.Search(elastic_cache.AddressHistoryIndex.Get()). + Query(query). + Sort("height", false). Size(1). Do(context.Background()) - if err != nil { - raven.CaptureError(err, nil) - return nil, err - } - - if results == nil || len(results.Hits.Hits) != 1 { - return nil, errors.New("Invalid result found") - } - - var address *explorer.Address - if err = json.Unmarshal(results.Hits.Hits[0].Source, &address); err != nil { + if err != nil || results.TotalHits() == 0 { + log.WithError(err).Error("Failed to find address") + err = ErrLatestHistoryNotFound return nil, err } - return address, nil -} - -func (r *Repository) GetTxsRangeForAddress(hash string, from uint64, to uint64) ([]*explorer.AddressTransaction, error) { - log.WithField("address", hash).Debugf("GetTxsRangeForAddress: from:%d to:%d", from, to) - - query := elastic.NewBoolQuery() - query = query.Must(elastic.NewMatchQuery("hash", hash)) - query = query.Must(elastic.NewRangeQuery("height").Gt(from).Lte(to)) - - service := r.Client.Scroll(elastic_cache.AddressTransactionIndex.Get()).Query(query).Size(10000).Sort("height", true) - txs := make([]*explorer.AddressTransaction, 0) - - for { - results, err := service.Do(context.Background()) - if err == io.EOF { - break - } - if err != nil || results == nil { - log.Fatal(err) - } - - for _, hit := range results.Hits.Hits { - var tx *explorer.AddressTransaction - if err = json.Unmarshal(hit.Source, &tx); err != nil { - raven.CaptureError(err, nil) - return nil, err - } - txs = append(txs, tx) - } - } - - return txs, nil -} - -func (r *Repository) GetTxsForAddress(hash string) ([]*explorer.AddressTransaction, error) { - log.Debugf("GetTxsForAddress(hash:%s)", hash) - - results, err := r.Client. - Search(elastic_cache.AddressTransactionIndex.Get()). - Query(elastic.NewMatchQuery("hash", hash)). - Size(50000000). - Sort("height", true). - Do(context.Background()) + var history *explorer.AddressHistory + hit := results.Hits.Hits[0] + err = json.Unmarshal(hit.Source, &history) if err != nil { - raven.CaptureError(err, nil) - return nil, err - } - - txs := make([]*explorer.AddressTransaction, 0) - for _, hit := range results.Hits.Hits { - var tx *explorer.AddressTransaction - if err = json.Unmarshal(hit.Source, &tx); err != nil { - raven.CaptureError(err, nil) - return nil, err - } - txs = append(txs, tx) - } - - return txs, nil -} - -func (r *Repository) GetAddressesByValidateAtDesc(size int) ([]*explorer.Address, error) { - log.Debug("ValidateAddresses()") - - results, err := r.Client.Search(elastic_cache.AddressIndex.Get()). - Size(size). - Sort("validatedAt", true). - TrackTotalHits(true). - Do(context.Background()) - if err != nil || results == nil { return nil, err } - addresses := make([]*explorer.Address, 0) - for _, hit := range results.Hits.Hits { - var address *explorer.Address - if err = json.Unmarshal(hit.Source, &address); err != nil { - return nil, err - } - addresses = append(addresses, address) - } - - return addresses, nil -} - -func (r *Repository) GetAllAddresses() ([]*explorer.Address, error) { - service := r.Client.Scroll(elastic_cache.AddressIndex.Get()).Size(10000).Sort("hash.keyword", true) - addresses := make([]*explorer.Address, 0) - - for { - results, err := service.Do(context.Background()) - if err == io.EOF { - break - } - if err != nil || results == nil { - log.Fatal(err) - } - - for _, hit := range results.Hits.Hits { - var a *explorer.Address - if err = json.Unmarshal(hit.Source, &a); err != nil { - raven.CaptureError(err, nil) - return nil, err - } - addresses = append(addresses, a) - } - } - - return addresses, nil + return history, err } diff --git a/internal/service/address/rewinder.go b/internal/service/address/rewinder.go index 204838e..e47a7e2 100644 --- a/internal/service/address/rewinder.go +++ b/internal/service/address/rewinder.go @@ -4,7 +4,6 @@ import ( "context" "github.com/NavExplorer/navexplorer-indexer-go/internal/elastic_cache" "github.com/NavExplorer/navexplorer-indexer-go/pkg/explorer" - "github.com/getsentry/raven-go" log "github.com/sirupsen/logrus" ) @@ -26,26 +25,14 @@ func (r *Rewinder) Rewind(height uint64) error { return err } - log.Infof("Rewinding %d addresses", len(addresses)) - err = r.elastic.DeleteHeightGT(height, elastic_cache.AddressTransactionIndex.Get()) + err = r.elastic.DeleteHeightGT(height, elastic_cache.AddressHistoryIndex.Get()) if err != nil { - log.Error("Failed to delete address transactions greater than ", height) + log.Error("Failed to delete address history greater than ", height) return err } - for idx, hash := range addresses { - Addresses.Delete(hash) - - log.Infof("Rewinding address %d: %s", idx+1, hash) - address, err := r.repo.GetAddress(hash) - if err != nil { - log.Error("Failed to get address with hash ", hash) - return err - } - - err = r.RewindAddressToHeight(address, height) - if err != nil { - log.Errorf("Failed to rewind address %s to height %s", address, height) + for _, address := range addresses { + if err = r.ResetAddress(address); err != nil { return err } } @@ -53,41 +40,25 @@ func (r *Rewinder) Rewind(height uint64) error { return nil } -func (r *Rewinder) ResetAddressToHeight(address *explorer.Address, height uint64) error { - log.Infof("Resetting address %s from %d to %d", address.Hash, 0, height) - - ResetAddress(address) - address.Height = height +func (r *Rewinder) ResetAddress(address *explorer.Address) error { + log.Infof("Resetting address: %s", address.Hash) - addressTxs, err := r.repo.GetTxsRangeForAddress(address.Hash, 0, height) - if err != nil { - log.WithField("address", address.Hash).WithError(err).Error("Failed to get tx range") - raven.CaptureError(err, nil) + latestHistory, err := r.repo.GetLatestHistoryByHash(address.Hash) + if err != nil && err != ErrLatestHistoryNotFound { return err } - for _, addressTx := range addressTxs { - ApplyTxToAddress(address, addressTx) - if addressTx.Cold == true { - address.ColdBalance = int64(addressTx.Balance) - } else { - address.Balance = int64(addressTx.Balance) - log.Debugf("Setting address balance to %d", address.Balance) - } - address.Height = addressTx.Height - } - - return nil -} - -func (r *Rewinder) RewindAddressToHeight(address *explorer.Address, height uint64) error { - log.WithField("address", address.Hash).Debugf("Rewind balance to height %d: %d", address.Height, address.Balance) - - err := r.ResetAddressToHeight(address, height) - if err != nil { - log.WithField("address", address.Hash).WithError(err).Error("Failed to rest the address") - raven.CaptureError(err, nil) - return err + if latestHistory == nil { + log.Error("Failed to find latest history for ", address.Hash) + address.Height = 0 + address.Spendable = 0 + address.Stakable = 0 + address.VotingWeight = 0 + } else { + address.Height = latestHistory.Height + address.Spendable = latestHistory.Balance.Spendable + address.Stakable = latestHistory.Balance.Stakable + address.VotingWeight = latestHistory.Balance.VotingWeight } _, err = r.elastic.Client.Index(). @@ -96,11 +67,5 @@ func (r *Rewinder) RewindAddressToHeight(address *explorer.Address, height uint6 Id(address.Slug()). Do(context.Background()) - if err != nil { - log.WithField("address", address.Hash).WithError(err).Error("Failed to persist the rewind") - raven.CaptureError(err, nil) - return err - } - return nil } diff --git a/internal/service/block/indexer.go b/internal/service/block/indexer.go index fda8ba3..3155c16 100644 --- a/internal/service/block/indexer.go +++ b/internal/service/block/indexer.go @@ -26,10 +26,12 @@ func NewIndexer(navcoin *navcoind.Navcoind, elastic *elastic_cache.Index, orphan func (i *Indexer) Index(height uint64, option IndexOption.IndexOption) (*explorer.Block, []*explorer.BlockTransaction, *navcoind.BlockHeader, error) { navBlock, err := i.getBlockAtHeight(height) if err != nil { + log.Error("Failed to get block at height ", height) return nil, nil, nil, err } header, err := i.navcoin.GetBlockheader(navBlock.Hash) if err != nil { + log.Error("Failed to get header at height ", height) return nil, nil, nil, err } @@ -39,23 +41,26 @@ func (i *Indexer) Index(height uint64, option IndexOption.IndexOption) (*explore if err != nil { log.WithError(err).Errorf("Failed to parse header.NcfSupply: %s", header.NcfSupply) } + locked, err := strconv.ParseFloat(header.NcfLocked, 64) if err != nil { log.WithError(err).Errorf("Failed to parse header.NcfLocked: %s", header.NcfLocked) } - block.Cfund = &explorer.Cfund{Available: available, Locked: locked} - LastBlockIndexed = block + block.Cfund = &explorer.Cfund{Available: available, Locked: locked} if option == IndexOption.SingleIndex { log.Info("Indexing in single block mode") - orphan, err := i.orphanService.IsOrphanBlock(block) + orphan, err := i.orphanService.IsOrphanBlock(block, LastBlockIndexed) if orphan == true || err != nil { log.WithFields(log.Fields{"block": block, "orphan": orphan}).WithError(err).Info("Orphan Block Found") - + LastBlockIndexed = nil return nil, nil, nil, ErrOrphanBlockFound } } + LastBlockIndexed = block + + i.elastic.AddIndexRequest(elastic_cache.BlockIndex.Get(), block) var txs = make([]*explorer.BlockTransaction, 0) for idx, txHash := range block.Tx { diff --git a/internal/service/block/orphan.go b/internal/service/block/orphan.go index 358b409..e55e486 100644 --- a/internal/service/block/orphan.go +++ b/internal/service/block/orphan.go @@ -3,7 +3,6 @@ package block import ( "errors" "github.com/NavExplorer/navexplorer-indexer-go/pkg/explorer" - "github.com/getsentry/raven-go" log "github.com/sirupsen/logrus" "time" ) @@ -20,7 +19,7 @@ var ( ErrOrphanBlockFound = errors.New("Orphan block_indexer found") ) -func (o *OrphanService) IsOrphanBlock(block *explorer.Block) (bool, error) { +func (o *OrphanService) IsOrphanBlock(block *explorer.Block, previousBlock *explorer.Block) (bool, error) { if block.Height == 1 { return false, nil } @@ -29,19 +28,21 @@ func (o *OrphanService) IsOrphanBlock(block *explorer.Block) (bool, error) { return o.repository.GetBlockByHeight(height - 1) } - previousBlock, err := getPreviousBlock(block.Height) - if err != nil { - log.Info("Retry get previous block in 5 seconds") - time.Sleep(5 * time.Second) + if previousBlock == nil { + var err error previousBlock, err = getPreviousBlock(block.Height) if err != nil { - log.WithError(err).WithField("height", block.Height-1).Fatal("Failed to get previous block") + log.Info("Retry get previous block in 1 seconds") + time.Sleep(1 * time.Second) + previousBlock, err = getPreviousBlock(block.Height) + if err != nil { + log.WithError(err).WithField("height", block.Height-1).Fatal("Failed to get previous block") + } } } orphan := previousBlock.Hash != block.Previousblockhash if orphan == true { - raven.CaptureError(err, nil) log.WithFields(log.Fields{"height": block.Height, "block": block.Hash, "previous": previousBlock.Hash}).Info("Orphan block found") } diff --git a/internal/service/block/repository.go b/internal/service/block/repository.go index 7f2e4bf..b811e39 100644 --- a/internal/service/block/repository.go +++ b/internal/service/block/repository.go @@ -77,6 +77,34 @@ func (r *Repository) GetBlockByHash(hash string) (*explorer.Block, error) { return block, nil } +func (r *Repository) GetTransactionsByBlock(block *explorer.Block) ([]*explorer.BlockTransaction, error) { + results, err := r.elastic.Client. + Search(elastic_cache.BlockTransactionIndex.Get()). + Query(elastic.NewMatchQuery("height", block.Height)). + Do(context.Background()) + if err != nil || results == nil { + raven.CaptureError(err, nil) + return nil, err + } + + if len(results.Hits.Hits) == 0 { + raven.CaptureError(err, nil) + return nil, elastic_cache.ErrRecordNotFound + } + + var txs []*explorer.BlockTransaction + for _, hit := range results.Hits.Hits { + var tx *explorer.BlockTransaction + if err = json.Unmarshal(hit.Source, &tx); err != nil { + raven.CaptureError(err, nil) + return nil, err + } + txs = append(txs, tx) + } + + return txs, nil +} + func (r *Repository) GetTransactionByHash(hash string) (*explorer.BlockTransaction, error) { request := r.elastic.GetRequest(explorer.CreateBlockTxSlug(hash)) if request != nil { @@ -147,6 +175,31 @@ func (r *Repository) GetBlockByHeight(height uint64) (*explorer.Block, error) { return block, nil } +func (r *Repository) GetBlocksBetweenHeight(start uint64, end uint64) ([]*explorer.Block, error) { + results, err := r.elastic.Client. + Search(elastic_cache.BlockIndex.Get()). + Query(elastic.NewRangeQuery("height").Gte(start).Lt(end)). + Sort("height", true). + Size(int(end - start)). + Do(context.Background()) + if err != nil || results == nil { + raven.CaptureError(err, nil) + return nil, err + } + + blocks := make([]*explorer.Block, 0) + for _, hit := range results.Hits.Hits { + var block *explorer.Block + if err = json.Unmarshal(hit.Source, &block); err != nil { + raven.CaptureError(err, nil) + return nil, err + } + blocks = append(blocks, block) + } + + return blocks, nil +} + func (r *Repository) GetTransactionsWithCfundPayment() error { query := elastic.NewBoolQuery() query = query.Must(elastic.NewMatchQuery("vout.scriptPubKey.type.keyword", explorer.VoutCfundContribution)) diff --git a/internal/service/dao/consensus/service.go b/internal/service/dao/consensus/service.go index a8ad40e..48c6873 100644 --- a/internal/service/dao/consensus/service.go +++ b/internal/service/dao/consensus/service.go @@ -54,9 +54,13 @@ func (s *Service) InitConsensusParameters() { func (s *Service) InitialState() ([]*explorer.ConsensusParameter, error) { parameters := make([]*explorer.ConsensusParameter, 0) - byteParams := []byte(mainnet) - if config.Get().Network != "mainnet" { + var byteParams []byte + if config.Get().SoftForkBlockCycle != 20160 { + log.Info("Initialising Testnet Consensus parameters") byteParams = []byte(testnet) + } else { + log.Info("Initialising Mainnet Consensus parameters") + byteParams = []byte(mainnet) } err := json.Unmarshal(byteParams, ¶meters) diff --git a/internal/service/dao/indexer.go b/internal/service/dao/indexer.go index 5e187b6..18ec60e 100644 --- a/internal/service/dao/indexer.go +++ b/internal/service/dao/indexer.go @@ -8,8 +8,8 @@ import ( "github.com/NavExplorer/navexplorer-indexer-go/internal/service/dao/proposal" "github.com/NavExplorer/navexplorer-indexer-go/internal/service/dao/vote" "github.com/NavExplorer/navexplorer-indexer-go/pkg/explorer" - "github.com/getsentry/raven-go" log "github.com/sirupsen/logrus" + "sync" ) type Indexer struct { @@ -40,17 +40,34 @@ func NewIndexer( } func (i *Indexer) Index(block *explorer.Block, txs []*explorer.BlockTransaction, header *navcoind.BlockHeader) { - if consensus.Parameters == nil { - err := i.consensusIndexer.Index() - if err != nil { - raven.CaptureError(err, nil) - log.WithError(err).Fatal("Failed to get Consensus") + var wg sync.WaitGroup + wg.Add(4) + + go func() { + defer wg.Done() + if consensus.Parameters == nil { + if err := i.consensusIndexer.Index(); err != nil { + log.WithError(err).Fatal("Failed to get Consensus") + } } - } + }() + + go func() { + defer wg.Done() + i.proposalIndexer.Index(txs) + }() + + go func() { + defer wg.Done() + i.paymentRequestIndexer.Index(txs) + }() + + go func() { + defer wg.Done() + i.consultationIndexer.Index(txs) + }() - i.proposalIndexer.Index(txs) - i.paymentRequestIndexer.Index(txs) - i.consultationIndexer.Index(txs) + wg.Wait() i.voteIndexer.IndexVotes(txs, block, header) i.proposalIndexer.Update(block.BlockCycle, block) @@ -58,10 +75,9 @@ func (i *Indexer) Index(block *explorer.Block, txs []*explorer.BlockTransaction, i.consultationIndexer.Update(block.BlockCycle, block) if block.BlockCycle.IsEnd() { - log.WithFields(log.Fields{ - "size": block.BlockCycle.Size, - "height": block.Height, - }).Infof("Blockcycle %d complete", block.BlockCycle.Cycle) + log. + WithFields(log.Fields{"size": block.BlockCycle.Size, "height": block.Height}). + Infof("Blockcycle %d complete", block.BlockCycle.Cycle) i.consensusIndexer.Update(block) } } diff --git a/internal/service/dao/proposal/repository.go b/internal/service/dao/proposal/repository.go index 1fe0bda..69a1aeb 100644 --- a/internal/service/dao/proposal/repository.go +++ b/internal/service/dao/proposal/repository.go @@ -22,7 +22,7 @@ func (r *Repository) GetPossibleVotingProposals(height uint64) ([]*explorer.Prop var proposals []*explorer.Proposal query := elastic.NewBoolQuery() - query = query.Should(elastic.NewMatchQuery("status", "pending accepted pending_voting_preq")) + query = query.Should(elastic.NewMatchQuery("status", "pending accepted pending_voting_preq pending_funds")) query = query.Should(elastic.NewRangeQuery("updatedOnBlock").Gte(height)) results, err := r.Client.Search(elastic_cache.ProposalIndex.Get()). diff --git a/internal/service/dao/vote/factory.go b/internal/service/dao/vote/factory.go index b5d7c0b..2712ce0 100644 --- a/internal/service/dao/vote/factory.go +++ b/internal/service/dao/vote/factory.go @@ -6,12 +6,12 @@ import ( log "github.com/sirupsen/logrus" ) -func CreateVotes(block *explorer.Block, tx *explorer.BlockTransaction, header *navcoind.BlockHeader) *explorer.DaoVotes { +func CreateVotes(block *explorer.Block, tx *explorer.BlockTransaction, header *navcoind.BlockHeader, votingAddress string) *explorer.DaoVotes { if !tx.IsCoinbase() { return nil } - daoVote := &explorer.DaoVotes{Cycle: block.BlockCycle.Cycle, Height: tx.Height, Address: block.StakedBy} + daoVote := &explorer.DaoVotes{Cycle: block.BlockCycle.Cycle, Height: tx.Height, Address: votingAddress} for _, v := range header.CfundVotes { vote := explorer.Vote{Type: explorer.ProposalVote, Hash: v.Hash, Vote: v.Vote} diff --git a/internal/service/dao/vote/indexer.go b/internal/service/dao/vote/indexer.go index 87398da..e9df8d4 100644 --- a/internal/service/dao/vote/indexer.go +++ b/internal/service/dao/vote/indexer.go @@ -4,6 +4,7 @@ import ( "github.com/NavExplorer/navcoind-go" "github.com/NavExplorer/navexplorer-indexer-go/internal/elastic_cache" "github.com/NavExplorer/navexplorer-indexer-go/pkg/explorer" + log "github.com/sirupsen/logrus" ) type Indexer struct { @@ -15,12 +16,25 @@ func NewIndexer(elastic *elastic_cache.Index) *Indexer { } func (i *Indexer) IndexVotes(txs []*explorer.BlockTransaction, block *explorer.Block, blockHeader *navcoind.BlockHeader) { + var votingAddress string + for _, tx := range txs { + var err error + votingAddress, err = tx.Vout.GetVotingAddress() + if err == nil && votingAddress != "" { + break + } + } + if votingAddress == "" { + log.WithField("height", block.Height).Error("Unable to identify the voting address") + return + } + for _, tx := range txs { if !tx.IsCoinbase() { continue } - if v := CreateVotes(block, tx, blockHeader); v != nil { + if v := CreateVotes(block, tx, blockHeader, votingAddress); v != nil { i.elastic.AddIndexRequest(elastic_cache.DaoVoteIndex.Get(), v) return } diff --git a/internal/service/softfork/indexer.go b/internal/service/softfork/indexer.go index 0c48ea0..401e35a 100644 --- a/internal/service/softfork/indexer.go +++ b/internal/service/softfork/indexer.go @@ -20,7 +20,7 @@ func NewIndexer(elastic *elastic_cache.Index, blocksInCycle uint, quorum int) *I func (i *Indexer) Index(block *explorer.Block) { signal := signal.CreateSignal(block, &SoftForks) if signal != nil { - i.updateSoftForks(signal, block) + i.updateSoftForks(signal, block.Height) } if block.BlockCycle.IsEnd() { @@ -39,6 +39,7 @@ func (i *Indexer) Index(block *explorer.Block) { if signal != nil { for _, s := range signal.SoftForks { if SoftForks.GetSoftFork(s) != nil && SoftForks.GetSoftFork(s).State == explorer.SoftForkActive { + log.Info("Delete the active softForks") signal.DeleteSoftFork(s) } } @@ -46,14 +47,13 @@ func (i *Indexer) Index(block *explorer.Block) { i.elastic.AddIndexRequest(elastic_cache.SignalIndex.Get(), signal) } } - } -func (i *Indexer) updateSoftForks(signal *explorer.Signal, block *explorer.Block) { +func (i *Indexer) updateSoftForks(signal *explorer.Signal, height uint64) { if signal == nil || !signal.IsSignalling() { return } - blockCycle := GetSoftForkBlockCycle(i.blocksInCycle, block.Height) + blockCycle := GetSoftForkBlockCycle(i.blocksInCycle, height) for _, s := range signal.SoftForks { softFork := SoftForks.GetSoftFork(s) @@ -65,12 +65,6 @@ func (i *Indexer) updateSoftForks(signal *explorer.Signal, block *explorer.Block softFork.State = explorer.SoftForkStarted } - if softFork.State == explorer.SoftForkStarted { - softFork.LockedInHeight = new(explorer.SoftFork).LockedInHeight - softFork.ActivationHeight = new(explorer.SoftFork).ActivationHeight - softFork.SignalHeight = block.Height - } - var cycle *explorer.SoftForkCycle if cycle = softFork.GetCycle(blockCycle.Cycle); cycle == nil { softFork.Cycles = append(softFork.Cycles, explorer.SoftForkCycle{Cycle: blockCycle.Cycle, BlocksSignalling: 0}) @@ -90,18 +84,23 @@ func (i *Indexer) updateState(block *explorer.Block) { } if SoftForks[idx].State == explorer.SoftForkStarted && block.Height >= SoftForks[idx].LockedInHeight { - log.Info("Moved softfork to locked in") - SoftForks[idx].State = explorer.SoftForkLockedIn - SoftForks[idx].LockedInHeight = uint64(i.blocksInCycle * GetSoftForkBlockCycle(i.blocksInCycle, block.Height).Cycle) - SoftForks[idx].ActivationHeight = SoftForks[idx].LockedInHeight + uint64(i.blocksInCycle) - if SoftForks[idx].Cycles[len(SoftForks[idx].Cycles)-1].BlocksSignalling == 1 { - SoftForks[idx].Cycles = SoftForks[idx].Cycles[:len(SoftForks[idx].Cycles)-1] + if SoftForks[idx].LatestCycle().BlocksSignalling >= explorer.GetQuorum(i.blocksInCycle, i.quorum) { + log.WithField("softfork", SoftForks[idx].Name).Infof("Softfork locked in with %d signals", SoftForks[idx].LatestCycle().BlocksSignalling) + SoftForks[idx].State = explorer.SoftForkLockedIn + log.Info("Set State to LockedIn") + + SoftForks[idx].LockedInHeight = uint64(i.blocksInCycle * GetSoftForkBlockCycle(i.blocksInCycle, block.Height).Cycle) + log.Info("Set LockedInHeight to ", SoftForks[idx].LockedInHeight) + + SoftForks[idx].ActivationHeight = SoftForks[idx].LockedInHeight + uint64(i.blocksInCycle) + log.Info("Set ActivationHeight to ", SoftForks[idx].ActivationHeight) + + i.elastic.AddUpdateRequest(elastic_cache.SoftForkIndex.Get(), SoftForks[idx]) } - i.elastic.AddUpdateRequest(elastic_cache.SoftForkIndex.Get(), SoftForks[idx]) } if SoftForks[idx].State == explorer.SoftForkLockedIn && block.Height >= SoftForks[idx].ActivationHeight-1 { - log.Info("Moved softfork to Active") + log.WithField("softfork", SoftForks[idx].Name).Info("SoftFork Activated at height ", block.Height) SoftForks[idx].State = explorer.SoftForkActive i.elastic.AddUpdateRequest(elastic_cache.SoftForkIndex.Get(), SoftForks[idx]) } diff --git a/internal/service/softfork/rewinder.go b/internal/service/softfork/rewinder.go index 04de06e..27a44be 100644 --- a/internal/service/softfork/rewinder.go +++ b/internal/service/softfork/rewinder.go @@ -33,6 +33,8 @@ func (r *Rewinder) Rewind(height uint64) error { Name: s.Name, SignalBit: s.SignalBit, State: explorer.SoftForkDefined, + StartTime: s.StartTime, + Timeout: s.Timeout, } } @@ -48,6 +50,7 @@ func (r *Rewinder) Rewind(height uint64) error { } signals := r.signalRepo.GetSignals(start, end) + for _, s := range signals { for _, sf := range s.SoftForks { softFork := SoftForks.GetSoftFork(sf) diff --git a/internal/service/softfork/service.go b/internal/service/softfork/service.go index dbf05df..1a047df 100644 --- a/internal/service/softfork/service.go +++ b/internal/service/softfork/service.go @@ -6,6 +6,7 @@ import ( "github.com/NavExplorer/navexplorer-indexer-go/internal/elastic_cache" "github.com/NavExplorer/navexplorer-indexer-go/pkg/explorer" log "github.com/sirupsen/logrus" + "time" ) var SoftForks explorer.SoftForks @@ -28,14 +29,17 @@ func (i *Service) InitSoftForks() { log.WithError(err).Fatal("Failed to get blockchaininfo") } - //SoftForks, err = i.repo.GetSoftForks() - //if err != nil { - // log.WithError(err).Fatal("Failed to get softforks from repo") - //} - for name, bip9fork := range info.Bip9SoftForks { - //if SoftForks.GetSoftFork(name) == nil { - softFork := &explorer.SoftFork{Name: name, SignalBit: bip9fork.Bit, State: explorer.SoftForkDefined, ActivationHeight: 0, LockedInHeight: 0} + softFork := &explorer.SoftFork{ + Name: name, + SignalBit: bip9fork.Bit, + State: explorer.SoftForkDefined, + StartTime: time.Unix(int64(bip9fork.StartTime), 0), + Timeout: time.Unix(int64(bip9fork.Timeout), 0), + ActivationHeight: 0, + LockedInHeight: 0, + } + _, err := i.elastic.Client. Index(). Index(elastic_cache.SoftForkIndex.Get()). @@ -47,7 +51,6 @@ func (i *Service) InitSoftForks() { } SoftForks = append(SoftForks, softFork) - //} } } diff --git a/internal/validator/addressValidator.go b/internal/validator/addressValidator.go deleted file mode 100644 index 03963b8..0000000 --- a/internal/validator/addressValidator.go +++ /dev/null @@ -1,54 +0,0 @@ -package validator - -import ( - "context" - _ "github.com/NavExplorer/navcoind-go" - "github.com/NavExplorer/navexplorer-indexer-go/generated/dic" - "github.com/NavExplorer/navexplorer-indexer-go/internal/config" - "github.com/NavExplorer/navexplorer-indexer-go/internal/elastic_cache" - "github.com/NavExplorer/navexplorer-indexer-go/internal/service/address" - "github.com/NavExplorer/navexplorer-indexer-go/pkg/explorer" - "github.com/sarulabs/dingo/v3" - log "github.com/sirupsen/logrus" - "time" -) - -var container *dic.Container - -type AddressValidator struct{} - -func (v *AddressValidator) Execute() { - log.Info("NavExplorer Address Validator") - config.Init() - container, _ = dic.NewContainer(dingo.App) - - a := &explorer.Address{Hash: "NaLKtxjrxQYa3e2J1kprqoHG4SYMj9Xnw1"} - log.Infof("Validating address: %s", a.Hash) - - address.ResetAddress(a) - if txs, err := container.GetBlockRepo().GetAllTransactionsThatIncludeAddress(a.Hash); err == nil { - log.Infof("Loaded block transactions") - for _, tx := range txs { - block, err := container.GetBlockRepo().GetBlockByHeight(tx.Height) - if err != nil { - log.WithError(err).Fatalf("Failed to get block at height %d", tx.Height) - } - addressTxs := container.GetAddressIndexer().GenerateAddressTransactions(a, tx, block) - for _, addressTx := range addressTxs { - address.ApplyTxToAddress(a, addressTx) - _, err = container.GetElastic().Client.Index(). - Index(elastic_cache.AddressTransactionIndex.Get()). - BodyJson(addressTx). - Id(addressTx.Slug()). - Do(context.Background()) - } - } - - a.ValidatedAt = uint64(time.Now().Unix()) - _, err = container.GetElastic().Client.Index(). - Index(elastic_cache.AddressIndex.Get()). - BodyJson(a). - Id(a.Slug()). - Do(context.Background()) - } -} diff --git a/internal/validator/cfundValidator.go b/internal/validator/cfundValidator.go deleted file mode 100644 index 10a4d69..0000000 --- a/internal/validator/cfundValidator.go +++ /dev/null @@ -1,16 +0,0 @@ -package validator - -import ( - "github.com/NavExplorer/navexplorer-indexer-go/generated/dic" - "github.com/NavExplorer/navexplorer-indexer-go/internal/config" - "github.com/sarulabs/dingo/v3" -) - -type CfundValidator struct{} - -func (v *CfundValidator) Execute() { - config.Init() - container, _ = dic.NewContainer(dingo.App) - - container.GetBlockRepo() -} diff --git a/pkg/explorer/address.go b/pkg/explorer/address.go index 3029ff9..6246e54 100644 --- a/pkg/explorer/address.go +++ b/pkg/explorer/address.go @@ -3,31 +3,28 @@ package explorer import ( "fmt" "github.com/gosimple/slug" + "time" ) type Address struct { Hash string `json:"hash"` Height uint64 `json:"height"` - Received int64 `json:"received"` - ReceivedCount uint `json:"receivedCount"` - Sent int64 `json:"sent"` - SentCount uint `json:"sentCount"` - Staked int64 `json:"staked"` - StakedCount uint `json:"stakedCount"` - Balance int64 `json:"balance"` + Spendable int64 `json:"spendable"` + Stakable int64 `json:"stakable"` + VotingWeight int64 `json:"voting_weight"` - ColdReceived int64 `json:"coldReceived"` - ColdReceivedCount uint `json:"coldReceivedCount"` - ColdSent int64 `json:"coldSent"` - ColdSentCount uint `json:"coldSentCount"` - ColdStaked int64 `json:"coldStaked"` - ColdStakedCount uint `json:"coldStakedCount"` - ColdBalance int64 `json:"coldBalance"` + CreatedTime time.Time `json:"created_time"` + CreatedBlock uint64 `json:"created_block"` - ValidatedAt uint64 `json:"validatedAt"` + // Transient + RichList RichList `json:"rich_list,omitempty"` +} - Position uint `json:"position"` +type RichList struct { + Spending uint64 `json:"spending"` + Staking uint64 `json:"staking"` + Voting uint64 `json:"voting"` } func (a *Address) Slug() string { diff --git a/pkg/explorer/address_history.go b/pkg/explorer/address_history.go new file mode 100644 index 0000000..1329768 --- /dev/null +++ b/pkg/explorer/address_history.go @@ -0,0 +1,56 @@ +package explorer + +import ( + "fmt" + "github.com/gosimple/slug" + "time" +) + +type AddressHistory struct { + Height uint64 `json:"height"` + TxIndex uint `json:"txindex"` + Time time.Time `json:"time"` + TxId string `json:"txid"` + Hash string `json:"hash"` + Changes AddressChanges `json:"changes"` + Balance AddressBalance `json:"balance"` + + Stake bool `json:"is_stake"` + CfundPayout bool `json:"is_cfund_payout"` + StakePayout bool `json:"is_stake_payout"` +} + +type AddressChanges struct { + Spendable int64 `json:"spendable"` + Stakable int64 `json:"stakable"` + VotingWeight int64 `json:"voting_weight"` + Proposal bool `json:"proposal,omitempty"` + PaymentRequest bool `json:"payment_request,omitempty"` + Consultation bool `json:"consultation,omitempty"` +} + +type AddressBalance struct { + Spendable int64 `json:"spendable"` + Stakable int64 `json:"stakable"` + VotingWeight int64 `json:"voting_weight"` +} + +type BalanceType string + +var ( + Spendable BalanceType = "spendable" + Stakable BalanceType = "stakable" + VotingWeight BalanceType = "voting_weight" +) + +func (a *AddressHistory) Slug() string { + return slug.Make(fmt.Sprintf("addresshistory-%s-%s", a.Hash, a.TxId)) +} + +func (a *AddressHistory) IsSpend() bool { + return a.Changes.Spendable < 0 +} + +func (a *AddressHistory) IsReceive() bool { + return a.Changes.Spendable > 0 +} diff --git a/pkg/explorer/block.go b/pkg/explorer/block.go index 46040a1..161e478 100644 --- a/pkg/explorer/block.go +++ b/pkg/explorer/block.go @@ -61,7 +61,7 @@ func (b *BlockCycle) IsEnd() bool { } func GetQuorum(size uint, quorum int) int { - return int((float64(quorum) / 100) * float64(size)) + return int((float64(quorum) / 100.0) * float64(size)) } type Cfund struct { diff --git a/pkg/explorer/soft_fork.go b/pkg/explorer/soft_fork.go index 6df7288..e2f1066 100644 --- a/pkg/explorer/soft_fork.go +++ b/pkg/explorer/soft_fork.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/gosimple/slug" "log" + "time" ) type SoftForks []*SoftFork @@ -11,6 +12,8 @@ type SoftForks []*SoftFork type SoftFork struct { Name string `json:"name"` SignalBit uint `json:"signalBit"` + StartTime time.Time `json:"startTime"` + Timeout time.Time `json:"timeout"` State SoftForkState `json:"state"` LockedInHeight uint64 `json:"lockedinheight,omitempty"` ActivationHeight uint64 `json:"activationheight,omitempty"` diff --git a/pkg/explorer/vout.go b/pkg/explorer/vout.go index d07947c..7400950 100644 --- a/pkg/explorer/vout.go +++ b/pkg/explorer/vout.go @@ -16,6 +16,16 @@ type RedeemedIn struct { Height uint64 `json:"height,omitempty"` } +func (o *Vout) HasAddress(hash string) bool { + for _, a := range o.ScriptPubKey.Addresses { + if a == hash { + return true + } + } + + return false +} + func (o *Vout) IsColdStaking() bool { return o.ScriptPubKey.Type == VoutColdStaking || o.ScriptPubKey.Type == VoutColdStakingV2 } diff --git a/pkg/explorer/vouts.go b/pkg/explorer/vouts.go index 657808c..73f6bf3 100644 --- a/pkg/explorer/vouts.go +++ b/pkg/explorer/vouts.go @@ -1,8 +1,30 @@ package explorer +import "errors" + type RawVouts []RawVout type Vouts []Vout +func (vouts *Vouts) Count() int { + count := 0 + for range *vouts { + count++ + } + return count +} + +func (vouts *Vouts) WithAddress(hash string) []Vout { + filtered := make([]Vout, 0) + + for _, v := range *vouts { + if v.HasAddress(hash) { + filtered = append(filtered, v) + } + } + + return filtered +} + func (vouts *Vouts) HasOutputOfType(txType VoutType) bool { for _, vout := range *vouts { if vout.ScriptPubKey.Type == txType { @@ -25,6 +47,25 @@ func (vouts *Vouts) HasAddress(address string) bool { return false } +func (vouts *Vouts) GetVotingAddress() (string, error) { + for _, vout := range *vouts { + if vout.ScriptPubKey.Type == VoutNonstandard { + continue + } + if vout.ScriptPubKey.Type == VoutColdStaking && len(vout.ScriptPubKey.Addresses) == 2 { + return vout.ScriptPubKey.Addresses[0], nil + } + if vout.ScriptPubKey.Type == VoutColdStakingV2 && len(vout.ScriptPubKey.Addresses) == 3 { + return vout.ScriptPubKey.Addresses[2], nil + } + if vout.ScriptPubKey.Type == VoutPubkey && len(vout.ScriptPubKey.Addresses) == 1 { + return vout.ScriptPubKey.Addresses[0], nil + } + } + + return "", errors.New("Unable to retrieve Voting Address") +} + func (vouts *Vouts) GetSpendableAmount() uint64 { var amount uint64 = 0 for _, o := range *vouts {