From 47b34dd0eeb926de1005da22b0e16e6d7fedecab Mon Sep 17 00:00:00 2001 From: diamondhands Date: Tue, 12 Apr 2022 21:18:27 -0700 Subject: [PATCH 1/2] WIP secondary indexes --- cmd/run.go | 26 +++++++++++++++++++++++--- config/config.go | 10 ++++++++++ go.mod | 6 +++++- go.sum | 27 +++++++++++++++++++++++++++ 4 files changed, 65 insertions(+), 4 deletions(-) diff --git a/cmd/run.go b/cmd/run.go index 7b951690..d72e25a2 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -1,6 +1,8 @@ package cmd import ( + "github.com/deso-protocol/backend/secondary_indexes" + "github.com/deso-protocol/core/lib" "os" "os/signal" "syscall" @@ -23,13 +25,25 @@ var runCmd = &cobra.Command{ } func Run(cmd *cobra.Command, args []string) { - // Start the core node + // Load up the configs for the core node, and this node. coreConfig := coreCmd.LoadConfig() + nodeConfig := config.LoadConfig(coreConfig) + + // Start the core node with a hook to compute secondary indexes. coreNode := coreCmd.NewNode(coreConfig) - coreNode.Start() + // Create an EventManager and run a hook every time we process a block. + // We use this hook to build up secondary indexes like Postgres once the + // node becomes fully synced. + eventManager := lib.NewEventManager() + secondaryIndex := secondary_indexes.NewSecondaryIndex( + coreNode) + eventManager.OnBlockAccepted(secondaryIndex.BuildSecondaryIndexesAfterSyncCompleted) + // Start the core node. + coreNode.StartWithOptions(eventManager) + + // Wire up the block processor // Start the backend node - nodeConfig := config.LoadConfig(coreConfig) node := NewNode(nodeConfig, coreNode) node.Start() @@ -163,6 +177,12 @@ func init() { // Tag transaction with node source runCmd.PersistentFlags().Uint64("node-source", 0, "Node ID to tag transaction with. Maps to ../core/lib/nodes.go") + runCmd.PersistentFlags().Bool("force-recompute-secondary-indexes-on-startup", false, + "Useful for testing. When set to true, the secondary indexes will "+ + "be recomputed on startup, making it easy to find bugs.") + runCmd.PersistentFlags().String("secondary-index-postgres-uri", "", + "When set, this Postgres DB is used to compute secondary indexes.") + runCmd.PersistentFlags().VisitAll(func(flag *pflag.Flag) { viper.BindPFlag(flag.Name, flag) }) diff --git a/config/config.go b/config/config.go index c4f73f84..d0391991 100644 --- a/config/config.go +++ b/config/config.go @@ -79,6 +79,12 @@ type Config struct { // ID to tag node source NodeSource uint64 + + // Useful for testing. When set to true, the secondary indexes will be + // recomputed on startup, making it easy to find bugs. + ForceRecomputeSecondaryIndexesOnStartup bool + // When set, this Postgres DB is used to compute secondary indexes. + SecondaryIndexPostgresURI string } func LoadConfig(coreConfig *coreCmd.Config) *Config { @@ -175,5 +181,9 @@ func LoadConfig(coreConfig *coreCmd.Config) *Config { // Node source ID config.NodeSource = viper.GetUint64("node-source") + // Params for secondary indexes + config.ForceRecomputeSecondaryIndexesOnStartup = viper.GetBool("force-recompute-secondary-indexes-on-startup") + config.SecondaryIndexPostgresURI = viper.GetString("secondary-index-postgres-uri") + return &config } diff --git a/go.mod b/go.mod index f0aa2765..d7233760 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,11 @@ require ( github.com/ttacon/builder v0.0.0-20170518171403-c099f663e1c2 // indirect github.com/ttacon/libphonenumber v1.2.1 // indirect github.com/tyler-smith/go-bip39 v1.1.0 - golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a + github.com/uptrace/bun v1.1.3 // indirect + github.com/uptrace/bun/dialect/pgdialect v1.1.3 // indirect + github.com/uptrace/bun/driver/pgdriver v1.1.3 // indirect + github.com/uptrace/bun/extra/bundebug v1.1.3 // indirect + golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064 google.golang.org/api v0.46.0 gopkg.in/DataDog/dd-trace-go.v1 v1.29.0 ) diff --git a/go.sum b/go.sum index 6b2a1885..5279ce8a 100644 --- a/go.sum +++ b/go.sum @@ -171,6 +171,8 @@ github.com/ethereum/go-ethereum v1.9.25/go.mod h1:vMkFiYLHI4tgPw4k2j4MHKoovchFE8 github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= @@ -391,12 +393,18 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/goveralls v0.0.6 h1:cr8Y0VMo/MnEZBjxNN/vh6G90SZ7IMb6lms1dzMoO+Y= @@ -561,6 +569,14 @@ github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3C github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/unrolled/secure v1.0.8 h1:JaMvKbe4CRt8oyxVXn+xY+6jlqd7pyJNSVkmsBxxQsM= github.com/unrolled/secure v1.0.8/go.mod h1:fO+mEan+FLB0CdEnHf6Q4ZZVNqG+5fuLFnP8p0BXDPI= +github.com/uptrace/bun v1.1.3 h1:v62tsUyKjVCR5q7J49uckM6CVVTqMO26aV73F3G6RFk= +github.com/uptrace/bun v1.1.3/go.mod h1:aQbKvxs7/n9MMef/b8lYOh5Rwlo4Jd5A31E4HlYNqSc= +github.com/uptrace/bun/dialect/pgdialect v1.1.3 h1:EMRCC98YKSpo/EXyujsr+5v0PKYkRE0rwxJKKEcrOuE= +github.com/uptrace/bun/dialect/pgdialect v1.1.3/go.mod h1:2GJogfkVHmCKxt6N88vRbJNSUV5wfPym/rp6N25dShc= +github.com/uptrace/bun/driver/pgdriver v1.1.3 h1:WWxEfGnJQCXgODtjU37E+XWEVvCGwvs2fRgCYFqmKAY= +github.com/uptrace/bun/driver/pgdriver v1.1.3/go.mod h1:D7tTNXLIR9udcf/Dm9W+x1qvY+GDCkYVIRLgQyMElCY= +github.com/uptrace/bun/extra/bundebug v1.1.3 h1:c/YKsH3l377tmIKpPWRn+kjtCTofmaW7ez+yT4coAaw= +github.com/uptrace/bun/extra/bundebug v1.1.3/go.mod h1:TBpazrrYLBGsUw/LzHaIZLcxxYXIpH4GOqD9c+3dmGI= github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/negroni v1.0.0 h1:kIimOitoypq34K7TG7DUaJ9kq/N4Ofuwi1sjz0KipXc= @@ -572,6 +588,8 @@ github.com/vmihailenco/msgpack/v5 v5.0.0-beta.1/go.mod h1:xlngVLeyQ/Qi05oQxhQ+oT github.com/vmihailenco/msgpack/v5 v5.0.0-beta.8/go.mod h1:HVxBVPUK/+fZMonk4bi1islLa8V3cfnBug0+4dykPzo= github.com/vmihailenco/msgpack/v5 v5.3.1 h1:0i85a4dsZh8mC//wmyyTEzidDLPQfQAxZIOLtafGbFY= github.com/vmihailenco/msgpack/v5 v5.3.1/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= +github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= @@ -617,6 +635,8 @@ golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a h1:kr2P4QFmQr29mSLA43kwrOcgcReGTfbE9N577tCTuBc= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064 h1:S25/rfnfsMVgORT4/J61MJ7rdyseOZOyvLIrZEZ7s6s= +golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -705,6 +725,7 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -759,6 +780,7 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -792,8 +814,13 @@ golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210503080704-8803ae5d1324/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886 h1:eJv7u3ksNXoLbGSKuv2s/SIO4tJVxc/A+MTpzxDgz/Q= +golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From 75ff84d67bb21be0a25452389c198d420788c5c7 Mon Sep 17 00:00:00 2001 From: diamondhands Date: Tue, 12 Apr 2022 21:48:11 -0700 Subject: [PATCH 2/2] WIP - forgot directory --- secondary_indexes/build_indexes.go | 129 ++++++++++ .../20220130000000_create_initial_tables.go | 233 ++++++++++++++++++ secondary_indexes/migrations/README | 6 + secondary_indexes/migrations/init.go | 11 + secondary_indexes/postgres_index.go | 1 + secondary_indexes/topic_index.go | 1 + secondary_indexes/txindex_index.go | 1 + 7 files changed, 382 insertions(+) create mode 100644 secondary_indexes/build_indexes.go create mode 100644 secondary_indexes/migrations/20220130000000_create_initial_tables.go create mode 100644 secondary_indexes/migrations/README create mode 100644 secondary_indexes/migrations/init.go create mode 100644 secondary_indexes/postgres_index.go create mode 100644 secondary_indexes/topic_index.go create mode 100644 secondary_indexes/txindex_index.go diff --git a/secondary_indexes/build_indexes.go b/secondary_indexes/build_indexes.go new file mode 100644 index 00000000..efc9ad12 --- /dev/null +++ b/secondary_indexes/build_indexes.go @@ -0,0 +1,129 @@ +package secondary_indexes + +import ( + "context" + "github.com/deso-protocol/backend/secondary_indexes/migrations" + coreCmd "github.com/deso-protocol/core/cmd" + "github.com/deso-protocol/core/lib" + "database/sql" + "github.com/dgraph-io/badger/v3" + "github.com/golang/glog" + "github.com/spf13/viper" + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect/pgdialect" + "github.com/uptrace/bun/driver/pgdriver" + "github.com/uptrace/bun/extra/bundebug" + "github.com/uptrace/bun/migrate" +) + +type SecondaryIndex struct { + Node *coreCmd.Node + + ForceRecomputeOnStartup bool + PostgresDB *bun.DB +} + +func NewSecondaryIndex( + coreNode *coreCmd.Node, pgURI string, forceRecomputeOnStartup bool) *SecondaryIndex { + + // FIXME: Provide an easy path for testing locally. See how daodao does it. + //if pgURI == "" { + // pgURI = "postgresql://postgres:postgres@localhost:5432/postgres?sslmode=disable" + //} + + // Open a PostgreSQL database. + pgdb := sql.OpenDB(pgdriver.NewConnector(pgdriver.WithDSN(pgURI))) + if pgdb == nil { + glog.Fatalf("Error connecting to postgres db at URI: %v", pgURI) + } + + // Create a Bun db on top of postgres for querying. + db := bun.NewDB(pgdb, pgdialect.New()) + + // Print all queries to stdout for debugging. + db.AddQueryHook(bundebug.NewQueryHook(bundebug.WithVerbose(true))) + + // Apply db migrations + ctx := context.Background() + glog.Info("Applying migrations...") + migrator := migrate.NewMigrator(db, migrations.Migrations) + if err := migrator.Init(ctx); err != nil { + glog.Fatal(err) + } + group, err := migrator.Migrate(ctx) + if err != nil { + glog.Fatal(err) + } + glog.Infof("Migrated to %s\n", group) + + return &SecondaryIndex{ + Node: coreNode, + PostgresDB: db, + ForceRecomputeOnStartup: forceRecomputeOnStartup, + } +} + + +func (secondaryindex *SecondaryIndex) BuildSecondaryIndexesAfterSyncCompleted( + event *lib.BlockEvent) { + + // FIXME: Store a variable in the DB to indicate whether or not we have built the + // "initial" indexes yet. Set this variable the first time we have built these initial + // indexes. + hasBuiltInitialIndexes := false + coreBlockchain := secondaryindex.Node.Server.GetBlockchain() + isFullySynced := coreBlockchain.ChainState() != lib.SyncStateFullyCurrent + if !hasBuiltInitialIndexes && isFullySynced { + // FIXME: To support DAODAO, do the following here: + // - Iterate through all posts in the node's "consensus" db. The inefficient function below + // is fine, but you have to fill it in: + // * DBGetAllPostsInefficient(coreBlockchain.DB()) + // - For each post, store it in Postgres with an index on the following fields PLUS + // indexed on timestamp AND amount of DAO coin held AND follower/following AND CommentCount, + // which comes standard on the PostEntry (will tell you how to look up DAO coin held in a second): + // * Parse the tags in the post and see if any of them correspond to a DAO. + // * PostApp (will be DAODAO for DAODAO) + // - To compute the amount of DAO coin held, use the function below and use the profile of + // the username in the PostGroup as the creatorPKID to look it up: + // * DBGetBalanceEntryForHODLerAndCreatorPKIDsWithTxn + + // You're done once you've indexed all of the above. + return + } + // If we get here, it means we already did the initial index build, and just need to + // process this one new block. This is pretty chill. + + // FIXME: Implement the following: + // - For each txn in the block that is a SubmitPost or a like or a Diamond, update + // the post in our DB. You don't need to do anything fancy here, just set the post + // in postgres to be whatever consensus has computed. + + // If you've done all of the above, you should be able to support queries of the following form, + // which is the whole point: + // - DAODAO subreddit default sort: + // * Give me all posts made in the last day/week/month filtered to a particular PostGroup and + // sorted by {CommentCount, DAO coin holdings of the poster} + // + + // Check a variable in the DB to determine if we need to do an "initial" build + // of secondary indexes or not. If yes, then build. Otherwise, don't. If we + // have ForceRecomputeOnStartup, then dump the entire database and build the + // indexes from scratch once sync is complete. + // Check to see if we're supposed to actually build secondary indexes. + // Check to see if we've run this function before. + + + // If you do the above, you'll be able to service the following queries, which is the goal + // of all this: + // - Give me all the posts made in the last 24 hours sorted by + // - Give me all the posts for a particular PostGroup ordered by WHEN they were posted. This + // will support the NEW sort on a DAODAO subreddit. + // - Give me all the posts for a particular PostGroup ordered by largest DAO coin holders first. + // We will need to filter this a bit to show unique posts, ut + // - Give me all the posts for a particular (user, app) ordered by WHEN they were posted + + isCurrent := event.Server.blockchain.ChainState() == SyncStateFullyCurrent + if event. { + + } +} diff --git a/secondary_indexes/migrations/20220130000000_create_initial_tables.go b/secondary_indexes/migrations/20220130000000_create_initial_tables.go new file mode 100644 index 00000000..c3ff13bc --- /dev/null +++ b/secondary_indexes/migrations/20220130000000_create_initial_tables.go @@ -0,0 +1,233 @@ +package migrations + +import ( + "context" + + "github.com/uptrace/bun" +) + +// FIXME: Rewrite all these migrations and rename this file + +func init() { + Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error { + // Install the uuid extension + _, err := db.Exec(` + CREATE EXTENSION IF NOT EXISTS "uuid-ossp" + `) + if err != nil { + return err + } + + // Each user is identified by their public key + // + // phone_number: Starts with +1 for US and represents the international number + _, err = db.Exec(` + CREATE TABLE app_user ( + pkid_base58check TEXT NOT NULL PRIMARY KEY, + username TEXT NOT NULL, + derived_pubkey_base58check TEXT NOT NULL, + derived_privkey_base58check TEXT NOT NULL, + + email TEXT NOT NULL, + email_verification_code TEXT NOT NULL, + email_is_verified BOOLEAN NOT NULL, + + phone_number TEXT NOT NULL, + phone_number_verification_code TEXT NOT NULL, + phone_number_is_verified BOOLEAN NOT NULL + ); + `) + if err != nil { + return err + } + + // A round has the following fields: + // - round_id: A unique identifier + // - round_name: A human-readable name set by the DAO owner + // - round_status: {PAUSED, OPEN, FINALIZED} + // - amount_to_raise_usd_cents: The amount in USD that the DAO owner wants to raise + // - allow_overflow: Whether or not investments are allowed after the target has been reached + // - off_chain_deso_balance_nanos: The total amount of DESO that has gone into this + // round. We store this DESO until the round is finalized, at which point it is + // published. + // - reserve_rate_basis_points: A percentage of each investment that goes to the founder + // - global_referral_rate_basis_points: The percentage of each investment that a referrer + // gets + // - start_time/end_time: The period during which investments are allowed. The round is paused + // at the end_time, and can be extended before being finalized if desired. + // - dao_coins_per_deso_nanos_hex: A uint256 expressing how many DAO coins are issued per + // DESO invested. Default is 1M DAO coins per DESO. + // - access_password: If set, only people with the password can invest. + // - is_investment_immediately_refundable: When set to true, investors can immediately + // get a refund of their investment until the round is finalized. Turning this off + // means investors can't get a refund while the round is OPEN, but they can get a refund + // afterward if the DAO owner specifies that he wants to issue refunds rather than accept + // the DAO funds. + _, err = db.Exec(` + CREATE TYPE ROUND_STATUS AS ENUM ('OPEN', 'PAUSED', 'FINALIZED'); + CREATE TABLE funding_round ( + round_id UUID NOT NULL PRIMARY KEY DEFAULT uuid_generate_v4 (), + round_name TEXT NOT NULL, + round_status ROUND_STATUS NOT NULL, + dao_owner_pkid_base58check TEXT NOT NULL, + amount_to_raise_usd_cents BIGINT NOT NULL, + allow_overflow BOOLEAN NOT NULL, + off_chain_deso_balance_nanos BIGINT NOT NULL, + reserve_rate_basis_points BIGINT NOT NULL, + global_referral_rate_basis_points BIGINT NOT NULL, + start_time TIMESTAMP, + end_time TIMESTAMP, + dao_coins_per_deso_nanos_hex TEXT NOT NULL, + access_password TEXT NOT NULL, + is_investment_immediately_refundable BOOLEAN NOT NULL, + created_at TIMESTAMP DEFAULT NOW(), + CONSTRAINT fk_funding_round_dao_owner_pkid + FOREIGN KEY(dao_owner_pkid_base58check) + REFERENCES app_user(pkid_base58check) + ) + `) + if err != nil { + return err + } + + // This represents an investment in a round. An investment has a few possible + // states: + // - status: {PENDING, FINALIZED} + // + // Users are issued DAO coins on-chain immediately after sending DESO. While the + // investment is pending, investors can redeem their DAO coins for a refund of the + // DESO they put in at any time. They can do this until the round is finalized. Note + // that investments are immediately finalized if a DAO owner sets their round to + // non-refundable. + // + // These are the rest of the fields: + // - investment_id: A unique ID for this object + // - round_id: The round this investment is associated with + // - investor_pkid: The pkid of the investor + // - amount_invested_deso_nanos: The amount of DESO the investor put in for this + // investment. This is the amount the investor would be refunded. + // - amount_refunded_deso_nanos: The amount of DESO the investor has been refunded. + // Useful for tracking a partial refund. + // - dao_coins_issued_hex: The number of DAO coins originally issued for this + // investment. + // - dao_coins_returned_hex: The number of DAO coins the investor has redeemed back + // for DESO. This is how the investor gets a refund: They must redeem their DAO + // coins. They can only do this until the investment is finalized. + // - referrer_pkid: The pkid of the person who referred this investment + // - referrer_basis_points: The percentage of the DESO that the referrer will get + // when the round is finalized. + // - reserve_rate_basis_points: The percentage of the DESO that the DAO owner will + // get when the round is finalized. + _, err = db.Exec(` + CREATE TYPE INVESTMENT_STATUS AS ENUM ('PENDING', 'REFUNDED', 'FINALIZED'); + CREATE TABLE investment ( + investment_id UUID NOT NULL PRIMARY KEY DEFAULT uuid_generate_v4 (), + round_id UUID NOT NULL, + investor_pkid_base58check TEXT NOT NULL, + amount_invested_deso_nanos BIGINT NOT NULL, + amount_refunded_deso_nanos BIGINT NOT NULL, + dao_coins_issued_hex TEXT NOT NULL, + dao_coins_redeemed_hex TEXT NOT NULL, + status INVESTMENT_STATUS NOT NULL, + referrer_pkid_base58check TEXT NOT NULL, + referrer_basis_points BIGINT NOT NULL, + + reserve_rate_basis_points BIGINT NOT NULL, + + deleted_at TIMESTAMPTZ + created_at TIMESTAMP DEFAULT NOW(), + CONSTRAINT fk_investment_investor_pkid FOREIGN KEY(investor_pkid_base58check) REFERENCES app_user(pkid_base58check), + CONSTRAINT fk_investment_round_id FOREIGN KEY(round_id) REFERENCES funding_round(round_id) + + ) + `) + if err != nil { + return err + } + + _, err = db.Exec(` + CREATE TABLE referral_info ( + round_id TEXT NOT NULL, + referrer_pkid_base58check TEXT NOT NULL, + referral_basis_points_nanos BIGINT NOT NULL, + PRIMARY KEY(round_id, referrer_pkid_base58check) + ) + `) + if err != nil { + return err + } + + // Anyone can distribute DESO in bulk to anyone else on the platform. + // The most common use-case for this will be a DAO owner distributing + // DESO pro rata to DAO coin holders. In any case, a distribution can + // require burning one's DAO coins in exchange for DESO. + // + // - user_pkid: The pkid of the user who is entitled to this distribution + // - deso_owed_nanos: The amount of DESO the user is entitled to + // - dao_owner_pkid: In the event that the user must burn DAO coins for DESO, + // this specifies the pkid of the DAO. + // - dao_coins_required_hex: The number of DAO coins that are required to be burned + // in order to redeem the full deso_owed_nanos. Partial redemption is supported by + // redeeming a fraction of the dao_coins_required. + _, err = db.Exec(` + CREATE TABLE distribution ( + distribution_id TEXT NOT NULL PRIMARY KEY, + user_pkid_base58check TEXT NOT NULL, + deso_owed_nanos BIGINT NOT NULL, + + dao_owner_pkid_base58check TEXT NOT NULL, + dao_coins_required_hex TEXT NOT NULL + ) + `) + if err != nil { + return err + } + + // For every blockchain transaction that a user needs signed - a derived key will be generated. + // This table will track each derived key that a user generates, and the purpose for that key. + // This table has a many -> one relationship to the app_users table to identify which user the key belongs to. + // This table also has a many -> one relationship to the investment table when the derived key will be used to purchase the coins of a DAO. + // This table also has a many -> one relationship to the funding round table when the derived key will be used to transfer DAO coins from the owner to investors. + // + // - derived_key_id: Primary key for the table. + // - derived_public_key: Public key of the derived key pair. + // - derived_private_key: Encrypted private key of the derived key pair. + // - app_user_pkid_base_58_check: Foreign key to relate this table to the app_user table. Represents the user who the derived key belongs to. + // - investment_id: Foreign key to relate this table to the investments table. Represents the investment a user is making in a DAO, where relevant. + // - funding_round_id: Foreign key to relate this table to the funding round table. Represents the funding round a DAO is holding, where relevant. + // - purpose: What this derived key is being used for. i.e. `invest_in_dao`, `transfer_dao_coins`, `send_diamonds`, `update_profile` + // - status: Status of the derived key i.e. `created`, `authorized`, `expired`, `deleted` + + _, err = db.Exec(` + CREATE TABLE derived_key ( + derived_key_id UUID NOT NULL PRIMARY KEY DEFAULT uuid_generate_v4 (), + + derived_public_key TEXT NOT NULL, + derived_private_key TEXT NOT NULL, + + app_user_pkid_base58_check TEXT NOT NULL, + investment_id TEXT, + funding_round_id TEXT, + + purpose TEXT NOT NULL, + status TEXT NOT NULL, + deleted_at TIMESTAMPTZ + ) + `) + if err != nil { + return err + } + + return nil + }, func(ctx context.Context, db *bun.DB) error { + _, err := db.Exec(` + DROP TABLE distribution; + DROP TABLE referral_info; + DROP TABLE investment; + DROP TABLE funding_round; + DROP TABLE app_user; + DROP TABLE derived_key; + `) + return err + }) +} diff --git a/secondary_indexes/migrations/README b/secondary_indexes/migrations/README new file mode 100644 index 00000000..a676079e --- /dev/null +++ b/secondary_indexes/migrations/README @@ -0,0 +1,6 @@ +To add a migration, simply do the following: +* Choose a new filename (timestamp and suffix) +* Copy an existing migration over to a new file with that name +* Modify the Exec commands to suit your needs +* The other modules will pick up the migration as long as it follows the same + filename format. It does this by importing the "Migrations" var defined in main.go diff --git a/secondary_indexes/migrations/init.go b/secondary_indexes/migrations/init.go new file mode 100644 index 00000000..781c88d5 --- /dev/null +++ b/secondary_indexes/migrations/init.go @@ -0,0 +1,11 @@ +package migrations + +import "github.com/uptrace/bun/migrate" + +var Migrations = migrate.NewMigrations() + +func init() { + if err := Migrations.DiscoverCaller(); err != nil { + panic(err) + } +} diff --git a/secondary_indexes/postgres_index.go b/secondary_indexes/postgres_index.go new file mode 100644 index 00000000..ff3e4a98 --- /dev/null +++ b/secondary_indexes/postgres_index.go @@ -0,0 +1 @@ +package secondary_indexes diff --git a/secondary_indexes/topic_index.go b/secondary_indexes/topic_index.go new file mode 100644 index 00000000..ff3e4a98 --- /dev/null +++ b/secondary_indexes/topic_index.go @@ -0,0 +1 @@ +package secondary_indexes diff --git a/secondary_indexes/txindex_index.go b/secondary_indexes/txindex_index.go new file mode 100644 index 00000000..ff3e4a98 --- /dev/null +++ b/secondary_indexes/txindex_index.go @@ -0,0 +1 @@ +package secondary_indexes