Migrate all RPC calls to use snapshot (#2719)

* Add snapshot support

* Generate snapshot on ConnectTip

* Get snapshot on demand

* Get changed map under lock

* Update snapshot based on height

* Revert "Update snapshot based on height"

This reverts commit adc22c6.

* Do not Discard or Flush snapshot

* Use snapshot in getblockcount

* Snapshot on each block

* Resolve issues post-merge

* lint: circular deps

* Only snapshot on each block near tip

* Use IBD

* Snapshot when current block near the current time

* lint: circular deps

* Add history snapshots

* lint: circular deps

* Add vault snapshots

* Add GetVaultSnapshot wrapper

* lint: circular deps

* Pass vaultDB

* Get all snapshots at once

* Migrate all RPC calls to use snapshot

* Remove coins DB usage. Add lock for burn DB

* Use account view

* Restore lock for vault history DB

* Use vault history snapshot

* Remove global usage from attributes

* Update to use GetSnapshots call

---------

Co-authored-by: Prasanna Loganathar <[email protected]>

src/dfi/govvariables/attributes.cpp

@@ -27,7 +27,9 @@ enum class EVMAttributesTypes : uint32_t {
     RbfIncrementMinPct = 4,
 };
 
-extern UniValue AmountsToJSON(const TAmounts &diffs, AmountFormat format = AmountFormat::Symbol);
+extern UniValue AmountsToJSON(const CCustomCSView &view,
+                              const TAmounts &diffs,
+                              AmountFormat format = AmountFormat::Symbol);
 
 static inline std::string trim_all_ws(std::string s) {
     s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](unsigned char ch) { return !std::isspace(ch); }));
@@ -1605,7 +1607,7 @@ static void ExportOracleSplit(UniValue &ret, const std::string &key, const bool
     ret.pushKV(key, keyValue);
 }
 
-UniValue ATTRIBUTES::ExportFiltered(GovVarsFilter filter, const std::string &prefix) const {
+UniValue ATTRIBUTES::ExportFiltered(GovVarsFilter filter, const std::string &prefix, CCustomCSView *view) const {
     UniValue ret(UniValue::VOBJ);
     for (const auto &attribute : attributes) {
         const auto attrV0 = std::get_if<CDataStructureV0>(&attribute.first);
@@ -1685,11 +1687,17 @@ UniValue ATTRIBUTES::ExportFiltered(GovVarsFilter filter, const std::string &pre
                     }
                 }
             } else if (const auto balances = std::get_if<CBalances>(&attribute.second)) {
-                ret.pushKV(key, AmountsToJSON(balances->balances));
+                // Related to the economy keys shown in listgovs/getgov
+                if (view) {
+                    ret.pushKV(key, AmountsToJSON(*view, balances->balances));
+                }
             } else if (const auto paybacks = std::get_if<CTokenPayback>(&attribute.second)) {
+                // Related to the economy keys shown in listgovs/getgov
                 UniValue result(UniValue::VOBJ);
-                result.pushKV("paybackfees", AmountsToJSON(paybacks->tokensFee.balances));
-                result.pushKV("paybacktokens", AmountsToJSON(paybacks->tokensPayback.balances));
+                if (view) {
+                    result.pushKV("paybackfees", AmountsToJSON(*view, paybacks->tokensFee.balances));
+                    result.pushKV("paybacktokens", AmountsToJSON(*view, paybacks->tokensPayback.balances));
+                }
                 ret.pushKV(key, result);
             } else if (const auto balances = std::get_if<CDexBalances>(&attribute.second)) {
                 for (const auto &pool : *balances) {

src/dfi/govvariables/attributes.h

@@ -419,7 +419,7 @@ class ATTRIBUTES : public GovVariable, public AutoRegistrator<GovVariable, ATTRI
     bool IsEmpty() const override;
     Res Import(const UniValue &val) override;
     UniValue Export() const override;
-    UniValue ExportFiltered(GovVarsFilter filter, const std::string &prefix) const;
+    UniValue ExportFiltered(GovVarsFilter filter, const std::string &prefix, CCustomCSView *view = nullptr) const;
     Res CheckKeys() const;
 
     Res Validate(const CCustomCSView &mnview) const override;

src/dfi/mn_checks.h

@@ -220,7 +220,7 @@ class TransactionContext {
 
 CCustomTxMessage customTypeToMessage(CustomTxType txType);
 bool IsMempooledCustomTxCreate(const CTxMemPool &pool, const uint256 &txid);
-Res RpcInfo(const CTransaction &tx, uint32_t height, CustomTxType &type, UniValue &results);
+Res RpcInfo(CCustomCSView &view, const CTransaction &tx, uint32_t height, CustomTxType &type, UniValue &results);
 Res CustomMetadataParse(uint32_t height,
                         const Consensus::Params &consensus,
                         const std::vector<unsigned char> &metadata,

src/dfi/mn_rpc.cpp

@@ -3,21 +3,21 @@
 // file LICENSE or http://www.opensource.org/licenses/mit-license.php.
 
 #include <base58.h>
+#include <dfi/accountshistory.h>
 #include <dfi/consensus/xvm.h>
 #include <dfi/govvariables/attributes.h>
 #include <dfi/mn_rpc.h>
+#include <dfi/vaulthistory.h>
 #include <policy/settings.h>
 #include <regex>
 
 extern bool EnsureWalletIsAvailable(bool avoidException);                // in rpcwallet.cpp
 extern bool DecodeHexTx(CTransaction &tx, const std::string &strHexTx);  // in core_io.h
 
-CAccounts GetAllMineAccounts(CWallet *const pwallet) {
+CAccounts GetAllMineAccounts(CWallet *const pwallet, CCustomCSView &mnview) {
     CAccounts walletAccounts;
 
-    LOCK(cs_main);
-    CCustomCSView mnview(*pcustomcsview);
-    auto targetHeight = ::ChainActive().Height() + 1;
+    int targetHeight = mnview.GetLastHeight() + 1;
 
     CalcMissingRewardTempFix(mnview, targetHeight, *pwallet);
 
@@ -291,9 +291,9 @@ static std::vector<CTxIn> GetInputs(const UniValue &inputs) {
     return vin;
 }
 
-std::optional<CScript> AmIFounder(CWallet *const pwallet) {
+std::optional<CScript> AmIFounder(CWallet *const pwallet, CCustomCSView &mnview) {
     auto members = Params().GetConsensus().foundationMembers;
-    const auto attributes = pcustomcsview->GetAttributes();
+    const auto attributes = mnview.GetAttributes();
     if (attributes->GetValue(CDataStructureV0{AttributeTypes::Param, ParamIDs::Feature, DFIPKeys::GovFoundation},
                              false)) {
         if (const auto databaseMembers = attributes->GetValue(
@@ -381,9 +381,9 @@ static CTransactionRef CreateAuthTx(CWalletCoinsUnlocker &pwallet,
     return fund(mtx, pwallet, {}, &coinControl, coinSelectOpts), sign(mtx, pwallet, {});
 }
 
-static std::optional<CTxIn> GetAnyFoundationAuthInput(CWalletCoinsUnlocker &pwallet) {
+static std::optional<CTxIn> GetAnyFoundationAuthInput(CWalletCoinsUnlocker &pwallet, CCustomCSView &mnview) {
     auto members = Params().GetConsensus().foundationMembers;
-    const auto attributes = pcustomcsview->GetAttributes();
+    const auto attributes = mnview.GetAttributes();
     if (attributes->GetValue(CDataStructureV0{AttributeTypes::Param, ParamIDs::Feature, DFIPKeys::GovFoundation},
                              false)) {
         if (const auto databaseMembers = attributes->GetValue(
@@ -412,6 +412,7 @@ std::vector<CTxIn> GetAuthInputsSmart(CWalletCoinsUnlocker &pwallet,
                                       bool needFounderAuth,
                                       CTransactionRef &optAuthTx,
                                       const UniValue &explicitInputs,
+                                      CCustomCSView &mnview,
                                       const CoinSelectionOptions &coinSelectOpts) {
     if (!explicitInputs.isNull() && !explicitInputs.empty()) {
         return GetInputs(explicitInputs);
@@ -438,10 +439,10 @@ std::vector<CTxIn> GetAuthInputsSmart(CWalletCoinsUnlocker &pwallet,
     }
     // Look for founder's auth. minttoken may already have an auth in result.
     if (needFounderAuth && result.empty()) {
-        auto anyFounder = AmIFounder(pwallet);
+        auto anyFounder = AmIFounder(pwallet, mnview);
         if (anyFounder) {
             auths.insert(anyFounder.value());
-            auto authInput = GetAnyFoundationAuthInput(pwallet);
+            auto authInput = GetAnyFoundationAuthInput(pwallet, mnview);
             if (authInput) {
                 result.push_back(authInput.value());
             } else {
@@ -510,10 +511,8 @@ CWalletCoinsUnlocker GetWallet(const JSONRPCRequest &request) {
     return CWalletCoinsUnlocker{std::move(wallet)};
 }
 
-std::optional<FutureSwapHeightInfo> GetFuturesBlock(const uint32_t typeId) {
-    LOCK(cs_main);
-
-    const auto attributes = pcustomcsview->GetAttributes();
+std::optional<FutureSwapHeightInfo> GetFuturesBlock(const uint32_t typeId, CCustomCSView &mnview) {
+    const auto attributes = mnview.GetAttributes();
 
     CDataStructureV0 activeKey{AttributeTypes::Param, typeId, DFIPKeys::Active};
     const auto active = attributes->GetValue(activeKey, false);
@@ -597,6 +596,8 @@ UniValue setgov(const JSONRPCRequest &request) {
 
     RPCTypeCheck(request.params, {UniValue::VOBJ, UniValue::VARR}, true);
 
+    auto [view, accountView, vaultView] = GetSnapshots();
+
     CDataStream varStream(SER_NETWORK, PROTOCOL_VERSION);
     if (request.params.size() > 0 && request.params[0].isObject()) {
         for (const std::string &name : request.params[0].getKeys()) {
@@ -627,8 +628,8 @@ UniValue setgov(const JSONRPCRequest &request) {
     const UniValue &txInputs = request.params[1];
     CTransactionRef optAuthTx;
     std::set<CScript> auths;
-    rawTx.vin =
-        GetAuthInputsSmart(pwallet, rawTx.nVersion, auths, true, optAuthTx, txInputs, request.metadata.coinSelectOpts);
+    rawTx.vin = GetAuthInputsSmart(
+        pwallet, rawTx.nVersion, auths, true, optAuthTx, txInputs, *view, request.metadata.coinSelectOpts);
 
     CCoinControl coinControl;
 
@@ -720,7 +721,8 @@ UniValue unsetgov(const JSONRPCRequest &request) {
     CScript scriptMeta;
     scriptMeta << OP_RETURN << ToByteVector(metadata);
 
-    int targetHeight = pcustomcsview->GetLastHeight() + 1;
+    auto [view, accountView, vaultView] = GetSnapshots();
+    auto targetHeight = view->GetLastHeight() + 1;
 
     const auto txVersion = GetTransactionVersion(targetHeight);
     CMutableTransaction rawTx(txVersion);
@@ -729,8 +731,8 @@ UniValue unsetgov(const JSONRPCRequest &request) {
     const UniValue &txInputs = request.params[1];
     CTransactionRef optAuthTx;
     std::set<CScript> auths;
-    rawTx.vin =
-        GetAuthInputsSmart(pwallet, rawTx.nVersion, auths, true, optAuthTx, txInputs, request.metadata.coinSelectOpts);
+    rawTx.vin = GetAuthInputsSmart(
+        pwallet, rawTx.nVersion, auths, true, optAuthTx, txInputs, *view, request.metadata.coinSelectOpts);
 
     CCoinControl coinControl;
 
@@ -803,6 +805,8 @@ UniValue setgovheight(const JSONRPCRequest &request) {
 
     RPCTypeCheck(request.params, {UniValue::VOBJ, UniValue::VNUM, UniValue::VARR}, true);
 
+    auto [view, accountView, vaultView] = GetSnapshots();
+
     CDataStream varStream(SER_NETWORK, PROTOCOL_VERSION);
     const auto keys = request.params[0].getKeys();
     if (!keys.empty()) {
@@ -837,8 +841,8 @@ UniValue setgovheight(const JSONRPCRequest &request) {
     const UniValue &txInputs = request.params[2];
     CTransactionRef optAuthTx;
     std::set<CScript> auths;
-    rawTx.vin =
-        GetAuthInputsSmart(pwallet, rawTx.nVersion, auths, true, optAuthTx, txInputs, request.metadata.coinSelectOpts);
+    rawTx.vin = GetAuthInputsSmart(
+        pwallet, rawTx.nVersion, auths, true, optAuthTx, txInputs, *view, request.metadata.coinSelectOpts);
 
     CCoinControl coinControl;
 
@@ -876,24 +880,30 @@ UniValue getgov(const JSONRPCRequest &request) {
         return *res;
     }
 
-    LOCK(cs_main);
+    auto [view, accountView, vaultView] = GetSnapshots();
 
     const auto name = request.params[0].getValStr();
-    if (const auto var = pcustomcsview->GetVariable(name)) {
+    if (const auto var = view->GetVariable(name)) {
         UniValue ret(UniValue::VOBJ);
-        ret.pushKV(var->GetName(), var->Export());
+        if (name == "ATTRIBUTES") {
+            if (auto attributes = std::dynamic_pointer_cast<ATTRIBUTES>(var)) {
+                ret.pushKV(var->GetName(), attributes->ExportFiltered(GovVarsFilter::All, "", view.get()));
+            }
+        } else {
+            ret.pushKV(var->GetName(), var->Export());
+        }
         return GetRPCResultCache().Set(request, ret);
     }
     throw JSONRPCError(RPC_INVALID_REQUEST, "Variable '" + name + "' not registered");
 }
 
-static void AddDefaultVars(uint64_t height, CChainParams params, ATTRIBUTES &attrs) {
+static void AddDefaultVars(uint64_t height, CChainParams params, ATTRIBUTES &attrs, CCustomCSView &view) {
     // OpReturnLimits
     const auto opReturnLimits = OpReturnLimits::From(height, params.GetConsensus(), attrs);
     opReturnLimits.SetToAttributesIfNotExists(attrs);
 
     // TransferDomainConfig
-    const auto tdConfig = TransferDomainConfig::From(*pcustomcsview);
+    const auto tdConfig = TransferDomainConfig::From(view);
     tdConfig.SetToAttributesIfNotExists(attrs);
 }
 
@@ -960,16 +970,16 @@ UniValue listgovs(const JSONRPCRequest &request) {
                                   "ORACLE_DEVIATION",
                                   "ATTRIBUTES"};
 
-    LOCK(cs_main);
+    auto [view, accountView, vaultView] = GetSnapshots();
 
     // Get all stored Gov var changes
-    auto pending = pcustomcsview->GetAllStoredVariables();
-    const auto height = pcustomcsview->GetLastHeight();
+    auto pending = view->GetAllStoredVariables();
+    const auto height = view->GetLastHeight();
 
     UniValue result(UniValue::VARR);
     for (const auto &name : vars) {
         UniValue innerResult(UniValue::VARR);
-        auto var = pcustomcsview->GetVariable(name);
+        auto var = view->GetVariable(name);
         if (var) {
             UniValue ret(UniValue::VOBJ);
             UniValue val;
@@ -979,13 +989,12 @@ UniValue listgovs(const JSONRPCRequest &request) {
                 if (mode == GovVarsFilter::NoAttributes) {
                     skip = true;
                 } else {
-                    if (height >= Params().GetConsensus().DF22MetachainHeight) {
-                        if (auto attributes = dynamic_cast<ATTRIBUTES *>(var.get()); attributes) {
-                            AddDefaultVars(height, Params(), *attributes);
+                    if (auto attributes = std::dynamic_pointer_cast<ATTRIBUTES>(var)) {
+                        if (height >= Params().GetConsensus().DF22MetachainHeight) {
+                            AddDefaultVars(height, Params(), *attributes, *view);
                         }
+                        val = attributes->ExportFiltered(mode, prefix, view.get());
                     }
-                    auto a = std::dynamic_pointer_cast<ATTRIBUTES>(var);
-                    val = a->ExportFiltered(mode, prefix);
                 }
             } else {
                 if (mode == GovVarsFilter::LiveAttributes || mode == GovVarsFilter::PrefixedAttributes ||
@@ -1028,17 +1037,19 @@ UniValue isappliedcustomtx(const JSONRPCRequest &request) {
 
     RPCTypeCheck(request.params, {UniValue::VSTR, UniValue::VNUM}, false);
 
-    LOCK(cs_main);
-
     UniValue result(UniValue::VBOOL);
     result.setBool(false);
 
     uint256 txHash = ParseHashV(request.params[0], "txid");
     int blockHeight = request.params[1].get_int();
 
-    auto blockindex = ::ChainActive()[blockHeight];
-    if (!blockindex) {
-        return result;
+    CBlockIndex *blockindex;
+    {
+        LOCK(cs_main);
+        blockindex = ::ChainActive()[blockHeight];
+        if (!blockindex) {
+            return result;
+        }
     }
 
     uint256 hashBlock;
@@ -1089,6 +1100,8 @@ UniValue listsmartcontracts(const JSONRPCRequest &request) {
     }
         .Check(request);
 
+    auto [view, accountView, vaultView] = GetSnapshots();
+
     UniValue arr(UniValue::VARR);
     for (const auto &item : Params().GetConsensus().smartContracts) {
         UniValue obj(UniValue::VOBJ);
@@ -1098,7 +1111,7 @@ UniValue listsmartcontracts(const JSONRPCRequest &request) {
         obj.pushKV("call", GetContractCall(item.first));
         obj.pushKV("address", EncodeDestination(dest));
 
-        pcustomcsview->ForEachBalance(
+        view->ForEachBalance(
             [&](const CScript &owner, CTokenAmount balance) {
                 if (owner != item.second) {
                     return false;

src/dfi/mn_rpc.h

@@ -87,16 +87,16 @@ std::vector<CTxIn> GetAuthInputsSmart(
     bool needFounderAuth,
     CTransactionRef &optAuthTx,
     const UniValue &explicitInputs,
+    CCustomCSView &mnview,
     const CoinSelectionOptions &coinSelectOpts = CoinSelectionOptions::CreateDefault());
 std::string ScriptToString(const CScript &script);
-CAccounts GetAllMineAccounts(CWallet *const pwallet);
+CAccounts GetAllMineAccounts(CWallet *const pwallet, CCustomCSView &mnview);
 CAccounts SelectAccountsByTargetBalances(const CAccounts &accounts,
                                          const CBalances &targetBalances,
                                          AccountSelectionMode selectionMode);
 void execTestTx(const CTransaction &tx, const uint32_t height, const CTransactionRef &optAuthTx = {});
 CScript CreateScriptForHTLC(const JSONRPCRequest &request, uint32_t &blocks, std::vector<unsigned char> &image);
 CPubKey PublickeyFromString(const std::string &pubkey);
-std::optional<CScript> AmIFounder(CWallet *const pwallet);
-std::optional<FutureSwapHeightInfo> GetFuturesBlock(const uint32_t typeId);
+std::optional<FutureSwapHeightInfo> GetFuturesBlock(const uint32_t typeId, CCustomCSView &mnview);
 std::string CTransferDomainToString(const VMDomain domain);
 #endif  // DEFI_DFI_MN_RPC_H