Aristo db use filter betw backend and tx cache (#1678)

* Provide deep copy for each transaction layer

why:
  Localising changes. Selective deep copy was just overlooked.

* Generalise vertex ID generator state reorg function `vidReorg()`

why:
  makes it somewhat easier to handle when saving layers.

* Provide dummy back end descriptor `NoneBackendRef`

* Optional read-only filter between backend and transaction cache

why:
  Some staging area for accumulating changes to the backend DB. This
  will eventually be an access layer for emulating a backend with
  multiple/historic state roots.

* Re-factor `persistent()` with filter between backend/tx-cache => `stow()`

why:
  The filter provides an abstraction from the physically stored data on
  disk. So, there can be several MPT instances using the same disk data
  with different state roots. Of course, all the MPT instances should
  not differ too much for practical reasons :).

TODO:
  Filter administration tools need to be provided.

nimbus/db/aristo.nim

@@ -13,11 +13,6 @@
 ##
 {.push raises: [].}
 
-import aristo/aristo_desc/[
-  aristo_types_identifiers, aristo_types_structural]
-export
-  aristo_types_identifiers, aristo_types_structural
-
 import aristo/[
   aristo_constants, aristo_delete, aristo_fetch, aristo_hashify, aristo_init,
   aristo_merge, aristo_nearby, aristo_tx, aristo_utils]
@@ -30,10 +25,13 @@ import
 export
   append, read
 
-import
-  aristo/aristo_vid
+import aristo/aristo_desc/[
+  aristo_types_identifiers, aristo_types_structural]
 export
-  vidFetch
+  AristoAccount,
+  PayloadRef,
+  PayloadType,
+  `==`
 
 import
   aristo/aristo_desc

nimbus/db/aristo/aristo_check.nim

@@ -17,7 +17,7 @@ import
   std/[algorithm, sequtils, sets, tables],
   eth/common,
   stew/[interval_set, results],
-  ./aristo_init/[aristo_memory, aristo_rocksdb],
+  ./aristo_walk/persistent,
   "."/[aristo_desc, aristo_get, aristo_init, aristo_vid, aristo_utils],
   ./aristo_check/[check_be, check_cache]
 
@@ -80,11 +80,11 @@ proc checkBE*(
     let be = db.to(TypedBackendRef)
     case be.kind:
     of BackendMemory:
-      return be.MemBackendRef.checkBE(db, cache=cache, relax=relax)
+      return MemBackendRef.checkBE(db, cache=cache, relax=relax)
     of BackendRocksDB:
-      return be.RdbBackendRef.checkBE(db, cache=cache, relax=relax)
+      return RdbBackendRef.checkBE(db, cache=cache, relax=relax)
     of BackendNone:
-      discard
+      return NoneBackendRef.checkBE(db, cache=cache, relax=relax)
   ok()
 
 # ------------------------------------------------------------------------------

nimbus/db/aristo/aristo_check/check_be.nim

@@ -14,8 +14,9 @@ import
   std/[algorithm, sequtils, sets, tables],
   eth/common,
   stew/interval_set,
-  ../aristo_init/[aristo_memory, aristo_rocksdb],
-  ".."/[aristo_desc, aristo_get, aristo_vid, aristo_transcode, aristo_utils]
+  ../../aristo,
+  ../aristo_walk/persistent,
+  ".."/[aristo_desc, aristo_get, aristo_vid, aristo_transcode]
 
 const
   Vid2 = @[VertexID(2)].toHashSet
@@ -79,8 +80,8 @@ proc toNodeBe(
 # Public functions
 # ------------------------------------------------------------------------------
 
-proc checkBE*[T](
-    be: T;                             # backend descriptor
+proc checkBE*[T: RdbBackendRef|MemBackendRef|NoneBackendRef](
+    _: type T;
     db: AristoDbRef;                   # Database, top layer
     relax: bool;                       # Not compiling hashes if `true`
     cache: bool;                       # Also verify cache
@@ -90,14 +91,14 @@ proc checkBE*[T](
   let vids = IntervalSetRef[VertexID,uint64].init()
   discard vids.merge Interval[VertexID,uint64].new(VertexID(1),high(VertexID))
 
-  for (_,vid,vtx) in be.walkVtx:
+  for (_,vid,vtx) in T.walkVtxBE db:
     if not vtx.isValid:
       return err((vid,CheckBeVtxInvalid))
     let rc = db.getKeyBackend vid
     if rc.isErr or not rc.value.isValid:
       return err((vid,CheckBeKeyMissing))
 
-  for (_,vid,key) in be.walkKey:
+  for (_,vid,key) in T.walkKeyBE db:
     if not key.isvalid:
       return err((vid,CheckBeKeyInvalid))
     let rc = db.getVtxBackend vid
@@ -116,7 +117,7 @@ proc checkBE*[T](
   block:
     # Extract vertex ID generator state
     var vGen: HashSet[VertexID]
-    for (_,_,w) in be.walkIdg:
+    for (_,_,w) in T.walkIdgBE db:
       vGen = vGen + w.toHashSet
     let
       vGenExpected = vids.invTo(HashSet[VertexID])
@@ -128,7 +129,6 @@ proc checkBE*[T](
 
   # Check cache against backend
   if cache:
-
     # Check structural table
     for (vid,vtx) in db.top.sTab.pairs:
       # A `kMap[]` entry must exist.
@@ -164,10 +164,8 @@ proc checkBE*[T](
           return err((vid,CheckBeCacheKeyMismatch))
 
     # Check vGen
-    var tmp = AristoDbRef(top: AristoLayerRef(vGen: db.top.vGen))
-    tmp.vidReorg()
     let
-      vGen = tmp.top.vGen.toHashSet
+      vGen = db.top.vGen.vidReorg.toHashSet
       vGenExpected = vids.invTo(HashSet[VertexID])
       delta = vGenExpected -+- vGen # symmetric difference
     if 0 < delta.len:

nimbus/db/aristo/aristo_debug.nim

@@ -17,6 +17,9 @@ import
   "."/[aristo_constants, aristo_desc, aristo_hike, aristo_init],
   ./aristo_init/[aristo_memory, aristo_rocksdb]
 
+export
+  TypedBackendRef, aristo_init.to
+
 # ------------------------------------------------------------------------------
 # Ptivate functions
 # ------------------------------------------------------------------------------
@@ -27,6 +30,9 @@ proc sortedKeys(lTab: Table[LeafTie,VertexID]): seq[LeafTie] =
 proc sortedKeys(kMap: Table[VertexID,HashLabel]): seq[VertexID] =
   kMap.keys.toSeq.mapIt(it.uint64).sorted.mapIt(it.VertexID)
 
+proc sortedKeys(kMap: Table[VertexID,HashKey]): seq[VertexID] =
+  kMap.keys.toSeq.mapIt(it.uint64).sorted.mapIt(it.VertexID)
+
 proc sortedKeys(sTab: Table[VertexID,VertexRef]): seq[VertexID] =
   sTab.keys.toSeq.mapIt(it.uint64).sorted.mapIt(it.VertexID)
 
@@ -311,7 +317,32 @@ proc ppXMap*(
   else:
     result &= "}"
 
-proc ppBe[T](be: T; db: AristoDbRef; indent: int): string =
+proc ppFilter(fl: AristoFilterRef; db: AristoDbRef; indent: int): string =
+  ## Walk over filter tables
+  let
+    pfx = indent.toPfx
+    pfx1 = indent.toPfx(1)
+    pfx2 = indent.toPfx(2)
+  result = "<filter>"
+  if db.roFilter.isNil:
+    result &= " n/a"
+    return
+  result &= pfx & "vGen" & pfx1 & "["
+  if fl.vGen.isSome:
+    result &= fl.vGen.unsafeGet.mapIt(it.ppVid).join(",")
+  result &= "]" & pfx & "sTab" & pfx1 & "{"
+  for n,vid in fl.sTab.sortedKeys:
+    let vtx = fl.sTab.getOrVoid vid
+    if 0 < n: result &= pfx2
+    result &= $(1+n) & "(" & vid.ppVid & "," & vtx.ppVtx(db,vid) & ")"
+  result &= "}" & pfx & "kMap" & pfx1 & "{"
+  for n,vid in fl.kMap.sortedKeys:
+    let key = fl.kMap.getOrVoid vid
+    if 0 < n: result &= pfx2
+    result &= $(1+n) & "(" & vid.ppVid & "," & key.ppKey & ")"
+  result &= "}"
+
+proc ppBeOnly[T](be: T; db: AristoDbRef; indent: int): string =
   ## Walk over backend tables
   let
     pfx = indent.toPfx
@@ -328,8 +359,12 @@ proc ppBe[T](be: T; db: AristoDbRef; indent: int): string =
       $(1+it[0]) & "(" & it[1].ppVid & "," & it[2].ppKey & ")"
     ).join(pfx2) & "}"
 
+proc ppBe[T](be: T; db: AristoDbRef; indent: int): string =
+  ## backend + filter
+  db.roFilter.ppFilter(db, indent) & indent.toPfx & be.ppBeOnly(db,indent)
 
-proc ppCache(
+proc ppLayer(
+    layer: AristoLayerRef;
     db: AristoDbRef;
     vGenOk: bool;
     sTabOk: bool;
@@ -355,35 +390,35 @@ proc ppCache(
       pfy = pfx2
     rc
 
-  if not db.top.isNil:
+  if not layer.isNil:
     if vGenOk:
       let
-        tLen = db.top.vGen.len
+        tLen = layer.vGen.len
         info = "vGen(" & $tLen & ")"
-      result &= info.doPrefix(0 < tLen) & db.top.vGen.ppVidList
+      result &= info.doPrefix(0 < tLen) & layer.vGen.ppVidList
     if sTabOk:
       let
-        tLen = db.top.sTab.len
+        tLen = layer.sTab.len
         info = "sTab(" & $tLen & ")"
-      result &= info.doPrefix(0 < tLen) & db.top.sTab.ppSTab(db,indent+1)
+      result &= info.doPrefix(0 < tLen) & layer.sTab.ppSTab(db,indent+1)
     if lTabOk:
       let
-        tlen = db.top.lTab.len
+        tlen = layer.lTab.len
         info = "lTab(" & $tLen & ")"
-      result &= info.doPrefix(0 < tLen) & db.top.lTab.ppLTab(indent+1)
+      result &= info.doPrefix(0 < tLen) & layer.lTab.ppLTab(indent+1)
     if kMapOk:
       let
-        tLen = db.top.kMap.len
-        ulen = db.top.pAmk.len
+        tLen = layer.kMap.len
+        ulen = layer.pAmk.len
         lInf = if tLen == uLen: $tLen else: $tLen & "," & $ulen
         info = "kMap(" & lInf & ")"
       result &= info.doPrefix(0 < tLen + uLen)
-      result &= db.ppXMap(db.top.kMap,db.top.pAmk,indent+1)
+      result &= db.ppXMap(layer.kMap, layer.pAmk,indent+1)
     if pPrfOk:
       let
-        tLen = db.top.pPrf.len
+        tLen = layer.pPrf.len
         info = "pPrf(" & $tLen & ")"
-      result &= info.doPrefix(0 < tLen) & db.top.pPrf.ppPPrf
+      result &= info.doPrefix(0 < tLen) & layer.pPrf.ppPPrf
 
 # ------------------------------------------------------------------------------
 # Public functions
@@ -522,20 +557,47 @@ proc pp*(
 
 # ---------------------
 
+proc pp*(
+    layer: AristoLayerRef;
+    db: AristoDbRef;
+    indent = 4;
+      ): string =
+  layer.ppLayer(
+    db, vGenOk=true, sTabOk=true, lTabOk=true, kMapOk=true, pPrfOk=true)
+
+proc pp*(
+    layer: AristoLayerRef;
+    db: AristoDbRef;
+    xTabOk: bool;
+    indent = 4;
+      ): string =
+  layer.ppLayer(
+    db, vGenOk=true, sTabOk=xTabOk, lTabOk=xTabOk, kMapOk=true, pPrfOk=true)
+
+proc pp*(
+    layer: AristoLayerRef;
+    db: AristoDbRef;
+    xTabOk: bool;
+    kMapOk: bool;
+    other = false;
+    indent = 4;
+      ): string =
+  layer.ppLayer(
+    db, vGenOk=other, sTabOk=xTabOk, lTabOk=xTabOk, kMapOk=kMapOk, pPrfOk=other)
+
+
 proc pp*(
     db: AristoDbRef;
     indent = 4;
       ): string =
-  db.ppCache(
-    vGenOk=true, sTabOk=true, lTabOk=true, kMapOk=true, pPrfOk=true)
+  db.top.pp(db, indent=indent)
 
 proc pp*(
     db: AristoDbRef;
     xTabOk: bool;
     indent = 4;
       ): string =
-  db.ppCache(
-    vGenOk=true, sTabOk=xTabOk, lTabOk=xTabOk, kMapOk=true, pPrfOk=true)
+  db.top.pp(db, xTabOk=xTabOk, indent=indent)
 
 proc pp*(
     db: AristoDbRef;
@@ -544,15 +606,15 @@ proc pp*(
     other = false;
     indent = 4;
       ): string =
-  db.ppCache(
-    vGenOk=other, sTabOk=xTabOk, lTabOk=xTabOk, kMapOk=kMapOk, pPrfOk=other)
+  db.top.pp(db, xTabOk=xTabOk, kMapOk=kMapOk, other=other, indent=indent)
+
 
 proc pp*(
   be: TypedBackendRef;
   db: AristoDbRef;
   indent = 4;
     ): string =
-
+  ## May be called as `db.to(TypedBackendRef).pp(db)`
   case (if be.isNil: BackendNone else: be.kind)
   of BackendMemory:
     be.MemBackendRef.ppBe(db, indent)
@@ -561,7 +623,7 @@ proc pp*(
     be.RdbBackendRef.ppBe(db, indent)
 
   of BackendNone:
-    "n/a"
+    db.roFilter.ppFilter(db, indent) & indent.toPfx & "<BackendNone>"
 
 # ------------------------------------------------------------------------------
 # End

nimbus/db/aristo/aristo_desc.nim

@@ -22,49 +22,35 @@
 {.push raises: [].}
 
 import
-  std/[sets, tables],
+  std/tables,
   eth/common,
   ./aristo_constants,
   ./aristo_desc/[
-    aristo_error, aristo_types_backend,
-    aristo_types_identifiers, aristo_types_structural]
+    aristo_error, aristo_types_identifiers, aristo_types_structural]
+
+from ./aristo_desc/aristo_types_backend
+  import AristoBackendRef
 
 export
   # Not auto-exporting backend
   aristo_constants, aristo_error, aristo_types_identifiers,
   aristo_types_structural
 
 type
-  AristoChangeLogRef* = ref object
-    ## Change log: database state before backend saving.
-    root*: HashKey                    ## Previous hash key for `VertexID(1)`
-    leafs*: Table[LeafTie,PayloadRef] ## Changed leafs after merge into backend
-
   AristoTxRef* = ref object
     ## Transaction descriptor
     db*: AristoDbRef                  ## Database descriptor
     parent*: AristoTxRef              ## Previous transaction
     txUid*: uint                      ## Unique ID among transactions
     stackInx*: int                    ## Stack index for this transaction
 
-  AristoLayerRef* = ref object
-    ## Hexary trie database layer structures. Any layer holds the full
-    ## change relative to the backend.
-    sTab*: Table[VertexID,VertexRef]  ## Structural vertex table
-    lTab*: Table[LeafTie,VertexID]    ## Direct access, path to leaf vertex
-    kMap*: Table[VertexID,HashLabel]  ## Merkle hash key mapping
-    pAmk*: Table[HashLabel,VertexID]  ## Reverse `kMap` entries, hash key lookup
-    pPrf*: HashSet[VertexID]          ## Locked vertices (proof nodes)
-    vGen*: seq[VertexID]              ## Unique vertex ID generator
-    txUid*: uint                      ## Transaction identifier if positive
-
   AristoDbRef* = ref AristoDbObj
   AristoDbObj* = object
     ## Set of database layers, supporting transaction frames
     top*: AristoLayerRef              ## Database working layer, mutable
     stack*: seq[AristoLayerRef]       ## Stashed immutable parent layers
+    roFilter*: AristoFilterRef        ## Apply read filter (locks writing)
     backend*: AristoBackendRef        ## Backend database (may well be `nil`)
-    history*: seq[AristoChangeLogRef] ## Backend saving history
 
     txRef*: AristoTxRef               ## Latest active transaction
     txUidGen*: uint                   ## Tx-relative unique number generator
@@ -82,6 +68,9 @@ func getOrVoid*[W](tab: Table[W,VertexRef]; w: W): VertexRef =
 func getOrVoid*[W](tab: Table[W,HashLabel]; w: W): HashLabel =
   tab.getOrDefault(w, VOID_HASH_LABEL)
 
+func getOrVoid*[W](tab: Table[W,HashKey]; w: W): HashKey =
+  tab.getOrDefault(w, VOID_HASH_KEY)
+
 func getOrVoid*[W](tab: Table[W,VertexID]; w: W): VertexID =
   tab.getOrDefault(w, VertexID(0))
 
@@ -106,7 +95,6 @@ func isValid*(vid: VertexID): bool =
   vid != VertexID(0)
 
 # ------------------------------------------------------------------------------
-
 # Public functions, miscellaneous
 # ------------------------------------------------------------------------------