Aristo db integrate hashify into tx (#1679)

* Renamed type `NoneBackendRef` => `VoidBackendRef`

* Clarify names: `BE=filter+backend` and `UBE=backend (unfiltered)`

why:
  Most functions used full names as `getVtxUnfilteredBackend()` or
  `getKeyBackend()`. After defining abbreviations (and its meaning) it
   seems easier to use `getVtxUBE()` and `getKeyBE()`.

* Integrate `hashify()` process into transaction logic

why:
  Is now transparent unless explicitly controlled.

details:
  Cache changes imply setting a `dirty` flag which in turn triggers
  `hashify()` processing in transaction and `pack()` directives.

* Removed `aristo_tx.exec()` directive

why:
  Inconsistent implementation, functionality will be provided with a
  different paradigm.

nimbus/db/aristo.nim

@@ -14,10 +14,10 @@
 {.push raises: [].}
 
 import aristo/[
-  aristo_constants, aristo_delete, aristo_fetch, aristo_hashify, aristo_init,
+  aristo_constants, aristo_delete, aristo_fetch, aristo_init,
   aristo_merge, aristo_nearby, aristo_tx, aristo_utils]
 export
-  aristo_constants, aristo_delete, aristo_fetch, aristo_hashify, aristo_init,
+  aristo_constants, aristo_delete, aristo_fetch, aristo_init,
   aristo_merge, aristo_nearby, aristo_tx, aristo_utils
 
 import

nimbus/db/aristo/aristo_check.nim

@@ -83,8 +83,8 @@ proc checkBE*(
       return MemBackendRef.checkBE(db, cache=cache, relax=relax)
     of BackendRocksDB:
       return RdbBackendRef.checkBE(db, cache=cache, relax=relax)
-    of BackendNone:
-      return NoneBackendRef.checkBE(db, cache=cache, relax=relax)
+    of BackendVoid:
+      return VoidBackendRef.checkBE(db, cache=cache, relax=relax)
   ok()
 
 # ------------------------------------------------------------------------------

nimbus/db/aristo/aristo_check/check_be.nim

@@ -36,7 +36,7 @@ proc invTo(s: IntervalSetRef[VertexID,uint64]; T: type HashSet[VertexID]): T =
         for pt in w.minPt .. w.maxPt:
           result.incl pt
 
-proc toNodeBe(
+proc toNodeBE(
     vtx: VertexRef;                    # Vertex to convert
     db: AristoDbRef;                   # Database, top layer
       ): Result[NodeRef,VertexID] =
@@ -47,7 +47,7 @@ proc toNodeBe(
     if vtx.lData.pType == AccountData:
       let vid = vtx.lData.account.storageID
       if vid.isValid:
-        let rc = db.getKeyBackend vid
+        let rc = db.getKeyBE vid
         if rc.isErr or not rc.value.isValid:
           return err(vid)
         node.key[0] = rc.value
@@ -58,7 +58,7 @@ proc toNodeBe(
     for n in 0 .. 15:
       let vid = vtx.bVid[n]
       if vid.isValid:
-        let rc = db.getKeyBackend vid
+        let rc = db.getKeyBE vid
         if rc.isOk and rc.value.isValid:
           node.key[n] = rc.value
         else:
@@ -69,7 +69,7 @@ proc toNodeBe(
   of Extension:
     let
       vid = vtx.eVid
-      rc = db.getKeyBackend vid
+      rc = db.getKeyBE vid
     if rc.isOk and rc.value.isValid:
       let node = NodeRef(vType: Extension, ePfx: vtx.ePfx, eVid: vid)
       node.key[0] = rc.value
@@ -80,7 +80,7 @@ proc toNodeBe(
 # Public functions
 # ------------------------------------------------------------------------------
 
-proc checkBE*[T: RdbBackendRef|MemBackendRef|NoneBackendRef](
+proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef](
     _: type T;
     db: AristoDbRef;                   # Database, top layer
     relax: bool;                       # Not compiling hashes if `true`
@@ -94,17 +94,17 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|NoneBackendRef](
   for (_,vid,vtx) in T.walkVtxBE db:
     if not vtx.isValid:
       return err((vid,CheckBeVtxInvalid))
-    let rc = db.getKeyBackend vid
+    let rc = db.getKeyBE vid
     if rc.isErr or not rc.value.isValid:
       return err((vid,CheckBeKeyMissing))
 
   for (_,vid,key) in T.walkKeyBE db:
     if not key.isvalid:
       return err((vid,CheckBeKeyInvalid))
-    let rc = db.getVtxBackend vid
+    let rc = db.getVtxBE vid
     if rc.isErr or not rc.value.isValid:
       return err((vid,CheckBeVtxMissing))
-    let rx = rc.value.toNodeBe db # backend only
+    let rx = rc.value.toNodeBE db # backend only
     if rx.isErr:
       return err((vid,CheckBeKeyCantCompile))
     if not relax:
@@ -143,7 +143,7 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|NoneBackendRef](
         if lbl.isValid:
           return err((vid,CheckBeCacheKeyNonEmpty))
         # There must be a representation on the backend DB
-        if db.getVtxBackend(vid).isErr:
+        if db.getVtxBE(vid).isErr:
           return err((vid,CheckBeCacheVidUnsynced))
         # Register deleted vid against backend generator state
         discard vids.merge Interval[VertexID,uint64].new(vid,vid)

nimbus/db/aristo/aristo_check/check_cache.nim

@@ -103,7 +103,7 @@ proc checkCacheCommon*(
   for (vid,vtx) in db.top.sTab.pairs:
     if not vtx.isValid:
       nNilVtx.inc
-      let rc = db.getVtxBackend vid
+      let rc = db.getVtxBE vid
       if rc.isErr:
         return err((vid,CheckAnyVidVtxMissing))
       if not db.top.kMap.hasKey vid:

nimbus/db/aristo/aristo_debug.nim

@@ -376,7 +376,8 @@ proc ppLayer(
   let
     pfx1 = indent.toPfx
     pfx2 = indent.toPfx(1)
-    tagOk = 1 < sTabOk.ord + lTabOk.ord + kMapOk.ord + pPrfOk.ord + vGenOk.ord
+    nOKs = sTabOk.ord + lTabOk.ord + kMapOk.ord + pPrfOk.ord + vGenOk.ord
+    tagOk = 1 < nOKs
   var
     pfy = ""
 
@@ -419,6 +420,10 @@ proc ppLayer(
         tLen = layer.pPrf.len
         info = "pPrf(" & $tLen & ")"
       result &= info.doPrefix(0 < tLen) & layer.pPrf.ppPPrf
+    if 0 < nOKs:
+      let
+        info = if layer.dirty: "dirty" else: "clean"
+      result &= info.doPrefix(false)
 
 # ------------------------------------------------------------------------------
 # Public functions
@@ -615,14 +620,14 @@ proc pp*(
   indent = 4;
     ): string =
   ## May be called as `db.to(TypedBackendRef).pp(db)`
-  case (if be.isNil: BackendNone else: be.kind)
+  case (if be.isNil: BackendVoid else: be.kind)
   of BackendMemory:
     be.MemBackendRef.ppBe(db, indent)
 
   of BackendRocksDB:
     be.RdbBackendRef.ppBe(db, indent)
 
-  of BackendNone:
+  of BackendVoid:
     db.roFilter.ppFilter(db, indent) & indent.toPfx & "<BackendNone>"
 
 # ------------------------------------------------------------------------------

nimbus/db/aristo/aristo_delete.nim

@@ -50,7 +50,7 @@ proc clearKey(
   if lbl.isValid:
     db.top.kMap.del vid
     db.top.pAmk.del lbl
-  elif db.getKeyBackend(vid).isOK:
+  elif db.getKeyBE(vid).isOK:
     # Register for deleting on backend
     db.top.kMap[vid] = VOID_HASH_LABEL
     db.top.pAmk.del lbl
@@ -60,7 +60,7 @@ proc doneWith(
     vid: VertexID;                     # Vertex IDs to clear
       ) =
   # Remove entry
-  if db.getVtxBackend(vid).isOk:
+  if db.getVtxBE(vid).isOk:
     db.top.sTab[vid] = VertexRef(nil)  # Will be propagated to backend
   else:
     db.top.sTab.del vid
@@ -278,14 +278,17 @@ proc deleteImpl(
 
   # Will be needed at the end. Just detect an error early enouhh
   let leafVidBe = block:
-    let rc = db.getVtxBackend lf.vid
+    let rc = db.getVtxBE lf.vid
     if rc.isErr:
       if rc.error != GetVtxNotFound:
         return err((lf.vid, rc.error))
       VertexRef(nil)
     else:
       rc.value
 
+  # Will modify top level cache
+  db.top.dirty = true
+
   db.doneWith lf.vid
 
   if 1 < hike.legs.len:

nimbus/db/aristo/aristo_desc.nim

@@ -42,7 +42,7 @@ type
     db*: AristoDbRef                  ## Database descriptor
     parent*: AristoTxRef              ## Previous transaction
     txUid*: uint                      ## Unique ID among transactions
-    stackInx*: int                    ## Stack index for this transaction
+    level*: int                       ## Stack index for this transaction
 
   AristoDbRef* = ref AristoDbObj
   AristoDbObj* = object

nimbus/db/aristo/aristo_desc/aristo_error.nim

@@ -191,14 +191,12 @@ type
     RdbBeIngestSstWriter
 
     # Transaction wrappers
-    TxNoPendingTx
     TxArgStaleTx
+    TxBackendMissing
+    TxNoPendingTx
     TxNotTopTx
-    TxExecNestingAttempt
-    TxExecBaseTxLocked
-    TxExecDirectiveLocked
+    TxStackGarbled
     TxStackUnderflow
-    TxBackendMissing
 
     # Miscelaneous handy helpers
     PayloadTypeUnsupported

nimbus/db/aristo/aristo_desc/aristo_types_structural.nim

@@ -106,6 +106,7 @@ type
     pPrf*: HashSet[VertexID]          ## Locked vertices (proof nodes)
     vGen*: seq[VertexID]              ## Unique vertex ID generator
     txUid*: uint                      ## Transaction identifier if positive
+    dirty*: bool                      ## Needs to be hashified if `true`
 
 # ------------------------------------------------------------------------------
 # Public helpers: `NodeRef` and `PayloadRef`

nimbus/db/aristo/aristo_filter.nim

@@ -29,7 +29,7 @@ type
 proc getBeStateRoot(
     db: AristoDbRef;
       ): Result[HashKey,AristoError] =
-  let rc = db.getKeyBackend VertexID(1)
+  let rc = db.getKeyBE VertexID(1)
   if rc.isOk:
     return ok(rc.value)
   if rc.error == GetKeyNotFound:
@@ -39,7 +39,7 @@ proc getBeStateRoot(
 proc getLayerStateRoots(
     db: AristoDbRef;
     layer: AristoLayerRef;
-    extendOK: bool;
+    chunkedMpt: bool;
       ): Result[StateRootPair,AristoError] =
   ## Get the Merkle hash key for target state root to arrive at after this
   ## reverse filter was applied.
@@ -53,7 +53,7 @@ proc getLayerStateRoots(
     spr.fg = layer.kMap.getOrVoid(VertexID 1).key
     if spr.fg.isValid:
       return ok(spr)
-  if extendOK:
+  if chunkedMpt:
     let vid = layer.pAmk.getOrVoid HashLabel(root: VertexID(1), key: spr.be)
     if vid == VertexID(1):
       spr.fg = spr.be
@@ -90,7 +90,7 @@ func bulk*(layer: AristolayerRef): int =
 proc fwdFilter*(
     db: AristoDbRef;
     layer: AristoLayerRef;
-    extendOK = false;
+    chunkedMpt = false;
       ): Result[AristoFilterRef,(VertexID,AristoError)] =
   ## Assemble forward delta, i.e. changes to the backend equivalent to applying
   ## the current top layer.
@@ -107,7 +107,7 @@ proc fwdFilter*(
   # Register the Merkle hash keys of the MPT where this reverse filter will be
   # applicable: `be => fg`
   let (srcRoot, trgRoot) = block:
-    let rc = db.getLayerStateRoots(layer, extendOk)
+    let rc = db.getLayerStateRoots(layer, chunkedMpt)
     if rc.isOK:
       (rc.value.be, rc.value.fg)
     elif rc.error == FilPrettyPointlessLayer:
@@ -182,7 +182,7 @@ proc merge*(
     if vtx.isValid or not newFilter.sTab.hasKey vid:
       newFilter.sTab[vid] = vtx
     elif newFilter.sTab.getOrVoid(vid).isValid:
-      let rc = db.getVtxUnfilteredBackend vid
+      let rc = db.getVtxUBE vid
       if rc.isOk:
         newFilter.sTab[vid] = vtx # VertexRef(nil)
       elif rc.error == GetVtxNotFound:
@@ -194,7 +194,7 @@ proc merge*(
     if key.isValid or not newFilter.kMap.hasKey vid:
       newFilter.kMap[vid] = key
     elif newFilter.kMap.getOrVoid(vid).isValid:
-      let rc = db.getKeyUnfilteredBackend vid
+      let rc = db.getKeyUBE vid
       if rc.isOk:
         newFilter.kMap[vid] = key # VOID_HASH_KEY
       elif rc.error == GetKeyNotFound:

nimbus/db/aristo/aristo_get.nim

@@ -27,68 +27,68 @@ type
 # Public functions
 # ------------------------------------------------------------------------------
 
-proc getIdgUnfilteredBackend*(
+proc getIdgUBE*(
     db: AristoDbRef;
       ): Result[seq[VertexID],AristoError] =
-  ## Get the ID generator state the `backened` layer if available.
+  ## Get the ID generator state from the unfiltered backened if available.
   let be = db.backend
   if not be.isNil:
     return be.getIdgFn()
   err(GetIdgNotFound)
 
-proc getVtxUnfilteredBackend*(
+proc getVtxUBE*(
     db: AristoDbRef;
     vid: VertexID;
       ): Result[VertexRef,AristoError] =
-  ## Get the vertex from the `backened` layer if available.
+  ## Get the vertex from the unfiltered backened if available.
   let be = db.backend
   if not be.isNil:
     return be.getVtxFn vid
   err GetVtxNotFound
 
-proc getKeyUnfilteredBackend*(
+proc getKeyUBE*(
     db: AristoDbRef;
     vid: VertexID;
       ): Result[HashKey,AristoError] =
-  ## Get the merkle hash/key from the backend
+  ## Get the merkle hash/key from the unfiltered backend if available.
   let be = db.backend
   if not be.isNil:
     return be.getKeyFn vid
   err GetKeyNotFound
 
 # ------------------
 
-proc getIdgBackend*(
+proc getIdgBE*(
     db: AristoDbRef;
       ): Result[seq[VertexID],AristoError] =
   ## Get the ID generator state the `backened` layer if available.
   if not db.roFilter.isNil and db.roFilter.vGen.isSome:
     return ok(db.roFilter.vGen.unsafeGet)
-  db.getIdgUnfilteredBackend()
+  db.getIdgUBE()
 
-proc getVtxBackend*(
+proc getVtxBE*(
     db: AristoDbRef;
     vid: VertexID;
       ): Result[VertexRef,AristoError] =
-  ## Get the vertex from the `backened` layer if available.
+  ## Get the vertex from the (filtered) backened if available.
   if not db.roFilter.isNil and db.roFilter.sTab.hasKey vid:
     let vtx = db.roFilter.sTab.getOrVoid vid
     if vtx.isValid:
       return ok(vtx)
     return err(GetVtxNotFound)
-  db.getVtxUnfilteredBackend vid
+  db.getVtxUBE vid
 
-proc getKeyBackend*(
+proc getKeyBE*(
     db: AristoDbRef;
     vid: VertexID;
       ): Result[HashKey,AristoError] =
-  ## Get the merkle hash/key from the backend
+  ## Get the merkle hash/key from the (filtered) backend if available.
   if not db.roFilter.isNil and db.roFilter.kMap.hasKey vid:
     let key = db.roFilter.kMap.getOrVoid vid
     if key.isValid:
       return ok(key)
     return err(GetKeyNotFound)
-  db.getKeyUnfilteredBackend vid
+  db.getKeyUBE vid
 
 # ------------------
 
@@ -126,7 +126,7 @@ proc getVtx*(db: AristoDbRef; vid: VertexID): VertexRef =
     # If the vertex is to be deleted on the backend, a `VertexRef(nil)` entry
     # is kept in the local table in which case it is OK to return this value.
     return db.top.sTab.getOrVoid vid
-  let rc = db.getVtxBackend vid
+  let rc = db.getVtxBE vid
   if rc.isOk:
     return rc.value
   VertexRef(nil)
@@ -139,7 +139,7 @@ proc getKey*(db: AristoDbRef; vid: VertexID): HashKey =
     # If the key is to be deleted on the backend, a `VOID_HASH_LABEL` entry
     # is kept on the local table in which case it is OK to return this value.
     return db.top.kMap.getOrVoid(vid).key
-  let rc = db.getKeyBackend vid
+  let rc = db.getKeyBE vid
   if rc.isOk:
     return rc.value
   VOID_HASH_KEY