Implement distributed merge of backend filters

status-im · Aug 17, 2023 · e109055 · e109055
1 parent 1466da2
commit e109055
Show file tree

Hide file tree

Showing 7 changed files with 290 additions and 115 deletions.
diff --git a/nimbus/db/aristo/aristo_debug.nim b/nimbus/db/aristo/aristo_debug.nim
@@ -200,7 +200,7 @@ proc ppSTab(
   "{" & sTab.sortedKeys
             .mapIt((it, sTab.getOrVoid it))
             .mapIt("(" & it[0].ppVid & "," & it[1].ppVtx(db,it[0]) & ")")
-            .join("," & indent.toPfx(1)) & "}"
+            .join(indent.toPfx(2)) & "}"
 
 proc ppLTab(
     lTab: Table[LeafTie,VertexID];
@@ -210,7 +210,7 @@ proc ppLTab(
   "{" & lTab.sortedKeys
             .mapIt((it, lTab.getOrVoid it))
             .mapIt("(" & it[0].ppLeafTie(db) & "," & it[1].ppVid & ")")
-            .join("," & indent.toPfx(1)) & "}"
+            .join(indent.toPfx(2)) & "}"
 
 proc ppPPrf(pPrf: HashSet[VertexID]): string =
   "{" & pPrf.sortedKeys.mapIt(it.ppVid).join(",") & "}"
@@ -324,9 +324,11 @@ proc ppFilter(fl: AristoFilterRef; db: AristoDbRef; indent: int): string =
     pfx1 = indent.toPfx(1)
     pfx2 = indent.toPfx(2)
   result = "<filter>"
-  if db.roFilter.isNil:
+  if fl.isNil:
     result &= " n/a"
     return
+  result &= pfx & "trg(" & fl.trg.ppKey & ")"
+  result &= pfx & "src(" & fl.src.ppKey & ")"
   result &= pfx & "vGen" & pfx1 & "["
   if fl.vGen.isSome:
     result &= fl.vGen.unsafeGet.mapIt(it.ppVid).join(",")
@@ -361,7 +363,7 @@ proc ppBeOnly[T](be: T; db: AristoDbRef; indent: int): string =
 
 proc ppBe[T](be: T; db: AristoDbRef; indent: int): string =
   ## backend + filter
-  db.roFilter.ppFilter(db, indent) & indent.toPfx & be.ppBeOnly(db,indent)
+  db.roFilter.ppFilter(db, indent+1) & indent.toPfx & be.ppBeOnly(db,indent+1)
 
 proc ppLayer(
     layer: AristoLayerRef;
@@ -374,8 +376,8 @@ proc ppLayer(
     indent = 4;
       ): string =
   let
-    pfx1 = indent.toPfx
-    pfx2 = indent.toPfx(1)
+    pfx1 = indent.toPfx(1)
+    pfx2 = indent.toPfx(2)
     nOKs = sTabOk.ord + lTabOk.ord + kMapOk.ord + pPrfOk.ord + vGenOk.ord
     tagOk = 1 < nOKs
   var
@@ -392,6 +394,8 @@ proc ppLayer(
     rc
 
   if not layer.isNil:
+    if 2 < nOKs:
+      result &= "<layer>".doPrefix(false)
     if vGenOk:
       let
         tLen = layer.vGen.len
@@ -613,6 +617,12 @@ proc pp*(
       ): string =
   db.top.pp(db, xTabOk=xTabOk, kMapOk=kMapOk, other=other, indent=indent)
 
+proc pp*(
+    filter: AristoFilterRef;
+    db = AristoDbRef();
+    indent = 4;
+      ): string =
+  filter.ppFilter(db, indent)
 
 proc pp*(
   be: TypedBackendRef;

diff --git a/nimbus/db/aristo/aristo_desc.nim b/nimbus/db/aristo/aristo_desc.nim
@@ -22,7 +22,7 @@
 {.push raises: [].}
 
 import
-  std/tables,
+  std/[sets, tables],
   eth/common,
   ./aristo_constants,
   ./aristo_desc/[
@@ -31,8 +31,8 @@ import
 from ./aristo_desc/aristo_types_backend
   import AristoBackendRef
 
+# Not auto-exporting backend
 export
-  # Not auto-exporting backend
   aristo_constants, aristo_error, aristo_types_identifiers,
   aristo_types_structural
 
@@ -44,6 +44,13 @@ type
     txUid*: uint                      ## Unique ID among transactions
     level*: int                       ## Stack index for this transaction
 
+  AristoDudesRef* = ref object
+    case rwOk*: bool
+    of true:
+      roDudes*: HashSet[AristoDbRef]  ## Read-only peers
+    else:
+      rwDb*: AristoDbRef              ## Link to writable descriptor
+
   AristoDbRef* = ref AristoDbObj
   AristoDbObj* = object
     ## Set of database layers, supporting transaction frames
@@ -54,6 +61,7 @@ type
 
     txRef*: AristoTxRef               ## Latest active transaction
     txUidGen*: uint                   ## Tx-relative unique number generator
+    dudes*: AristoDudesRef            ## Related DB descriptors
 
     # Debugging data below, might go away in future
     xMap*: Table[HashLabel,VertexID]  ## For pretty printing, extends `pAmk`

diff --git a/nimbus/db/aristo/aristo_desc/aristo_error.nim b/nimbus/db/aristo/aristo_desc/aristo_error.nim
@@ -169,9 +169,11 @@ type
     DelVidStaleVtx
 
     # Functions from  `aristo_filter.nim`
+    FilRoBackendOrMissing
     FilStateRootMissing
     FilStateRootMismatch
     FilPrettyPointlessLayer
+    FilDudeFilterUpdateError
 
     # Get functions form `aristo_get.nim`
     GetLeafNotFound
@@ -194,8 +196,9 @@ type
 
     # Transaction wrappers
     TxArgStaleTx
-    TxBackendMissing
+    TxRoBackendOrMissing
     TxNoPendingTx
+    TxPendingTx
     TxNotTopTx
     TxStackGarbled
     TxStackUnderflow

diff --git a/nimbus/db/aristo/aristo_desc/aristo_types_structural.nim b/nimbus/db/aristo/aristo_desc/aristo_types_structural.nim
@@ -250,6 +250,16 @@ proc dup*(layer: AristoLayerRef): AristoLayerRef =
   for (k,v) in layer.sTab.pairs:
     result.sTab[k] = v.dup
 
+proc dup*(filter: AristoFilterRef): AristoFilterRef =
+  ## Duplicate layer.
+  result = AristoFilterRef(
+    src:  filter.src,
+    kMap: filter.kMap,
+    vGen: filter.vGen,
+    trg:  filter.trg)
+  for (k,v) in filter.sTab.pairs:
+    result.sTab[k] = v.dup
+
 proc to*(node: NodeRef; T: type VertexRef): T =
   ## Extract a copy of the `VertexRef` part from a `NodeRef`.
   node.VertexRef.dup