Skip to content

Commit

Permalink
Core db update providing additional results code interface (#1776)
Browse files Browse the repository at this point in the history
* Split `core_db/base.nim` into several sources

* Rename `core_db/legacy.nim` => `core_db/legacy_db.nim`

* Update `CoreDb` API, dual methods returning `Result[]` or plain value

detail:
  Plain value methods implemet the legacy API, they defect on error results

* Redesign `CoreDB` direct backend access

why:
  Made the `backend` directive integral part of the API

* Discontinue providing unused or otherwise available functions

details:
+ setTransactionID() removed, not used and not easily replicable in Aristo
+ maybeGet() removed, available via direct backend access
+ newPhk() removed, never used & was experimental anyway

* Update/reorg backend API

why:
+ Added error print function `$$()`
+ General descriptor completion (and optional validation) via `bless()`

* Update `Aristo`/`Kvt` exception handling

why:
  Avoid `CatchableError` exceptions, rather pass them as error code where
  appropriate.

* More `CoreDB` compliant `Aristo` and `Kvt` methods

details:
+ Providing functions like `contains()`, `getVtxRc()` (returns `Result[]`).
+ Additional error code: `NotImplemented`

* Rewrite/reorg of Aristo DB constructor

why:
  Previously used global object `DefaultQidLayoutRef` as default
  initialiser. This object was created at compile time which lead to
  non-gc safe functions.

* Update nimbus/db/core_db/legacy_db.nim

Co-authored-by: Kim De Mey <[email protected]>

* Update nimbus/db/aristo/aristo_transcode.nim

Co-authored-by: Kim De Mey <[email protected]>

* Update nimbus/db/core_db/legacy_db.nim

Co-authored-by: Kim De Mey <[email protected]>

---------

Co-authored-by: Kim De Mey <[email protected]>
  • Loading branch information
mjfh and kdeme authored Sep 26, 2023
1 parent 6848b20 commit 6d13281
Show file tree
Hide file tree
Showing 34 changed files with 1,312 additions and 818 deletions.
2 changes: 1 addition & 1 deletion nimbus/db/accounts_cache.nim
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ proc beginSavepoint*(ac: var AccountsCache): SavePoint {.gcsafe.}
proc rawTrie*(ac: AccountsCache): AccountsTrie = ac.trie

func db(ac: AccountsCache): CoreDbRef = ac.trie.db
func kvt(ac: AccountsCache): CoreDbKvtObj = ac.db.kvt
func kvt(ac: AccountsCache): CoreDbKvtRef = ac.db.kvt

# The AccountsCache is modeled after TrieDatabase for it's transaction style
proc init*(x: typedesc[AccountsCache], db: CoreDbRef,
Expand Down
5 changes: 5 additions & 0 deletions nimbus/db/aristo.nim
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,11 @@ import
export
append, read, serialise

import
aristo/aristo_get
export
getKeyRc

import aristo/aristo_desc/[desc_identifiers, desc_structural]
export
AristoAccount,
Expand Down
25 changes: 13 additions & 12 deletions nimbus/db/aristo/aristo_desc.nim
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ type
# Debugging data below, might go away in future
xMap*: Table[HashLabel,VertexID] ## For pretty printing, extends `pAmk`

AristoDbAction* = proc(db: AristoDbRef) {.gcsafe, raises: [CatchableError].}
AristoDbAction* = proc(db: AristoDbRef) {.gcsafe, raises: [].}
## Generic call back function/closure.

# ------------------------------------------------------------------------------
Expand Down Expand Up @@ -324,19 +324,20 @@ proc forget*(db: AristoDbRef): Result[void,AristoError] =
## A non centre descriptor should always be destructed after use (see also
## comments on `fork()`.)
##
if db.isCentre:
return err(NotAllowedOnCentre)
if not db.isNil:
if db.isCentre:
return err(NotAllowedOnCentre)

# Unlink argument `db`
let parent = db.dudes.rwDb
if parent.dudes.roDudes.len < 2:
parent.dudes = DudesRef(nil)
else:
parent.dudes.roDudes.excl db
parent.dudes.txDudes.excl db # might be empty, anyway
# Unlink argument `db`
let parent = db.dudes.rwDb
if parent.dudes.roDudes.len < 2:
parent.dudes = DudesRef(nil)
else:
parent.dudes.roDudes.excl db
parent.dudes.txDudes.excl db # might be empty, anyway

# Clear descriptor so it would not do harm if used wrongly
db[] = AristoDbObj(top: LayerRef())
# Clear descriptor so it would not do harm if used wrongly
db[] = AristoDbObj(top: LayerRef())
ok()

proc forgetOthers*(db: AristoDbRef): Result[void,AristoError] =
Expand Down
2 changes: 2 additions & 0 deletions nimbus/db/aristo/aristo_desc/desc_error.nim
Original file line number Diff line number Diff line change
Expand Up @@ -245,9 +245,11 @@ type

# Miscelaneous handy helpers
PayloadTypeUnsupported
LeafKeyInvalid
AccountRlpDecodingError
AccountStorageKeyMissing
AccountVtxUnsupported
AccountNodeUnsupported
NotImplemented

# End
27 changes: 25 additions & 2 deletions nimbus/db/aristo/aristo_fetch.nim
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
{.push raises: [].}

import
eth/[common, trie/nibbles],
eth/trie/nibbles,
results,
"."/[aristo_desc, aristo_hike]

Expand All @@ -32,6 +32,13 @@ proc fetchPayloadImpl(
return err((vid, rc.error[1]))
ok rc.value.legs[^1].wp.vtx.lData

proc fetchPayloadImpl(
db: AristoDbRef;
root: VertexID;
path: openArray[byte];
): Result[PayloadRef,(VertexID,AristoError)] =
path.initNibbleRange.hikeUp(root, db).fetchPayloadImpl

# ------------------------------------------------------------------------------
# Public functions
# ------------------------------------------------------------------------------
Expand All @@ -52,7 +59,23 @@ proc fetchPayload*(
): Result[PayloadRef,(VertexID,AristoError)] =
## Variant of `fetchPayload()`
##
path.initNibbleRange.hikeUp(root, db).fetchPayloadImpl
if path.len == 0:
return err((VertexID(0),LeafKeyInvalid))
db.fetchPayloadImpl(root, path)

proc contains*(
db: AristoDbRef; # Database
root: VertexID;
path: openArray[byte]; # Key of database record
): Result[bool,(VertexID,AristoError)] =
## Variant of `fetchPayload()`
##
if path.len == 0:
return err((VertexID(0),LeafKeyInvalid))
let rc = db.fetchPayloadImpl(root, path)
if rc.isOk:
return ok(true)
return ok(false)

# ------------------------------------------------------------------------------
# End
Expand Down
36 changes: 24 additions & 12 deletions nimbus/db/aristo/aristo_get.nim
Original file line number Diff line number Diff line change
Expand Up @@ -137,36 +137,48 @@ proc getLeafVtx*(db: AristoDbRef; lty: LeafTie): VertexRef =

# ------------------

proc getVtx*(db: AristoDbRef; vid: VertexID): VertexRef =
proc getVtxRc*(db: AristoDbRef; vid: VertexID): Result[VertexRef,AristoError] =
## Cascaded attempt to fetch a vertex from the top layer or the backend.
## The function returns `nil` on error or failure.
##
if db.top.sTab.hasKey vid:
# If the vertex is to be deleted on the backend, a `VertexRef(nil)` entry
# is kept in the local table in which case it is OK to return this value.
return db.top.sTab.getOrVoid vid
let rc = db.getVtxBE vid
let vtx = db.top.sTab.getOrVoid vid
if vtx.isValid:
return ok(vtx)
return err(GetVtxNotFound)
db.getVtxBE vid

proc getVtx*(db: AristoDbRef; vid: VertexID): VertexRef =
## Cascaded attempt to fetch a vertex from the top layer or the backend.
## The function returns `nil` on error or failure.
##
let rc = db.getVtxRc vid
if rc.isOk:
return rc.value
VertexRef(nil)

proc getKey*(db: AristoDbRef; vid: VertexID): HashKey =
proc getKeyRc*(db: AristoDbRef; vid: VertexID): Result[HashKey,AristoError] =
## Cascaded attempt to fetch a Merkle hash from the top layer or the backend.
## The function returns `VOID_HASH_KEY` on error or failure.
##
if db.top.kMap.hasKey vid:
# If the key is to be deleted on the backend, a `VOID_HASH_LABEL` entry
# is kept on the local table in which case it is OK to return this value.
return db.top.kMap.getOrVoid(vid).key
let rc = db.getKeyBE vid
let key = db.top.kMap.getOrVoid(vid).key
if key.isValid:
return ok(key)
return err(GetKeyNotFound)
db.getKeyBE vid

proc getKey*(db: AristoDbRef; vid: VertexID): HashKey =
## Cascaded attempt to fetch a vertex from the top layer or the backend.
## The function returns `nil` on error or failure.
##
let rc = db.getKeyRc vid
if rc.isOk:
return rc.value
VOID_HASH_KEY

proc getRootKey*(db: AristoDbRef; vid: VertexID): HashKey =
## Shortcut for `db.getkey VertexID(1)`
db.getKey VertexID(1)

# ------------------------------------------------------------------------------
# End
# ------------------------------------------------------------------------------
20 changes: 16 additions & 4 deletions nimbus/db/aristo/aristo_init/memory_only.nim
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,6 @@ export
MemBackendRef,
QidLayoutRef

let
DefaultQidLayoutRef* = DEFAULT_QID_QUEUES.to(QidLayoutRef)

# ------------------------------------------------------------------------------
# Public helpers
# ------------------------------------------------------------------------------
Expand All @@ -50,10 +47,24 @@ proc kind*(
# Public database constuctors, destructor
# ------------------------------------------------------------------------------

proc init*(
T: type AristoDbRef; # Target type
B: type MemBackendRef; # Backend type
qidLayout: QidLayoutRef; # Optional fifo schedule
): T =
## Memory backend constructor.
##
## If the `qidLayout` argument is set `QidLayoutRef(nil)`, the a backend
## database will not provide filter history management. Providing a different
## scheduler layout shoud be used with care as table access with different
## layouts might render the filter history data unmanageable.
##
when B is MemBackendRef:
AristoDbRef(top: LayerRef(), backend: memoryBackend(qidLayout))

proc init*(
T: type AristoDbRef; # Target type
B: type MemOnlyBackend; # Backend type
qidLayout = DefaultQidLayoutRef; # Optional fifo schedule
): T =
## Memory backend constructor.
##
Expand All @@ -66,6 +77,7 @@ proc init*(
AristoDbRef(top: LayerRef())

elif B is MemBackendRef:
let qidLayout = DEFAULT_QID_QUEUES.to(QidLayoutRef)
AristoDbRef(top: LayerRef(), backend: memoryBackend(qidLayout))

proc init*(
Expand Down
44 changes: 31 additions & 13 deletions nimbus/db/aristo/aristo_init/persistent.nim
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,33 @@ export
RdbBackendRef,
memory_only

# ------------------------------------------------------------------------------
# Private helpers
# ------------------------------------------------------------------------------

proc newAristoRdbDbRef(
basePath: string;
qidLayout: QidLayoutRef;
): Result[AristoDbRef, AristoError]=
let
be = ? rocksDbBackend(basePath, qidLayout)
vGen = block:
let rc = be.getIdgFn()
if rc.isErr:
be.closeFn(flush = false)
return err(rc.error)
rc.value
ok AristoDbRef(top: LayerRef(vGen: vGen), backend: be)

# ------------------------------------------------------------------------------
# Public database constuctors, destructor
# ------------------------------------------------------------------------------

proc init*[W: MemOnlyBackend|RdbBackendRef](
proc init*[W: RdbBackendRef](
T: type AristoDbRef;
B: type W;
basePath: string;
qidLayout = DefaultQidLayoutRef;
qidLayout: QidLayoutRef;
): Result[T, AristoError] =
## Generic constructor, `basePath` argument is ignored for memory backend
## databases (which also unconditionally succeed initialising.)
Expand All @@ -45,17 +63,17 @@ proc init*[W: MemOnlyBackend|RdbBackendRef](
## layouts might render the filter history data unmanageable.
##
when B is RdbBackendRef:
let
be = ? rocksDbBackend(basePath, qidLayout)
vGen = block:
let rc = be.getIdgFn()
if rc.isErr:
be.closeFn(flush = false)
return err(rc.error)
rc.value
ok AristoDbRef(top: LayerRef(vGen: vGen), backend: be)
else:
ok AristoDbRef.init(B, qidLayout)
basePath.newAristoRdbDbRef qidLayout

proc init*[W: RdbBackendRef](
T: type AristoDbRef;
B: type W;
basePath: string;
): Result[T, AristoError] =
## Variant of `init()` using default schedule.
##
when B is RdbBackendRef:
basePath.newAristoRdbDbRef DEFAULT_QID_QUEUES.to(QidLayoutRef)

# ------------------------------------------------------------------------------
# End
Expand Down
4 changes: 2 additions & 2 deletions nimbus/db/aristo/aristo_init/rocks_db/rdb_put.nim
Original file line number Diff line number Diff line change
Expand Up @@ -49,13 +49,13 @@ proc getFileSize(fileName: string): int64 =
defer: f.close
try:
result = f.getFileSize
except:
except CatchableError:
discard

proc rmFileIgnExpt(fileName: string) =
try:
fileName.removeFile
except:
except CatchableError:
discard

# ------------------------------------------------------------------------------
Expand Down
Loading

0 comments on commit 6d13281

Please sign in to comment.