Merge pull request #7 from BCWResearch/exploratory

Commander Crate (modified leader web-server)

.env

@@ -6,3 +6,12 @@ KECCAK_CIRCUIT_SIZE=14..20
 KECCAK_SPONGE_CIRCUIT_SIZE=9..15
 LOGIC_CIRCUIT_SIZE=12..18
 MEMORY_CIRCUIT_SIZE=17..28
+# LOG
+# RUST_LOG=persistent-leader=debug
+# Persistent-Leader specific
+PALADIN_SERIALIZER=POSTCARD # POSTCARD/CMOP
+PALADIN_RUNTIME=MEMORY # MEMORY/AMQP
+PALADIN_AMQP_NUM_WORKERS=4 # ONLY WHEN RUNTIME IS "MEMORY"
+# SERVER_ADDR_ENVKEY=127.0.0.1:8080 # Defaults to "127.0.0.1:8080"
+# PROOF_OUT_LOCAL_DIR_ENVKEY=proofs_out/ 
+# LOCAL_BENCHMARK_DIRPATH = "benchmark_out/" 

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members = ["leader", "worker", "common", "ops", "verifier", "rpc", "prover", "gascalc"]
+members = ["leader", "worker", "common", "ops", "verifier", "rpc", "prover", "coordinator", "gascalc"]
 resolver = "2"
 
 [workspace.dependencies]
@@ -23,6 +23,11 @@ evm_arithmetization = "0.1.2"
 trace_decoder = "0.2.0"
 proof_gen = "0.1.2"
 
+# GCP benchmark test dependencies
+google-cloud-storage = "0.16.0"
+google-cloud-googleapis = "0.12.0"
+google-cloud-auth = "0.13.2"
+
 [workspace.package]
 edition = "2021"
 license = "MIT OR Apache-2.0"

coordinator/Cargo.toml

@@ -0,0 +1,34 @@
+[package]
+name = "coordinator"
+version = "0.1.0"
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+authors.workspace = true
+keywords.workspace = true
+categories.workspace = true
+
+[dependencies]
+
+# Local dependencies
+ops = { path = "../ops" }
+prover = { path = "../prover" }
+rpc = { path = "../rpc" }
+common = { path = "../common" }
+leader = { path = "../leader" }
+
+# 3rd parties
+tokio = { workspace = true }
+anyhow = { workspace = true }
+clap = { workspace = true }
+dotenvy = { workspace = true }
+paladin-core = { workspace = true }
+proof_gen = { workspace = true }
+google-cloud-storage = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+log = "0.4.21"
+actix-web = "4.5.1"
+actix-rt = "2.9.0"
+env_logger = "0.11.3"
+

coordinator/README.md

@@ -0,0 +1,59 @@
+# Coordinator
+
+Coordinator serves as modified Leader for evaluating multiple blocks.  Coordinator is a persistent webserver used to start the proving process blocks while recording the proofs along with benchmark statistics.
+
+## Requests
+
+To start the benchmarking process, you need to send a post request to the running endpoint.  It accepts the data formatted as a json.
+
+### Fields
+
+Subject to change, if any issues review the structs in the `input` module.
+
+#### Required fields
+
+- `start_block_number`: the first block to be included
+- ``
+
+#### Optional Fields
+
+- `checkpoint_block_number`: The checkpoint block number, otherwise will be 0
+- `terminate_on`: The conditions for termination.
+- `proof_out`: If not provided, will not output the proofs.  
+
+#### Terminate On
+
+TODO: Describe the Termination settings
+
+#### Proof Output
+
+TODO: Describe the Proof Output settings
+
+#### Benchmark Output
+
+TODO: Describe the Benchmark Output settings.
+
+### Examples
+
+The example below proves blocks [1,10] using the RPC function listed in ZeroBin, outputs the proofs to a local directory where each proof will have a prefix of "test" (i.e. "test_1" for block 1, "test_2" for block 2, ...), and output the benchmark statistics locally to "test.csv".  The directories in which these files appear are established by the local environment.
+
+```json
+{
+  "start_block_number": 1,
+  "checkpoint_block_number": 1,
+  "terminate_on": {
+    "EndBlock": {"block_number": 10}
+  },
+  "block_source": {
+    "ZeroBinRpc": {"rpc_url": "http://35.208.84.178:8545/"}
+  },
+  "check_gas": false,
+  "proof_output": {
+    "LocalDirectory": {"prefix": "test"}
+  },
+  "benchmark_output": {
+    "LocalCsv": {"file_name": "test.csv"}
+  }
+}
+
+```

coordinator/benchmark_out/test.csv

@@ -0,0 +1,5 @@
+block_number, number_txs, fetch_duration, proof_duration, gas_used
+2, 0, 0.242030699, 39.143463741, 
+3, 0, 0.290185162, 35.751815781, 
+4, 0, 0.247933683, 35.727297472, 
+5, 0, 0.353537149, 36.226769061, 

coordinator/src/README.md

@@ -0,0 +1,10 @@
+
+
+# Module Descsriptions
+
+- benchmarking -> Contains the necessities of storing and outputting the benchmark statistics in the app-layer gathered during the proving process
+- fetch -> Our means of "fetching" blocks as prover_input
+- input -> Some data structures used for the input to the http server endpoint.  Note, some of the structures are distributed to other modules if they better relate to that module.
+- manyprover -> Contains the function that runs the proving process for many blocks.  Most of the logic happens here.
+- proofout -> Contains the necessities of outputting the proofs.
+- psm -> Creates a ProverStateManager from the env, should only really be necessary when operating InMemory rather than using AMQP