Skip to content

Commit

Permalink
Coordinator: DRY refactor test helpers and data for future reuse (#100)
Browse files Browse the repository at this point in the history
* coordinator: adds slice ByteArrayExtensions.kt

* coordinator: adds testing file system helper

* coordinator: adds testing l1-blob-proof-submission submitter

* coordinator: rename findFile function

* coordinator: move prover responses to common test data for code reuse

* coordinator: fix ByteArray.sliceOf off-by-one bug

* coordinator: remove unnecessary!!

* coordinator: fix gradle deps

* coordinator: fix test

* coordinator: reduce transitive dependencies
  • Loading branch information
jpnovais authored Sep 26, 2024
1 parent c37c904 commit 1456389
Show file tree
Hide file tree
Showing 46 changed files with 332 additions and 151 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,4 @@ dependencies {
testImplementation "io.vertx:vertx-junit5"
testImplementation "tech.pegasys.teku.internal:spec:${libs.versions.teku.get()}"
testImplementation "tech.pegasys.teku.internal:spec:${libs.versions.teku.get()}:test-fixtures"

testFixturesImplementation(testFixtures(project(":coordinator:persistence:db")))
}
1 change: 1 addition & 0 deletions coordinator/ethereum/blob-submitter/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ dependencies {
}
implementation project(":jvm-libs:teku-execution-client")

testImplementation(project(":jvm-libs:testing:l1-blob-and-proof-submission"))
testImplementation(project(":coordinator:persistence:aggregation"))
testImplementation(testFixtures(project(":coordinator:persistence:db")))
testImplementation(testFixtures(project(":coordinator:ethereum:gas-pricing")))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,19 +6,14 @@ import io.vertx.junit5.VertxExtension
import io.vertx.junit5.VertxTestContext
import net.consensys.FakeFixedClock
import net.consensys.linea.ethereum.gaspricing.FakeGasPriceCapProvider
import net.consensys.zkevm.coordinator.clients.prover.serialization.BlobCompressionProofJsonResponse
import net.consensys.zkevm.coordinator.clients.prover.serialization.ProofToFinalizeJsonResponse
import net.consensys.linea.testing.submission.loadBlobsAndAggregations
import net.consensys.zkevm.coordinator.clients.smartcontract.LineaContractVersion
import net.consensys.zkevm.coordinator.clients.smartcontract.LineaRollupSmartContractClient
import net.consensys.zkevm.domain.Aggregation
import net.consensys.zkevm.domain.BlobRecord
import net.consensys.zkevm.domain.Constants.LINEA_BLOCK_INTERVAL
import net.consensys.zkevm.domain.createAggregation
import net.consensys.zkevm.domain.createBlobRecords
import net.consensys.zkevm.ethereum.Account
import net.consensys.zkevm.ethereum.ContractsManager
import net.consensys.zkevm.ethereum.MakeFileDelegatedContractsManager
import net.consensys.zkevm.ethereum.findFile
import net.consensys.zkevm.ethereum.submission.BlobSubmissionCoordinator
import net.consensys.zkevm.ethereum.submission.L1ShnarfBasedAlreadySubmittedBlobsFilter
import net.consensys.zkevm.persistence.AggregationsRepository
Expand All @@ -34,7 +29,6 @@ import org.awaitility.Awaitility.waitAtMost
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.extension.ExtendWith
import tech.pegasys.teku.infrastructure.async.SafeFuture
import java.io.File
import java.util.concurrent.TimeUnit
import kotlin.time.Duration.Companion.minutes
import kotlin.time.Duration.Companion.seconds
Expand All @@ -50,7 +44,7 @@ class BlobAndAggregationFinalizationIntTest : CleanDbTestSuiteParallel() {
private lateinit var blobsRepository: BlobsRepository
private lateinit var blobSubmissionCoordinator: BlobSubmissionCoordinator
private lateinit var aggregationFinalizationCoordinator: AggregationFinalizationCoordinator
private val testDataDir = "coordinator/ethereum/blob-submitter/src/integrationTest/test-data"
private val testDataDir = "testdata/coordinator/prover/v2/"

// 1-block-per-blob test data has 3 aggregations: 1..7, 8..14, 15..21.
// We will upgrade the contract in the middle of 2nd aggregation: 12
Expand All @@ -61,13 +55,19 @@ class BlobAndAggregationFinalizationIntTest : CleanDbTestSuiteParallel() {

private fun setupTest(
vertx: Vertx,
smartContractVersion: LineaContractVersion,
overridingTestDataDir: String? = null
smartContractVersion: LineaContractVersion
) {
if (smartContractVersion != LineaContractVersion.V5) {
// V6 with prover V3 is soon comming, so we will need to update/extend this test setup
throw IllegalArgumentException("Only V5 contract version is supported")
}
val rollupDeploymentFuture = ContractsManager.get()
.deployLineaRollup(numberOfOperators = 2, contractVersion = LineaContractVersion.V5)
// load files from FS while smc deploy
loadBlobsAndAggregations(smartContractVersion, overridingTestDataDir)
loadBlobsAndAggregations(
blobsResponsesDir = "$testDataDir/compression/responses",
aggregationsResponsesDir = "$testDataDir/aggregation/responses"
)
.let { (blobs, aggregations) ->
this.blobs = blobs
this.aggregations = aggregations
Expand Down Expand Up @@ -149,10 +149,9 @@ class BlobAndAggregationFinalizationIntTest : CleanDbTestSuiteParallel() {
private fun testSubmission(
vertx: Vertx,
testContext: VertxTestContext,
smartContractVersion: LineaContractVersion,
overridingTestDataDir: String? = null
smartContractVersion: LineaContractVersion
) {
setupTest(vertx, smartContractVersion, overridingTestDataDir)
setupTest(vertx, smartContractVersion)

SafeFuture.allOf(
SafeFuture.collectAll(blobs.map { blobsRepository.saveNewBlob(it) }.stream()),
Expand All @@ -177,58 +176,4 @@ class BlobAndAggregationFinalizationIntTest : CleanDbTestSuiteParallel() {
testContext.completeNow()
}.whenException(testContext::failNow)
}

private fun proverResponsesFromDir(dir: String): List<File> {
return findFile(dir)
.toFile()
.listFiles()
?.filter { it.name.endsWith(".json") }
?: emptyList()
}

private fun <T> loadProverResponses(responsesDir: String, mapper: (String) -> T): List<T> {
return proverResponsesFromDir(responsesDir)
.map { mapper.invoke(it.readText()) }
}

private fun loadAggregations(aggregationsDir: String): List<Aggregation> {
return loadProverResponses(aggregationsDir) {
createAggregation(aggregationProof = ProofToFinalizeJsonResponse.fromJsonString(it).toDomainObject())
}.sortedBy { it.startBlockNumber }
}

private fun loadBlobs(blobsDir: String, aggregations: List<Aggregation>): List<BlobRecord> {
return loadProverResponses(blobsDir) {
BlobCompressionProofJsonResponse.fromJsonString(it).toDomainObject()
}
.let { compressionProofs ->
val firstAggregationBlockTime = aggregations.first().let { agg ->
agg.aggregationProof!!.finalTimestamp
.minus(LINEA_BLOCK_INTERVAL.times((agg.endBlockNumber - agg.startBlockNumber).toInt()))
}
createBlobRecords(
compressionProofs = compressionProofs,
firstBlockStartBlockTime = firstAggregationBlockTime
)
}
.sortedBy { it.startBlockNumber }
}

private fun loadBlobsAndAggregations(
smartContractVersion: LineaContractVersion,
overridingTestDataDir: String? = null
): Pair<List<BlobRecord>, List<Aggregation>> {
val testCaseDataDir = overridingTestDataDir
?: (
testDataDir + if (smartContractVersion == LineaContractVersion.V5) {
"/start-at-v5"
} else {
throw IllegalArgumentException("Unsupported contract version: $smartContractVersion")
}
)

val aggregations = loadAggregations("$testCaseDataDir/prover-aggregation/responses")
val blobs = loadBlobs("$testCaseDataDir/prover-compression/responses", aggregations)
return blobs to aggregations
}
}
1 change: 1 addition & 0 deletions coordinator/ethereum/test-utils/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ dependencies {
implementation(project(":coordinator:clients:smart-contract-client"))
implementation(project(":jvm-libs:web3j-extensions"))
implementation(project(":coordinator:ethereum:common"))
implementation(project(":jvm-libs:testing:file-system"))
implementation("org.web3j:core:${libs.versions.web3j.get()}") {
exclude group: 'org.slf4j', module: 'slf4j-nop'
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.fasterxml.jackson.module.kotlin.readValue
import kotlinx.datetime.Clock
import net.consensys.linea.contract.AsyncFriendlyTransactionManager
import net.consensys.linea.testing.filesystem.getPathTo
import net.consensys.toULong
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger
Expand Down Expand Up @@ -221,12 +222,12 @@ private open class WhaleBasedAccountManager(

object L1AccountManager : AccountManager by WhaleBasedAccountManager(
web3jClient = Web3jClientManager.l1Client,
genesisFile = findFile(System.getProperty("L1_GENESIS", "docker/config/l1-node/el/genesis.json")),
genesisFile = getPathTo(System.getProperty("L1_GENESIS", "docker/config/l1-node/el/genesis.json")),
log = LogManager.getLogger(L1AccountManager::class.java)
)

object L2AccountManager : AccountManager by WhaleBasedAccountManager(
web3jClient = Web3jClientManager.l2Client,
genesisFile = findFile(System.getProperty("L2_GENESIS", "docker/config/linea-local-dev-genesis.json")),
genesisFile = getPathTo(System.getProperty("L2_GENESIS", "docker/config/linea-local-dev-genesis.json")),
log = LogManager.getLogger(L2AccountManager::class.java)
)
Original file line number Diff line number Diff line change
@@ -1,29 +1,16 @@
package net.consensys.zkevm.ethereum

import net.consensys.linea.async.toSafeFuture
import net.consensys.linea.testing.filesystem.getPathTo
import net.consensys.zkevm.coordinator.clients.smartcontract.LineaContractVersion
import org.apache.logging.log4j.LogManager
import tech.pegasys.teku.infrastructure.async.SafeFuture
import java.io.BufferedReader
import java.io.File
import java.io.InputStreamReader
import java.nio.file.Path
import java.nio.file.Paths
import java.util.regex.Matcher
import java.util.regex.Pattern

fun findFile(target: String): Path {
var current = Paths.get("").toAbsolutePath()
while (current != Paths.get("/")) {
val targetFile = current.resolve(target).toFile()
if (targetFile.exists()) {
return targetFile.toPath()
}
current = current.parent
}
throw IllegalStateException("Couldn't find file $target")
}

data class CommandResult(
val exitCode: Int,
val stdOut: List<String>,
Expand All @@ -33,7 +20,7 @@ data class CommandResult(
fun executeCommand(
command: String,
envVars: Map<String, String> = emptyMap(),
executionDir: File = findFile("Makefile").parent.toFile()
executionDir: File = getPathTo("Makefile").parent.toFile()
): SafeFuture<CommandResult> {
val log = LogManager.getLogger("net.consensys.zkevm.ethereum.CommandExecutor")
val processBuilder = ProcessBuilder("/bin/sh", "-c", command)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package net.consensys

fun ByteArray.assertSize(expectedSize: UInt, fieldName: String = ""): ByteArray = apply {
require(size == expectedSize.toInt()) { "$fieldName expected to have $expectedSize bytes, but got $size" }
}

fun ByteArray.assertIs32Bytes(fieldName: String = ""): ByteArray = assertSize(32u, fieldName)

fun ByteArray.assertIs20Bytes(fieldName: String = ""): ByteArray = assertSize(20u, fieldName)

fun ByteArray.setFirstByteToZero(): ByteArray {
this[0] = 0
return this
}

/**
* Slices the ByteArray into sliceSize bytes chunks and returns the sliceNumber-th chunk.
*/
fun ByteArray.sliceOf(
sliceSize: Int,
sliceNumber: Int,
allowIncompleteLastSlice: Boolean = false
): ByteArray {
assert(sliceSize > 0) {
"sliceSize=$sliceSize should be greater than 0"
}

val startIndex = sliceNumber * sliceSize
val endIndex = (sliceNumber * sliceSize + sliceSize - 1)
.let {
if (it >= this.size && allowIncompleteLastSlice) {
this.size - 1
} else {
it
}
}

assert(startIndex <= this.size && endIndex < this.size) {
"slice $startIndex..$endIndex is out of array size=${this.size}"
}

return this.sliceArray(startIndex..endIndex)
}

/**
* Slices the ByteArray into 32 bytes chunks and returns the sliceNumber-th chunk.
*/
fun ByteArray.sliceOf32(sliceNumber: Int): ByteArray {
return this.sliceOf(sliceSize = 32, sliceNumber)
}
Original file line number Diff line number Diff line change
Expand Up @@ -73,16 +73,3 @@ fun <T : Comparable<T>> ClosedRange<T>.toIntervalString(): String {
}
return "[${this.start}..${this.endInclusive}]${size.toInt()}"
}

fun ByteArray.assertSize(expectedSize: UInt, fieldName: String = ""): ByteArray = apply {
require(size == expectedSize.toInt()) { "$fieldName expected to have $expectedSize bytes, but got $size" }
}

fun ByteArray.assertIs32Bytes(fieldName: String = ""): ByteArray = assertSize(32u, fieldName)

fun ByteArray.assertIs20Bytes(fieldName: String = ""): ByteArray = assertSize(20u, fieldName)

fun ByteArray.setFirstByteToZero(): ByteArray {
this[0] = 0
return this
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
package net.consensys

import org.assertj.core.api.Assertions.assertThat
import org.assertj.core.api.Assertions.assertThatThrownBy
import org.junit.jupiter.api.Test
import kotlin.random.Random

class ByteArrayExtensionsTest {
@Test
fun `ByteArray#encodeHex`() {
assertThat(byteArrayOf().encodeHex()).isEqualTo("0x")
assertThat(byteArrayOf(0).encodeHex()).isEqualTo("0x00")
assertThat(byteArrayOf(1).encodeHex()).isEqualTo("0x01")
assertThat(byteArrayOf(0x12, 0x34, 0x56).encodeHex()).isEqualTo("0x123456")
}

@Test
fun `ByteArray#assertSize`() {
assertThatThrownBy {
byteArrayOf(1, 2, 3).assertSize(2u, "shortNumber")
}.isInstanceOf(IllegalArgumentException::class.java)
.hasMessage("shortNumber expected to have 2 bytes, but got 3")

assertThat(byteArrayOf(1, 2, 3).assertSize(3u)).isEqualTo(byteArrayOf(1, 2, 3))
}

@Test
fun `ByteArray#assertIs32Bytes`() {
assertThatThrownBy {
byteArrayOf(1, 2, 3).assertIs32Bytes("hash")
}.isInstanceOf(IllegalArgumentException::class.java)
.hasMessage("hash expected to have 32 bytes, but got 3")
ByteArray(32).assertIs32Bytes()
}

@Test
fun `ByteArray#assertIs20Bytes`() {
assertThatThrownBy {
byteArrayOf(1, 2, 3).assertIs20Bytes("address")
}.isInstanceOf(IllegalArgumentException::class.java)
.hasMessage("address expected to have 20 bytes, but got 3")
ByteArray(20).assertIs20Bytes()
}

@Test
fun `ByteArray#setFirstByteToZero`() {
assertThat(Random.Default.nextBytes(32).setFirstByteToZero()[0]).isEqualTo(0)
}

@Test
fun `BigInteger#toKWei`() {
assertThat(1_234_000.toBigInteger().toKWei().toUInt()).isEqualTo(1234u)
assertThat(1_234_400.toBigInteger().toKWei().toUInt()).isEqualTo(1234u)
assertThat(1_234_500.toBigInteger().toKWei().toUInt()).isEqualTo(1235u)
assertThat(1_234_600.toBigInteger().toKWei().toUInt()).isEqualTo(1235u)
}

@Test
fun `ByteArray#sliceOf`() {
val bytes = Random.Default.nextBytes(64)
bytes.sliceOf(sliceSize = 5, sliceNumber = 0).also {
assertThat(it).hasSize(5)
assertThat(it).isEqualTo(bytes.sliceArray(0..4))
}
bytes.sliceOf(sliceSize = 10, sliceNumber = 0).also {
assertThat(it).hasSize(10)
assertThat(it).isEqualTo(bytes.sliceArray(0..9))
}
bytes.sliceOf(sliceSize = 10, sliceNumber = 2).also {
assertThat(it).hasSize(10)
assertThat(it).isEqualTo(bytes.sliceArray(20..29))
}
bytes.sliceOf(sliceSize = 1, sliceNumber = 63, allowIncompleteLastSlice = false).also {
assertThat(it).hasSize(1)
assertThat(it).isEqualTo(bytes.sliceArray(63..63))
}
assertThatThrownBy {
bytes.sliceOf(sliceSize = 1, sliceNumber = 64, allowIncompleteLastSlice = false)
}
.isInstanceOf(AssertionError::class.java)
.hasMessage("slice 64..64 is out of array size=64")

bytes.sliceOf(sliceSize = 10, sliceNumber = 6, allowIncompleteLastSlice = true).also {
assertThat(it).hasSize(4)
assertThat(it).isEqualTo(bytes.sliceArray(60..63))
}

assertThatThrownBy {
bytes.sliceOf(sliceSize = 10, sliceNumber = 6, allowIncompleteLastSlice = false)
}
.isInstanceOf(AssertionError::class.java)
.hasMessage("slice 60..69 is out of array size=64")

assertThatThrownBy {
bytes.sliceOf(sliceSize = 10, sliceNumber = 7)
}
.isInstanceOf(AssertionError::class.java)
.hasMessage("slice 70..79 is out of array size=64")
}

@Test
fun `ByteArray#sliceOf32Bytes`() {
val bytes = Random.Default.nextBytes(64)
assertThat(bytes.sliceOf32(0)).isEqualTo(bytes.sliceArray(0..31))
assertThat(bytes.sliceOf32(1)).isEqualTo(bytes.sliceArray(32..63))
assertThatThrownBy {
Random.Default.nextBytes(64 + 16).sliceOf32(sliceNumber = 2)
}
.isInstanceOf(AssertionError::class.java)
.hasMessage("slice 64..95 is out of array size=80")
}
}
Loading

0 comments on commit 1456389

Please sign in to comment.