From 021f10e38ea5e526f05e877b045657c7494f884f Mon Sep 17 00:00:00 2001 From: kstroobants Date: Tue, 22 Oct 2024 15:24:17 +0100 Subject: [PATCH 01/35] feat: add rollup transaction db table --- .../1ecaa2085aec_rollup_transactions.py | 40 +++++++++++++++++++ backend/database_handler/models.py | 34 +++++++++++----- .../transactions_processor.py | 36 ++++++++++++++--- 3 files changed, 96 insertions(+), 14 deletions(-) create mode 100644 backend/database_handler/migration/versions/1ecaa2085aec_rollup_transactions.py diff --git a/backend/database_handler/migration/versions/1ecaa2085aec_rollup_transactions.py b/backend/database_handler/migration/versions/1ecaa2085aec_rollup_transactions.py new file mode 100644 index 000000000..9558822b8 --- /dev/null +++ b/backend/database_handler/migration/versions/1ecaa2085aec_rollup_transactions.py @@ -0,0 +1,40 @@ +"""rollup_transactions + +Revision ID: 1ecaa2085aec +Revises: ab256b41602a +Create Date: 2024-10-22 15:12:02.316347 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '1ecaa2085aec' +down_revision: Union[str, None] = 'ab256b41602a' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('rollup_transactions', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('from_', sa.String(length=255), nullable=False), + sa.Column('to_', sa.String(length=255), nullable=True), + sa.Column('gas', sa.Integer(), nullable=False), + sa.Column('gas_price', sa.Integer(), nullable=False), + sa.Column('value', sa.Integer(), nullable=True), + sa.Column('input', sa.Text(), nullable=False), + sa.Column('nonce', sa.BigInteger(), server_default=sa.text('(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) * 1000)::BIGINT'), nullable=False), + sa.PrimaryKeyConstraint('id', name='rollup_transactions_pkey') + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('rollup_transactions') + # ### end Alembic commands ### diff --git a/backend/database_handler/models.py b/backend/database_handler/models.py index 73d640685..c5085bec3 100644 --- a/backend/database_handler/models.py +++ b/backend/database_handler/models.py @@ -13,6 +13,7 @@ func, text, ForeignKey, + Text ) from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import ( @@ -117,18 +118,33 @@ class Transactions(Base): ) -class TransactionsAudit(Base): - __tablename__ = "transactions_audit" - __table_args__ = (PrimaryKeyConstraint("id", name="transactions_audit_pkey"),) +class RollupTransactions(Base): + __tablename__ = "rollup_transactions" + __table_args__ = (PrimaryKeyConstraint("id", name="rollup_transactions_pkey"),) id: Mapped[int] = mapped_column(Integer, primary_key=True, init=False) - transaction_hash: Mapped[Optional[str]] = mapped_column( - String(66), - ForeignKey("transactions.hash", ondelete="CASCADE"), + from_: Mapped[str] = mapped_column( + String(255), ) - data: Mapped[Optional[dict]] = mapped_column(JSONB) - created_at: Mapped[Optional[datetime.datetime]] = mapped_column( - DateTime(True), server_default=func.current_timestamp(), init=False + to_: Mapped[Optional[dict]] = mapped_column( + String(255), + ) + gas: Mapped[int] = mapped_column( + Integer, + ) + gas_price: Mapped[int] = mapped_column( + Integer, + ) + value: Mapped[Optional[int]] = mapped_column( + Integer, + ) + input: Mapped[str] = mapped_column( + Text, + ) + nonce: Mapped[int] = mapped_column( + BigInteger, + server_default=text("(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) * 1000)::BIGINT"), + init=False ) diff --git a/backend/database_handler/transactions_processor.py b/backend/database_handler/transactions_processor.py index faa1a76bc..b03b0e5a0 100644 --- a/backend/database_handler/transactions_processor.py +++ b/backend/database_handler/transactions_processor.py @@ -2,7 +2,7 @@ from enum import Enum import rlp -from .models import Transactions, TransactionsAudit +from .models import Transactions, RollupTransactions from sqlalchemy.orm import Session from sqlalchemy import or_, and_ @@ -158,12 +158,16 @@ def insert_transaction( self.session.flush() # So that `created_at` gets set - transaction_audit_record = TransactionsAudit( - transaction_hash=new_transaction.hash, - data=self._parse_transaction_data(new_transaction), + rollup_transaction_record = RollupTransactions( + from_=from_address, + to_=to_address, + gas=0, + gas_price=0, + value=value, + input=self._transaction_data_to_str(self._parse_transaction_data(new_transaction)), ) - self.session.add(transaction_audit_record) + self.session.add(rollup_transaction_record) return new_transaction.hash @@ -189,6 +193,17 @@ def update_transaction_status( transaction.status = new_status + rollup_transaction_record = RollupTransactions( + from_=transaction.from_address, + to_=transaction.to_address, + gas=0, + gas_price=0, + value=transaction.value, + input=self._transaction_data_to_str(self._parse_transaction_data(transaction)), + ) + + self.session.add(rollup_transaction_record) + def set_transaction_result(self, transaction_hash: str, consensus_data: dict): transaction = ( self.session.query(Transactions).filter_by(hash=transaction_hash).one() @@ -203,6 +218,17 @@ def set_transaction_result(self, transaction_hash: str, consensus_data: dict): TransactionStatus.FINALIZED.value, ) + rollup_transaction_record = RollupTransactions( + from_=transaction.from_address, + to_=transaction.to_address, + gas=0, + gas_price=0, + value=transaction.value, + input=self._transaction_data_to_str(self._parse_transaction_data(transaction)), + ) + + self.session.add(rollup_transaction_record) + def get_transaction_count(self, address: str) -> int: count = ( self.session.query(Transactions) From eca17e5fbf511c790d2025babe11cdf8c3bec6b7 Mon Sep 17 00:00:00 2001 From: kstroobants Date: Wed, 23 Oct 2024 19:41:02 +0100 Subject: [PATCH 02/35] fix: drop audit table, change id and nonce of rollup, only make one finalized rollup transaction --- backend/consensus/base.py | 12 ++--- .../1ecaa2085aec_rollup_transactions.py | 17 ++++-- backend/database_handler/models.py | 10 ++-- .../transactions_processor.py | 52 +++++++++++++------ 4 files changed, 63 insertions(+), 28 deletions(-) diff --git a/backend/consensus/base.py b/backend/consensus/base.py index 3886cbd38..e8d5d6fe3 100644 --- a/backend/consensus/base.py +++ b/backend/consensus/base.py @@ -334,6 +334,12 @@ async def exec_transaction( else: contract_snapshot.update_contract_state(leader_receipt.contract_state) + # Finalize transaction + consensus_data["final"] = True + transactions_processor.set_transaction_result( + transaction.hash, + consensus_data, + ) ConsensusAlgorithm.dispatch_transaction_status_update( transactions_processor, transaction.hash, @@ -341,12 +347,6 @@ async def exec_transaction( msg_handler, ) - # Finalize transaction - transactions_processor.set_transaction_result( - transaction.hash, - consensus_data, - ) - # Insert pending transactions generated by contract-to-contract calls pending_transactions_to_insert = leader_receipt.pending_transactions for pending_transaction in pending_transactions_to_insert: diff --git a/backend/database_handler/migration/versions/1ecaa2085aec_rollup_transactions.py b/backend/database_handler/migration/versions/1ecaa2085aec_rollup_transactions.py index 9558822b8..abf78970f 100644 --- a/backend/database_handler/migration/versions/1ecaa2085aec_rollup_transactions.py +++ b/backend/database_handler/migration/versions/1ecaa2085aec_rollup_transactions.py @@ -9,6 +9,7 @@ from alembic import op import sqlalchemy as sa +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. @@ -21,20 +22,30 @@ def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.create_table('rollup_transactions', - sa.Column('id', sa.Integer(), nullable=False), + sa.Column('transaction_hash', sa.String(length=66), nullable=False), sa.Column('from_', sa.String(length=255), nullable=False), sa.Column('to_', sa.String(length=255), nullable=True), sa.Column('gas', sa.Integer(), nullable=False), sa.Column('gas_price', sa.Integer(), nullable=False), sa.Column('value', sa.Integer(), nullable=True), sa.Column('input', sa.Text(), nullable=False), - sa.Column('nonce', sa.BigInteger(), server_default=sa.text('(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) * 1000)::BIGINT'), nullable=False), - sa.PrimaryKeyConstraint('id', name='rollup_transactions_pkey') + sa.Column('nonce', sa.BigInteger(), nullable=False), + sa.PrimaryKeyConstraint('transaction_hash', name='rollup_transactions_pkey'), + sa.UniqueConstraint('transaction_hash') ) + op.drop_table('transactions_audit') # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### + op.create_table('transactions_audit', + sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column('data', postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=True), + sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('CURRENT_TIMESTAMP'), autoincrement=False, nullable=True), + sa.Column('transaction_hash', sa.VARCHAR(length=66), autoincrement=False, nullable=True), + sa.ForeignKeyConstraint(['transaction_hash'], ['transactions.hash'], name='transaction_hash_fkey', ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name='transactions_audit_pkey') + ) op.drop_table('rollup_transactions') # ### end Alembic commands ### diff --git a/backend/database_handler/models.py b/backend/database_handler/models.py index c5085bec3..b795268ef 100644 --- a/backend/database_handler/models.py +++ b/backend/database_handler/models.py @@ -120,9 +120,13 @@ class Transactions(Base): class RollupTransactions(Base): __tablename__ = "rollup_transactions" - __table_args__ = (PrimaryKeyConstraint("id", name="rollup_transactions_pkey"),) + __table_args__ = (PrimaryKeyConstraint("transaction_hash", name="rollup_transactions_pkey"),) - id: Mapped[int] = mapped_column(Integer, primary_key=True, init=False) + transaction_hash: Mapped[str] = mapped_column( + String(66), + primary_key=True, + unique=True + ) from_: Mapped[str] = mapped_column( String(255), ) @@ -143,8 +147,6 @@ class RollupTransactions(Base): ) nonce: Mapped[int] = mapped_column( BigInteger, - server_default=text("(EXTRACT(EPOCH FROM CURRENT_TIMESTAMP) * 1000)::BIGINT"), - init=False ) diff --git a/backend/database_handler/transactions_processor.py b/backend/database_handler/transactions_processor.py index b03b0e5a0..59841fb53 100644 --- a/backend/database_handler/transactions_processor.py +++ b/backend/database_handler/transactions_processor.py @@ -10,6 +10,7 @@ from eth_utils import to_bytes, keccak, is_address import json import base64 +import time class TransactionAddressFilter(Enum): @@ -158,15 +159,21 @@ def insert_transaction( self.session.flush() # So that `created_at` gets set + rollup_input_data = self._transaction_data_to_str(self._parse_transaction_data(new_transaction)) + rollup_nonce = int(time.time() * 1000) + rollup_transaction_hash = self._generate_transaction_hash( + from_address, to_address, rollup_input_data, value, 0, rollup_nonce + ) rollup_transaction_record = RollupTransactions( + transaction_hash=rollup_transaction_hash, from_=from_address, to_=to_address, gas=0, gas_price=0, value=value, - input=self._transaction_data_to_str(self._parse_transaction_data(new_transaction)), + input=rollup_input_data, + nonce=rollup_nonce ) - self.session.add(rollup_transaction_record) return new_transaction.hash @@ -191,18 +198,27 @@ def update_transaction_status( self.session.query(Transactions).filter_by(hash=transaction_hash).one() ) - transaction.status = new_status + # Do not make a rollup transaction when transaction is already finalized. + # This is done in set_transaction_result() for exec_transaction(). + if (transaction.status != TransactionStatus.FINALIZED): + transaction.status = new_status - rollup_transaction_record = RollupTransactions( - from_=transaction.from_address, - to_=transaction.to_address, - gas=0, - gas_price=0, - value=transaction.value, - input=self._transaction_data_to_str(self._parse_transaction_data(transaction)), - ) - - self.session.add(rollup_transaction_record) + rollup_input_data = self._transaction_data_to_str(self._parse_transaction_data(transaction)) + rollup_nonce = int(time.time() * 1000) + rollup_transaction_hash = self._generate_transaction_hash( + transaction.from_address, transaction.to_address, rollup_input_data, transaction.value, 0, rollup_nonce + ) + rollup_transaction_record = RollupTransactions( + transaction_hash=rollup_transaction_hash, + from_=transaction.from_address, + to_=transaction.to_address, + gas=0, + gas_price=0, + value=transaction.value, + input=rollup_input_data, + nonce=rollup_nonce + ) + self.session.add(rollup_transaction_record) def set_transaction_result(self, transaction_hash: str, consensus_data: dict): transaction = ( @@ -218,15 +234,21 @@ def set_transaction_result(self, transaction_hash: str, consensus_data: dict): TransactionStatus.FINALIZED.value, ) + rollup_input_data = self._transaction_data_to_str(self._parse_transaction_data(transaction)) + rollup_nonce = int(time.time() * 1000) + rollup_transaction_hash = self._generate_transaction_hash( + transaction.from_address, transaction.to_address, rollup_input_data, transaction.value, 0, rollup_nonce + ) rollup_transaction_record = RollupTransactions( + transaction_hash=rollup_transaction_hash, from_=transaction.from_address, to_=transaction.to_address, gas=0, gas_price=0, value=transaction.value, - input=self._transaction_data_to_str(self._parse_transaction_data(transaction)), + input=rollup_input_data, + nonce=rollup_nonce ) - self.session.add(rollup_transaction_record) def get_transaction_count(self, address: str) -> int: From 9184749588a83826a907cb1c29dc2f3ff4b42c23 Mon Sep 17 00:00:00 2001 From: kstroobants Date: Wed, 6 Nov 2024 18:39:16 +0000 Subject: [PATCH 03/35] create rollup transaction for every validator --- backend/consensus/base.py | 57 ++++++++++++---- .../1ecaa2085aec_rollup_transactions.py | 68 +++++++++++++------ backend/database_handler/models.py | 10 +-- .../transactions_processor.py | 64 ++++------------- backend/database_handler/types.py | 6 +- .../transactions_processor_test.py | 5 ++ 6 files changed, 118 insertions(+), 92 deletions(-) diff --git a/backend/consensus/base.py b/backend/consensus/base.py index e8d5d6fe3..19865fd05 100644 --- a/backend/consensus/base.py +++ b/backend/consensus/base.py @@ -187,6 +187,16 @@ async def exec_transaction( ) for validators in rotate(involved_validators): + consensus_data = ConsensusData( + final=False, + votes={}, + leader_receipt=None, + validators=[], + ) + transactions_processor.set_transaction_result( + transaction.hash, + consensus_data.to_dict(), + ) # Update transaction status ConsensusAlgorithm.dispatch_transaction_status_update( transactions_processor, @@ -194,6 +204,7 @@ async def exec_transaction( TransactionStatus.PROPOSING, msg_handler, ) + transactions_processor.create_rollup_transaction(transaction.hash) [leader, *remaining_validators] = validators @@ -217,6 +228,12 @@ async def exec_transaction( # Leader executes transaction leader_receipt = await leader_node.exec_transaction(transaction) votes = {leader["address"]: leader_receipt.vote.value} + consensus_data.votes = votes + consensus_data.leader_receipt = leader_receipt + transactions_processor.set_transaction_result( + transaction.hash, + consensus_data.to_dict(), + ) # Update transaction status ConsensusAlgorithm.dispatch_transaction_status_update( transactions_processor, @@ -224,6 +241,7 @@ async def exec_transaction( TransactionStatus.COMMITTING, msg_handler, ) + transactions_processor.create_rollup_transaction(transaction.hash) # Create Validators validator_nodes = [ @@ -247,8 +265,6 @@ async def exec_transaction( ] validation_results = await asyncio.gather(*validation_tasks) - for i, validation_result in enumerate(validation_results): - votes[validator_nodes[i].address] = validation_result.vote.value ConsensusAlgorithm.dispatch_transaction_status_update( transactions_processor, transaction.hash, @@ -256,6 +272,20 @@ async def exec_transaction( msg_handler, ) + for i, validation_result in enumerate(validation_results): + votes[validator_nodes[i].address] = validation_result.vote.value + single_reveal_votes = { + leader["address"]: leader_receipt.vote.value, + validator_nodes[i].address: validation_result.vote.value, + } + consensus_data.votes = single_reveal_votes + consensus_data.validators = [validation_result] + transactions_processor.set_transaction_result( + transaction.hash, + consensus_data.to_dict(), + ) + transactions_processor.create_rollup_transaction(transaction.hash) + if ( len([vote for vote in votes.values() if vote == Vote.AGREE.value]) >= num_validators // 2 @@ -290,22 +320,20 @@ async def exec_transaction( TransactionStatus.ACCEPTED, msg_handler, ) - - final = False - consensus_data = ConsensusData( - final=final, - votes=votes, - leader_receipt=leader_receipt, - validators=validation_results, - ).to_dict() - + consensus_data.votes = votes + consensus_data.validators = validation_results + transactions_processor.set_transaction_result( + transaction.hash, + consensus_data.to_dict(), + ) + transactions_processor.create_rollup_transaction(transaction.hash) msg_handler.send_message( LogEvent( "consensus_reached", EventType.SUCCESS, EventScope.CONSENSUS, "Reached consensus", - consensus_data, + consensus_data.to_dict(), ) ) @@ -335,10 +363,10 @@ async def exec_transaction( contract_snapshot.update_contract_state(leader_receipt.contract_state) # Finalize transaction - consensus_data["final"] = True + consensus_data.final = True transactions_processor.set_transaction_result( transaction.hash, - consensus_data, + consensus_data.to_dict(), ) ConsensusAlgorithm.dispatch_transaction_status_update( transactions_processor, @@ -346,6 +374,7 @@ async def exec_transaction( TransactionStatus.FINALIZED, msg_handler, ) + transactions_processor.create_rollup_transaction(transaction.hash) # Insert pending transactions generated by contract-to-contract calls pending_transactions_to_insert = leader_receipt.pending_transactions diff --git a/backend/database_handler/migration/versions/1ecaa2085aec_rollup_transactions.py b/backend/database_handler/migration/versions/1ecaa2085aec_rollup_transactions.py index abf78970f..473ad16a9 100644 --- a/backend/database_handler/migration/versions/1ecaa2085aec_rollup_transactions.py +++ b/backend/database_handler/migration/versions/1ecaa2085aec_rollup_transactions.py @@ -5,6 +5,7 @@ Create Date: 2024-10-22 15:12:02.316347 """ + from typing import Sequence, Union from alembic import op @@ -13,39 +14,62 @@ # revision identifiers, used by Alembic. -revision: str = '1ecaa2085aec' -down_revision: Union[str, None] = 'ab256b41602a' +revision: str = "1ecaa2085aec" +down_revision: Union[str, None] = "ab256b41602a" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.create_table('rollup_transactions', - sa.Column('transaction_hash', sa.String(length=66), nullable=False), - sa.Column('from_', sa.String(length=255), nullable=False), - sa.Column('to_', sa.String(length=255), nullable=True), - sa.Column('gas', sa.Integer(), nullable=False), - sa.Column('gas_price', sa.Integer(), nullable=False), - sa.Column('value', sa.Integer(), nullable=True), - sa.Column('input', sa.Text(), nullable=False), - sa.Column('nonce', sa.BigInteger(), nullable=False), - sa.PrimaryKeyConstraint('transaction_hash', name='rollup_transactions_pkey'), - sa.UniqueConstraint('transaction_hash') + op.create_table( + "rollup_transactions", + sa.Column("transaction_hash", sa.String(length=66), nullable=False), + sa.Column("from_", sa.String(length=255), nullable=False), + sa.Column("to_", sa.String(length=255), nullable=True), + sa.Column("gas", sa.Integer(), nullable=False), + sa.Column("gas_price", sa.Integer(), nullable=False), + sa.Column("value", sa.Integer(), nullable=True), + sa.Column("input", sa.Text(), nullable=False), + sa.Column("nonce", sa.BigInteger(), nullable=False), + sa.PrimaryKeyConstraint("transaction_hash", name="rollup_transactions_pkey"), + sa.UniqueConstraint("transaction_hash"), ) - op.drop_table('transactions_audit') + op.drop_table("transactions_audit") # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.create_table('transactions_audit', - sa.Column('id', sa.INTEGER(), autoincrement=True, nullable=False), - sa.Column('data', postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=True), - sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('CURRENT_TIMESTAMP'), autoincrement=False, nullable=True), - sa.Column('transaction_hash', sa.VARCHAR(length=66), autoincrement=False, nullable=True), - sa.ForeignKeyConstraint(['transaction_hash'], ['transactions.hash'], name='transaction_hash_fkey', ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id', name='transactions_audit_pkey') + op.create_table( + "transactions_audit", + sa.Column("id", sa.INTEGER(), autoincrement=True, nullable=False), + sa.Column( + "data", + postgresql.JSONB(astext_type=sa.Text()), + autoincrement=False, + nullable=True, + ), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("CURRENT_TIMESTAMP"), + autoincrement=False, + nullable=True, + ), + sa.Column( + "transaction_hash", + sa.VARCHAR(length=66), + autoincrement=False, + nullable=True, + ), + sa.ForeignKeyConstraint( + ["transaction_hash"], + ["transactions.hash"], + name="transaction_hash_fkey", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name="transactions_audit_pkey"), ) - op.drop_table('rollup_transactions') + op.drop_table("rollup_transactions") # ### end Alembic commands ### diff --git a/backend/database_handler/models.py b/backend/database_handler/models.py index b795268ef..b33d864a2 100644 --- a/backend/database_handler/models.py +++ b/backend/database_handler/models.py @@ -13,7 +13,7 @@ func, text, ForeignKey, - Text + Text, ) from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import ( @@ -120,12 +120,12 @@ class Transactions(Base): class RollupTransactions(Base): __tablename__ = "rollup_transactions" - __table_args__ = (PrimaryKeyConstraint("transaction_hash", name="rollup_transactions_pkey"),) + __table_args__ = ( + PrimaryKeyConstraint("transaction_hash", name="rollup_transactions_pkey"), + ) transaction_hash: Mapped[str] = mapped_column( - String(66), - primary_key=True, - unique=True + String(66), primary_key=True, unique=True ) from_: Mapped[str] = mapped_column( String(255), diff --git a/backend/database_handler/transactions_processor.py b/backend/database_handler/transactions_processor.py index 59841fb53..f7db9c5b6 100644 --- a/backend/database_handler/transactions_processor.py +++ b/backend/database_handler/transactions_processor.py @@ -159,22 +159,7 @@ def insert_transaction( self.session.flush() # So that `created_at` gets set - rollup_input_data = self._transaction_data_to_str(self._parse_transaction_data(new_transaction)) - rollup_nonce = int(time.time() * 1000) - rollup_transaction_hash = self._generate_transaction_hash( - from_address, to_address, rollup_input_data, value, 0, rollup_nonce - ) - rollup_transaction_record = RollupTransactions( - transaction_hash=rollup_transaction_hash, - from_=from_address, - to_=to_address, - gas=0, - gas_price=0, - value=value, - input=rollup_input_data, - nonce=rollup_nonce - ) - self.session.add(rollup_transaction_record) + self.create_rollup_transaction(new_transaction.hash) return new_transaction.hash @@ -193,51 +178,32 @@ def get_transaction_by_hash(self, transaction_hash: str) -> dict | None: def update_transaction_status( self, transaction_hash: str, new_status: TransactionStatus ): - transaction = ( self.session.query(Transactions).filter_by(hash=transaction_hash).one() ) - - # Do not make a rollup transaction when transaction is already finalized. - # This is done in set_transaction_result() for exec_transaction(). - if (transaction.status != TransactionStatus.FINALIZED): - transaction.status = new_status - - rollup_input_data = self._transaction_data_to_str(self._parse_transaction_data(transaction)) - rollup_nonce = int(time.time() * 1000) - rollup_transaction_hash = self._generate_transaction_hash( - transaction.from_address, transaction.to_address, rollup_input_data, transaction.value, 0, rollup_nonce - ) - rollup_transaction_record = RollupTransactions( - transaction_hash=rollup_transaction_hash, - from_=transaction.from_address, - to_=transaction.to_address, - gas=0, - gas_price=0, - value=transaction.value, - input=rollup_input_data, - nonce=rollup_nonce - ) - self.session.add(rollup_transaction_record) + transaction.status = new_status def set_transaction_result(self, transaction_hash: str, consensus_data: dict): transaction = ( self.session.query(Transactions).filter_by(hash=transaction_hash).one() ) - - transaction.status = TransactionStatus.FINALIZED transaction.consensus_data = consensus_data - print( - "Updating transaction status", - transaction_hash, - TransactionStatus.FINALIZED.value, + def create_rollup_transaction(self, transaction_hash: str): + transaction = ( + self.session.query(Transactions).filter_by(hash=transaction_hash).one() + ) + rollup_input_data = self._transaction_data_to_str( + self._parse_transaction_data(transaction) ) - - rollup_input_data = self._transaction_data_to_str(self._parse_transaction_data(transaction)) rollup_nonce = int(time.time() * 1000) rollup_transaction_hash = self._generate_transaction_hash( - transaction.from_address, transaction.to_address, rollup_input_data, transaction.value, 0, rollup_nonce + transaction.from_address, + transaction.to_address, + rollup_input_data, + transaction.value, + 0, + rollup_nonce, ) rollup_transaction_record = RollupTransactions( transaction_hash=rollup_transaction_hash, @@ -247,7 +213,7 @@ def set_transaction_result(self, transaction_hash: str, consensus_data: dict): gas_price=0, value=transaction.value, input=rollup_input_data, - nonce=rollup_nonce + nonce=rollup_nonce, ) self.session.add(rollup_transaction_record) diff --git a/backend/database_handler/types.py b/backend/database_handler/types.py index a8c2b3bd2..6f318fd8e 100644 --- a/backend/database_handler/types.py +++ b/backend/database_handler/types.py @@ -6,13 +6,15 @@ class ConsensusData: final: bool votes: dict[str, str] - leader_receipt: Receipt + leader_receipt: Receipt | None validators: list[Receipt] | None = None def to_dict(self): return { "final": self.final, "votes": self.votes, - "leader_receipt": self.leader_receipt.to_dict(), + "leader_receipt": ( + self.leader_receipt.to_dict() if self.leader_receipt else None + ), "validators": [receipt.to_dict() for receipt in self.validators], } diff --git a/tests/db-sqlalchemy/transactions_processor_test.py b/tests/db-sqlalchemy/transactions_processor_test.py index e959060b2..4c0cf6d51 100644 --- a/tests/db-sqlalchemy/transactions_processor_test.py +++ b/tests/db-sqlalchemy/transactions_processor_test.py @@ -78,6 +78,11 @@ def test_transactions_processor(transactions_processor: TransactionsProcessor): actual_transaction_hash, consensus_data ) + new_status = TransactionStatus.FINALIZED + transactions_processor.update_transaction_status( + actual_transaction_hash, new_status + ) + actual_transaction = transactions_processor.get_transaction_by_hash( actual_transaction_hash ) From 6daee807f91835e7deedc4fb1b8c311ceb660d9e Mon Sep 17 00:00:00 2001 From: kstroobants Date: Wed, 6 Nov 2024 18:51:28 +0000 Subject: [PATCH 04/35] add function to mock in consensus test --- tests/unit/consensus/test_base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/consensus/test_base.py b/tests/unit/consensus/test_base.py index a19f07f5f..0733fcd03 100644 --- a/tests/unit/consensus/test_base.py +++ b/tests/unit/consensus/test_base.py @@ -45,6 +45,9 @@ def set_transaction_result(self, transaction_hash: str, consensus_data: dict): transaction = self.get_transaction_by_hash(transaction_hash) transaction["consensus_data"] = consensus_data + def create_rollup_transaction(self, transaction_hash: str): + pass + class SnapshotMock: def __init__(self, nodes): From 90ea2c1c0dd98e04a2a7cecd8c75c171babc9e71 Mon Sep 17 00:00:00 2001 From: kstroobants Date: Thu, 7 Nov 2024 16:13:00 +0000 Subject: [PATCH 05/35] feat: add appeal window, accepted queue, basic test --- backend/consensus/base.py | 104 ++++++- backend/database_handler/chain_snapshot.py | 18 ++ .../versions/37196a51038e_appeals.py | 37 +++ backend/database_handler/models.py | 2 + .../transactions_processor.py | 22 ++ backend/domain/types.py | 6 + backend/protocol_rpc/endpoints.py | 10 + backend/protocol_rpc/server.py | 4 + .../components/Simulator/TransactionItem.vue | 36 ++- .../Simulator/TransactionStatusBadge.vue | 14 +- frontend/src/services/IJsonRpcService.ts | 1 + frontend/src/services/JsonRpcService.ts | 8 + frontend/src/stores/transactions.ts | 5 + tests/unit/consensus/test_base.py | 275 +++++++++++++++++- 14 files changed, 528 insertions(+), 14 deletions(-) create mode 100644 backend/database_handler/migration/versions/37196a51038e_appeals.py diff --git a/backend/consensus/base.py b/backend/consensus/base.py index 19865fd05..9453ff69c 100644 --- a/backend/consensus/base.py +++ b/backend/consensus/base.py @@ -2,12 +2,15 @@ DEFAULT_VALIDATORS_COUNT = 5 DEFAULT_CONSENSUS_SLEEP_TIME = 5 +DEFAULT_FINALITY_WINDOW = 30 * 60 # 30 minutes import asyncio from collections import deque import json import traceback from typing import Callable, Iterator +import time +import base64 from sqlalchemy.orm import Session from backend.consensus.vrf import get_validators_for_transaction @@ -314,6 +317,8 @@ async def exec_transaction( ) return + transactions_processor.set_transaction_timestamp_accepted(transaction.hash) + ConsensusAlgorithm.dispatch_transaction_status_update( transactions_processor, transaction.hash, @@ -337,18 +342,62 @@ async def exec_transaction( ) ) + def commit_reveal_accept_transaction( + self, + transaction: Transaction, + transactions_processor: TransactionsProcessor, + ): + # temporary, reuse existing code + # and add other possible states the transaction can go to + ConsensusAlgorithm.dispatch_transaction_status_update( + transactions_processor, + transaction.hash, + TransactionStatus.COMMITTING, + self.msg_handler, + ) + transactions_processor.create_rollup_transaction(transaction.hash) + + ConsensusAlgorithm.dispatch_transaction_status_update( + transactions_processor, + transaction.hash, + TransactionStatus.REVEALING, + self.msg_handler, + ) + transactions_processor.create_rollup_transaction(transaction.hash) + + time.sleep(2) # remove this + + transactions_processor.set_transaction_timestamp_accepted(transaction.hash) + ConsensusAlgorithm.dispatch_transaction_status_update( + transactions_processor, + transaction.hash, + TransactionStatus.ACCEPTED, + self.msg_handler, + ) + transactions_processor.create_rollup_transaction(transaction.hash) + + def finalize_transaction( + self, + transaction: Transaction, + transactions_processor: TransactionsProcessor, + contract_snapshot_factory: Callable[[str], ContractSnapshot], + ): + consensus_data = transaction.consensus_data + leader_receipt = consensus_data["leader_receipt"] + contract_snapshot = contract_snapshot_factory(transaction.to_address) + # Register contract if it is a new contract if transaction.type == TransactionType.DEPLOY_CONTRACT: new_contract = { "id": transaction.data["contract_address"], "data": { - "state": leader_receipt.contract_state, + "state": leader_receipt["contract_state"], "code": transaction.data["contract_code"], }, } contract_snapshot.register_contract(new_contract) - msg_handler.send_message( + self.msg_handler.send_message( LogEvent( "deployed_contract", EventType.SUCCESS, @@ -360,31 +409,31 @@ async def exec_transaction( # Update contract state if it is an existing contract else: - contract_snapshot.update_contract_state(leader_receipt.contract_state) + contract_snapshot.update_contract_state(leader_receipt["contract_state"]) # Finalize transaction - consensus_data.final = True + consensus_data["final"] = True transactions_processor.set_transaction_result( transaction.hash, - consensus_data.to_dict(), + consensus_data, ) ConsensusAlgorithm.dispatch_transaction_status_update( transactions_processor, transaction.hash, TransactionStatus.FINALIZED, - msg_handler, + self.msg_handler, ) transactions_processor.create_rollup_transaction(transaction.hash) # Insert pending transactions generated by contract-to-contract calls - pending_transactions_to_insert = leader_receipt.pending_transactions + pending_transactions_to_insert = leader_receipt["pending_transactions"] for pending_transaction in pending_transactions_to_insert: nonce = transactions_processor.get_transaction_count(transaction.to_address) transactions_processor.insert_transaction( transaction.to_address, # new calls are done by the contract - pending_transaction.address, + pending_transaction["address"], { - "calldata": pending_transaction.calldata, + "calldata": base64.b64decode(pending_transaction["calldata"]), }, value=0, # we only handle EOA transfers at the moment, so no value gets transferred type=TransactionType.RUN_CONTRACT.value, @@ -474,6 +523,43 @@ def dispatch_transaction_status_update( ) ) + def run_appeal_window_loop(self): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.run_until_complete(self._appeal_window()) + loop.close() + + async def _appeal_window(self): + while True: + with self.get_session() as session: + chain_snapshot = ChainSnapshot(session) + transactions_processor = TransactionsProcessor(session) + accepted_transactions = chain_snapshot.get_accepted_transactions() + for transaction in accepted_transactions: + transaction = transaction_from_dict(transaction) + if not transaction.appeal: + if ( + int(time.time()) - transaction.timestamp_accepted + ) > DEFAULT_FINALITY_WINDOW: + self.finalize_transaction( + transaction, + transactions_processor, + lambda contract_address, session=session: ContractSnapshot( + contract_address, session + ), + ) + session.commit() + else: + transactions_processor.set_transaction_appeal( + transaction.hash, False + ) + self.commit_reveal_accept_transaction( + transaction, transactions_processor + ) + session.commit() + + await asyncio.sleep(1) + def rotate(nodes: list) -> Iterator[list]: nodes = deque(nodes) diff --git a/backend/database_handler/chain_snapshot.py b/backend/database_handler/chain_snapshot.py index 33c2ed134..33847e1fc 100644 --- a/backend/database_handler/chain_snapshot.py +++ b/backend/database_handler/chain_snapshot.py @@ -18,6 +18,7 @@ def __init__(self, session: Session): self.all_validators = self.validators_registry.get_all_validators() self.pending_transactions = self._load_pending_transactions() self.num_validators = len(self.all_validators) + self.accepted_transaction = self._load_accepted_transactions() def _load_pending_transactions(self) -> List[dict]: """Load and return the list of pending transactions from the database.""" @@ -39,3 +40,20 @@ def get_pending_transactions(self): def get_all_validators(self): """Return the list of all validators.""" return self.all_validators + + def _load_accepted_transactions(self) -> List[dict]: + """Load and return the list of accepted transactions from the database.""" + + accepted_transactions = ( + self.session.query(Transactions) + .filter(Transactions.status == TransactionStatus.ACCEPTED) + .all() + ) + return [ + TransactionsProcessor._parse_transaction_data(transaction) + for transaction in accepted_transactions + ] + + def get_accepted_transactions(self): + """Return the list of accepted transactions.""" + return self.accepted_transaction diff --git a/backend/database_handler/migration/versions/37196a51038e_appeals.py b/backend/database_handler/migration/versions/37196a51038e_appeals.py new file mode 100644 index 000000000..54019546b --- /dev/null +++ b/backend/database_handler/migration/versions/37196a51038e_appeals.py @@ -0,0 +1,37 @@ +"""appeals + +Revision ID: 37196a51038e +Revises: 1ecaa2085aec +Create Date: 2024-10-25 17:39:00.130046 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "37196a51038e" +down_revision: Union[str, None] = "1ecaa2085aec" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_unique_constraint(None, "rollup_transactions", ["transaction_hash"]) + op.add_column("transactions", sa.Column("appeal", sa.Boolean(), nullable=False)) + op.add_column( + "transactions", sa.Column("timestamp_accepted", sa.BigInteger(), nullable=True) + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("transactions", "timestamp_accepted") + op.drop_column("transactions", "appeal") + op.drop_constraint(None, "rollup_transactions", type_="unique") + # ### end Alembic commands ### diff --git a/backend/database_handler/models.py b/backend/database_handler/models.py index b33d864a2..a272c4bcb 100644 --- a/backend/database_handler/models.py +++ b/backend/database_handler/models.py @@ -116,6 +116,8 @@ class Transactions(Base): back_populates="triggered_by", init=False, ) + appeal: Mapped[bool] = mapped_column(Boolean, default=False) + timestamp_accepted: Mapped[Optional[int]] = mapped_column(BigInteger, default=None) class RollupTransactions(Base): diff --git a/backend/database_handler/transactions_processor.py b/backend/database_handler/transactions_processor.py index f7db9c5b6..f2ee372a5 100644 --- a/backend/database_handler/transactions_processor.py +++ b/backend/database_handler/transactions_processor.py @@ -50,6 +50,8 @@ def _parse_transaction_data(transaction_data: Transactions) -> dict: transaction.hash for transaction in transaction_data.triggered_transactions ], + "appeal": transaction_data.appeal, + "timestamp_accepted": transaction_data.timestamp_accepted, } @staticmethod @@ -153,6 +155,8 @@ def insert_transaction( if triggered_by_hash else None ), + appeal=False, + timestamp_accepted=None, ) self.session.add(new_transaction) @@ -182,6 +186,7 @@ def update_transaction_status( self.session.query(Transactions).filter_by(hash=transaction_hash).one() ) transaction.status = new_status + self.session.commit() def set_transaction_result(self, transaction_hash: str, consensus_data: dict): transaction = ( @@ -249,3 +254,20 @@ def get_transactions_for_address( return [ self._parse_transaction_data(transaction) for transaction in transactions ] + + def set_transaction_appeal(self, transaction_hash: str, appeal: bool): + transaction = ( + self.session.query(Transactions).filter_by(hash=transaction_hash).one() + ) + transaction.appeal = appeal + + def set_transaction_timestamp_accepted( + self, transaction_hash: str, timestamp_accepted: int = None + ): + transaction = ( + self.session.query(Transactions).filter_by(hash=transaction_hash).one() + ) + if timestamp_accepted: + transaction.timestamp_accepted = timestamp_accepted + else: + transaction.timestamp_accepted = int(time.time()) diff --git a/backend/domain/types.py b/backend/domain/types.py index 6ed2e0dcc..3f69c1e5b 100644 --- a/backend/domain/types.py +++ b/backend/domain/types.py @@ -80,6 +80,8 @@ class Transaction: False # Flag to indicate if this transaction should be processed only by the leader. Used for fast and cheap execution of transactions. ) client_session_id: str | None = None + appeal: bool = False + timestamp_accepted: int | None = None def to_dict(self): return { @@ -99,6 +101,8 @@ def to_dict(self): "v": self.v, "leader_only": self.leader_only, "client_session_id": self.client_session_id, + "appeal": self.appeal, + "timestamp_accepted": self.timestamp_accepted, } @@ -120,4 +124,6 @@ def transaction_from_dict(input: dict) -> Transaction: v=input.get("v"), leader_only=input.get("leader_only", False), client_session_id=input["client_session_id"], + appeal=input.get("appeal"), + timestamp_accepted=input.get("timestamp_accepted"), ) diff --git a/backend/protocol_rpc/endpoints.py b/backend/protocol_rpc/endpoints.py index 6ac1d840f..83e770649 100644 --- a/backend/protocol_rpc/endpoints.py +++ b/backend/protocol_rpc/endpoints.py @@ -505,6 +505,12 @@ def get_transactions_for_address( ) +def set_transaction_appeal( + transactions_processor: TransactionsProcessor, transaction_hash: str +) -> None: + transactions_processor.set_transaction_appeal(transaction_hash, True) + + def register_all_rpc_endpoints( jsonrpc: JSONRPC, msg_handler: MessageHandler, @@ -623,3 +629,7 @@ def register_all_rpc_endpoints( partial(get_transactions_for_address, transactions_processor, accounts_manager), method_name="sim_getTransactionsForAddress", ) + register_rpc_endpoint( + partial(set_transaction_appeal, transactions_processor), + method_name="sim_setTransactionAppeal", + ) diff --git a/backend/protocol_rpc/server.py b/backend/protocol_rpc/server.py index 904564938..037c8d9d6 100644 --- a/backend/protocol_rpc/server.py +++ b/backend/protocol_rpc/server.py @@ -136,3 +136,7 @@ def run_socketio(): # Thread for the run_consensus method thread_consensus = threading.Thread(target=consensus.run_consensus_loop) thread_consensus.start() + +# Thread for the appeal_window method +thread_consensus = threading.Thread(target=consensus.run_appeal_window_loop) +thread_consensus.start() diff --git a/frontend/src/components/Simulator/TransactionItem.vue b/frontend/src/components/Simulator/TransactionItem.vue index 11f17d69f..4503e1916 100644 --- a/frontend/src/components/Simulator/TransactionItem.vue +++ b/frontend/src/components/Simulator/TransactionItem.vue @@ -1,17 +1,19 @@ diff --git a/frontend/src/components/Simulator/HomeTab.vue b/frontend/src/components/Simulator/HomeTab.vue index 7de001339..b432fec0e 100644 --- a/frontend/src/components/Simulator/HomeTab.vue +++ b/frontend/src/components/Simulator/HomeTab.vue @@ -56,7 +56,7 @@ const hasAnySampleContract = computed(() => { class="rounded-md bg-slate-50 p-4 @[1024px]:p-8 dark:bg-slate-600 dark:bg-opacity-10" >

- Welcome to the GenLayer Simulator + Welcome to the GenLayer Studio

@@ -66,7 +66,7 @@ const hasAnySampleContract = computed(() => { natural language.

- This Simulator is an interactive sandbox for developers to explore + This Studio is an interactive sandbox for developers to explore GenLayer’s Intelligent Contracts. It mirrors the GenLayer network’s environment and consensus, allowing you to test ideas locally. @@ -107,7 +107,7 @@ const hasAnySampleContract = computed(() => {

- The simulator currently does not support token transfers, + The Studio currently does not support token transfers, contract-to-contract interactions, or gas consumption. These features will be added in future updates. diff --git a/frontend/src/components/Simulator/TransactionItem.vue b/frontend/src/components/Simulator/TransactionItem.vue index 4503e1916..f55988011 100644 --- a/frontend/src/components/Simulator/TransactionItem.vue +++ b/frontend/src/components/Simulator/TransactionItem.vue @@ -18,7 +18,6 @@ const transactionsStore = useTransactionsStore(); const props = defineProps<{ transaction: TransactionItem; }>(); -console.log('πŸš€ ~ transaction:', props.transaction); const isDetailsModalOpen = ref(false); diff --git a/frontend/src/components/Simulator/ValidatorItem.vue b/frontend/src/components/Simulator/ValidatorItem.vue index ba1d3f9be..07e322944 100644 --- a/frontend/src/components/Simulator/ValidatorItem.vue +++ b/frontend/src/components/Simulator/ValidatorItem.vue @@ -10,8 +10,10 @@ import ValidatorModal from '@/components/Simulator/ValidatorModal.vue'; import { ref } from 'vue'; import { useNodeStore } from '@/stores'; import { notify } from '@kyvg/vue3-notification'; +import { useConfig } from '@/hooks'; const nodeStore = useNodeStore(); +const { canUpdateValidators } = useConfig(); const isUpdateModalMopen = ref(false); const showConfirmDelete = ref(false); @@ -75,7 +77,10 @@ async function handleDeleteValidator() {
-