From bb516d50dd7e5a128fcfe6a6cfc3619bbd73d4c6 Mon Sep 17 00:00:00 2001 From: Shubh Agarwal Date: Thu, 15 Aug 2024 02:00:40 -0400 Subject: [PATCH 01/19] added fees and file for single tx slippage checking --- contracts/gpv2_settlement_abi.py | 389 ++++++++++++++++++ src/constants.py | 2 + src/fees/__init__.py | 0 src/fees/compute_fees.py | 606 +++++++++++++++++++++++++++++ src/helpers/database.py | 28 ++ src/sql/insert_coingecko_price.sql | 4 - src/sql/insert_fee.sql | 4 + src/test_single_hash.py | 76 ++++ src/transaction_processor.py | 66 +++- 9 files changed, 1169 insertions(+), 6 deletions(-) create mode 100644 contracts/gpv2_settlement_abi.py create mode 100644 src/fees/__init__.py create mode 100644 src/fees/compute_fees.py delete mode 100644 src/sql/insert_coingecko_price.sql create mode 100644 src/sql/insert_fee.sql create mode 100644 src/test_single_hash.py diff --git a/contracts/gpv2_settlement_abi.py b/contracts/gpv2_settlement_abi.py new file mode 100644 index 0000000..f09f251 --- /dev/null +++ b/contracts/gpv2_settlement_abi.py @@ -0,0 +1,389 @@ +"""Contract ABI of the CoW Protocol settlement contract +""" + +gpv2_settlement_abi = [ + { + "inputs": [ + { + "internalType": "contract GPv2Authentication", + "name": "authenticator_", + "type": "address", + }, + {"internalType": "contract IVault", "name": "vault_", "type": "address"}, + ], + "stateMutability": "nonpayable", + "type": "constructor", + }, + { + "anonymous": False, + "inputs": [ + { + "indexed": True, + "internalType": "address", + "name": "target", + "type": "address", + }, + { + "indexed": False, + "internalType": "uint256", + "name": "value", + "type": "uint256", + }, + { + "indexed": False, + "internalType": "bytes4", + "name": "selector", + "type": "bytes4", + }, + ], + "name": "Interaction", + "type": "event", + }, + { + "anonymous": False, + "inputs": [ + { + "indexed": True, + "internalType": "address", + "name": "owner", + "type": "address", + }, + { + "indexed": False, + "internalType": "bytes", + "name": "orderUid", + "type": "bytes", + }, + ], + "name": "OrderInvalidated", + "type": "event", + }, + { + "anonymous": False, + "inputs": [ + { + "indexed": True, + "internalType": "address", + "name": "owner", + "type": "address", + }, + { + "indexed": False, + "internalType": "bytes", + "name": "orderUid", + "type": "bytes", + }, + { + "indexed": False, + "internalType": "bool", + "name": "signed", + "type": "bool", + }, + ], + "name": "PreSignature", + "type": "event", + }, + { + "anonymous": False, + "inputs": [ + { + "indexed": True, + "internalType": "address", + "name": "solver", + "type": "address", + } + ], + "name": "Settlement", + "type": "event", + }, + { + "anonymous": False, + "inputs": [ + { + "indexed": True, + "internalType": "address", + "name": "owner", + "type": "address", + }, + { + "indexed": False, + "internalType": "contract IERC20", + "name": "sellToken", + "type": "address", + }, + { + "indexed": False, + "internalType": "contract IERC20", + "name": "buyToken", + "type": "address", + }, + { + "indexed": False, + "internalType": "uint256", + "name": "sellAmount", + "type": "uint256", + }, + { + "indexed": False, + "internalType": "uint256", + "name": "buyAmount", + "type": "uint256", + }, + { + "indexed": False, + "internalType": "uint256", + "name": "feeAmount", + "type": "uint256", + }, + { + "indexed": False, + "internalType": "bytes", + "name": "orderUid", + "type": "bytes", + }, + ], + "name": "Trade", + "type": "event", + }, + { + "inputs": [], + "name": "authenticator", + "outputs": [ + { + "internalType": "contract GPv2Authentication", + "name": "", + "type": "address", + } + ], + "stateMutability": "view", + "type": "function", + }, + { + "inputs": [], + "name": "domainSeparator", + "outputs": [{"internalType": "bytes32", "name": "", "type": "bytes32"}], + "stateMutability": "view", + "type": "function", + }, + { + "inputs": [{"internalType": "bytes", "name": "", "type": "bytes"}], + "name": "filledAmount", + "outputs": [{"internalType": "uint256", "name": "", "type": "uint256"}], + "stateMutability": "view", + "type": "function", + }, + { + "inputs": [{"internalType": "bytes[]", "name": "orderUids", "type": "bytes[]"}], + "name": "freeFilledAmountStorage", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function", + }, + { + "inputs": [{"internalType": "bytes[]", "name": "orderUids", "type": "bytes[]"}], + "name": "freePreSignatureStorage", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function", + }, + { + "inputs": [ + {"internalType": "uint256", "name": "offset", "type": "uint256"}, + {"internalType": "uint256", "name": "length", "type": "uint256"}, + ], + "name": "getStorageAt", + "outputs": [{"internalType": "bytes", "name": "", "type": "bytes"}], + "stateMutability": "view", + "type": "function", + }, + { + "inputs": [{"internalType": "bytes", "name": "orderUid", "type": "bytes"}], + "name": "invalidateOrder", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function", + }, + { + "inputs": [{"internalType": "bytes", "name": "", "type": "bytes"}], + "name": "preSignature", + "outputs": [{"internalType": "uint256", "name": "", "type": "uint256"}], + "stateMutability": "view", + "type": "function", + }, + { + "inputs": [ + {"internalType": "bytes", "name": "orderUid", "type": "bytes"}, + {"internalType": "bool", "name": "signed", "type": "bool"}, + ], + "name": "setPreSignature", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function", + }, + { + "inputs": [ + { + "internalType": "contract IERC20[]", + "name": "tokens", + "type": "address[]", + }, + { + "internalType": "uint256[]", + "name": "clearingPrices", + "type": "uint256[]", + }, + { + "components": [ + { + "internalType": "uint256", + "name": "sellTokenIndex", + "type": "uint256", + }, + { + "internalType": "uint256", + "name": "buyTokenIndex", + "type": "uint256", + }, + {"internalType": "address", "name": "receiver", "type": "address"}, + { + "internalType": "uint256", + "name": "sellAmount", + "type": "uint256", + }, + {"internalType": "uint256", "name": "buyAmount", "type": "uint256"}, + {"internalType": "uint32", "name": "validTo", "type": "uint32"}, + {"internalType": "bytes32", "name": "appData", "type": "bytes32"}, + {"internalType": "uint256", "name": "feeAmount", "type": "uint256"}, + {"internalType": "uint256", "name": "flags", "type": "uint256"}, + { + "internalType": "uint256", + "name": "executedAmount", + "type": "uint256", + }, + {"internalType": "bytes", "name": "signature", "type": "bytes"}, + ], + "internalType": "struct GPv2Trade.Data[]", + "name": "trades", + "type": "tuple[]", + }, + { + "components": [ + {"internalType": "address", "name": "target", "type": "address"}, + {"internalType": "uint256", "name": "value", "type": "uint256"}, + {"internalType": "bytes", "name": "callData", "type": "bytes"}, + ], + "internalType": "struct GPv2Interaction.Data[][3]", + "name": "interactions", + "type": "tuple[][3]", + }, + ], + "name": "settle", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function", + }, + { + "inputs": [ + {"internalType": "address", "name": "targetContract", "type": "address"}, + {"internalType": "bytes", "name": "calldataPayload", "type": "bytes"}, + ], + "name": "simulateDelegatecall", + "outputs": [{"internalType": "bytes", "name": "response", "type": "bytes"}], + "stateMutability": "nonpayable", + "type": "function", + }, + { + "inputs": [ + {"internalType": "address", "name": "targetContract", "type": "address"}, + {"internalType": "bytes", "name": "calldataPayload", "type": "bytes"}, + ], + "name": "simulateDelegatecallInternal", + "outputs": [{"internalType": "bytes", "name": "response", "type": "bytes"}], + "stateMutability": "nonpayable", + "type": "function", + }, + { + "inputs": [ + { + "components": [ + {"internalType": "bytes32", "name": "poolId", "type": "bytes32"}, + { + "internalType": "uint256", + "name": "assetInIndex", + "type": "uint256", + }, + { + "internalType": "uint256", + "name": "assetOutIndex", + "type": "uint256", + }, + {"internalType": "uint256", "name": "amount", "type": "uint256"}, + {"internalType": "bytes", "name": "userData", "type": "bytes"}, + ], + "internalType": "struct IVault.BatchSwapStep[]", + "name": "swaps", + "type": "tuple[]", + }, + { + "internalType": "contract IERC20[]", + "name": "tokens", + "type": "address[]", + }, + { + "components": [ + { + "internalType": "uint256", + "name": "sellTokenIndex", + "type": "uint256", + }, + { + "internalType": "uint256", + "name": "buyTokenIndex", + "type": "uint256", + }, + {"internalType": "address", "name": "receiver", "type": "address"}, + { + "internalType": "uint256", + "name": "sellAmount", + "type": "uint256", + }, + {"internalType": "uint256", "name": "buyAmount", "type": "uint256"}, + {"internalType": "uint32", "name": "validTo", "type": "uint32"}, + {"internalType": "bytes32", "name": "appData", "type": "bytes32"}, + {"internalType": "uint256", "name": "feeAmount", "type": "uint256"}, + {"internalType": "uint256", "name": "flags", "type": "uint256"}, + { + "internalType": "uint256", + "name": "executedAmount", + "type": "uint256", + }, + {"internalType": "bytes", "name": "signature", "type": "bytes"}, + ], + "internalType": "struct GPv2Trade.Data", + "name": "trade", + "type": "tuple", + }, + ], + "name": "swap", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function", + }, + { + "inputs": [], + "name": "vault", + "outputs": [{"internalType": "contract IVault", "name": "", "type": "address"}], + "stateMutability": "view", + "type": "function", + }, + { + "inputs": [], + "name": "vaultRelayer", + "outputs": [ + {"internalType": "contract GPv2VaultRelayer", "name": "", "type": "address"} + ], + "stateMutability": "view", + "type": "function", + }, + {"stateMutability": "payable", "type": "receive"}, +] \ No newline at end of file diff --git a/src/constants.py b/src/constants.py index c24d7d6..cc7363a 100644 --- a/src/constants.py +++ b/src/constants.py @@ -14,6 +14,8 @@ "0x83F20F44975D03b1b09e64809B757c47f942BEeA" ) +REQUEST_TIMEOUT = 5 + # Time limit after which Coingecko Token List is re-fetched (in seconds) COINGECKO_TOKEN_LIST_RELOAD_TIME = 86400 diff --git a/src/fees/__init__.py b/src/fees/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/fees/compute_fees.py b/src/fees/compute_fees.py new file mode 100644 index 0000000..e275e2d --- /dev/null +++ b/src/fees/compute_fees.py @@ -0,0 +1,606 @@ +from abc import ABC, abstractmethod +from copy import deepcopy +from dataclasses import dataclass +from fractions import Fraction +import math +import os +from typing import Any + + +from dotenv import load_dotenv +from eth_typing import Address +from hexbytes import HexBytes +from web3 import Web3 +from web3.logs import DISCARD +from web3.types import TxData, TxReceipt, EventData + +from src.constants import ( + SETTLEMENT_CONTRACT_ADDRESS, + REQUEST_TIMEOUT, +) +from contracts.gpv2_settlement_abi import gpv2_settlement_abi + +import requests + +# types for trades + + +@dataclass +class Trade: + """Class for""" + + order_uid: HexBytes + sell_amount: int + buy_amount: int + owner: HexBytes + sell_token: HexBytes + buy_token: HexBytes + limit_sell_amount: int + limit_buy_amount: int + kind: str + sell_token_clearing_price: int + buy_token_clearing_price: int + + def volume(self) -> int: + """Compute volume of a trade in the surplus token""" + if self.kind == "sell": + return self.buy_amount + if self.kind == "buy": + return self.sell_amount + raise ValueError(f"Order kind {self.kind} is invalid.") + + def surplus(self) -> int: + """Compute surplus of a trade in the surplus token + For partially fillable orders, rounding is such that the reference for computing surplus is + such that it gives the worst price still allowed by the smart contract. That means that for + sell orders the limit buy amount is rounded up and for buy orders the limit sell amount is + rounded down. + """ + if self.kind == "sell": + current_limit_buy_amount = math.ceil( + self.limit_buy_amount + * Fraction(self.sell_amount, self.limit_sell_amount) + ) + return self.buy_amount - current_limit_buy_amount + if self.kind == "buy": + current_limit_sell_amount = int( + self.limit_sell_amount + * Fraction(self.buy_amount, self.limit_buy_amount) + ) + return current_limit_sell_amount - self.sell_amount + raise ValueError(f"Order kind {self.kind} is invalid.") + + def raw_surplus(self, fee_policies: list["FeePolicy"]) -> int: + """Compute raw surplus of a trade in the surplus token + First, the application of protocol fees is reversed. Then, surplus of the resulting trade + is computed.""" + raw_trade = deepcopy(self) + for fee_policy in reversed(fee_policies): + raw_trade = fee_policy.reverse_protocol_fee(raw_trade) + return raw_trade.surplus() + + def protocol_fee(self, fee_policies): + """Compute protocol fees of a trade in the surplus token + Protocol fees are computed as the difference of raw surplus and surplus.""" + + return self.raw_surplus(fee_policies) - self.surplus() + + def surplus_token(self) -> HexBytes: + """Returns the surplus token""" + if self.kind == "sell": + return self.buy_token + if self.kind == "buy": + return self.sell_token + raise ValueError(f"Order kind {self.kind} is invalid.") + + def price_improvement(self, quote: "Quote") -> int: + """Compute price improvement + For partially fillable orders, rounding is such that the reference for computing price + improvement is as if the quote would determine the limit price. That means that for sell + orders the quote buy amount is rounded up and for buy orders the quote sell amount is + rounded down. + """ + effective_sell_amount = quote.effective_sell_amount(self.kind) + effective_buy_amount = quote.effective_buy_amount(self.kind) + if self.kind == "sell": + current_limit_quote_amount = math.ceil( + effective_buy_amount * Fraction(self.sell_amount, effective_sell_amount) + ) + return self.buy_amount - current_limit_quote_amount + if self.kind == "buy": + current_quote_sell_amount = int( + effective_sell_amount * Fraction(self.buy_amount, effective_buy_amount) + ) + return current_quote_sell_amount - self.sell_amount + raise ValueError(f"Order kind {self.kind} is invalid.") + + def compute_surplus_fee(self) -> int: + if self.kind == "sell": + buy_amount_clearing_prices = math.ceil( + self.sell_amount + * Fraction( + self.sell_token_clearing_price, self.buy_token_clearing_price + ) + ) + return buy_amount_clearing_prices - self.buy_amount + if self.kind == "buy": + sell_amount_clearing_prices = int( + self.buy_amount + * Fraction( + self.buy_token_clearing_price, self.sell_token_clearing_price + ) + ) + return self.sell_amount - sell_amount_clearing_prices + raise ValueError(f"Order kind {self.kind} is invalid.") + + +# types for protocol fees + + +class FeePolicy(ABC): + """Abstract class for protocol fees + Concrete implementations have to implement a reverse_protocol_fee method. + """ + + # pylint: disable=too-few-public-methods + + @abstractmethod + def reverse_protocol_fee(self, trade: Trade) -> Trade: + """Reverse application of protocol fee + Returns a new trade object + """ + + +@dataclass +class VolumeFeePolicy(FeePolicy): + """Volume based protocol fee""" + + volume_factor: Fraction + + def reverse_protocol_fee(self, trade: Trade) -> Trade: + new_trade = deepcopy(trade) + volume = trade.volume() + if trade.kind == "sell": + fee = round(volume * self.volume_factor / (1 - self.volume_factor)) + new_trade.buy_amount = trade.buy_amount + fee + elif trade.kind == "buy": + fee = round(volume * self.volume_factor / (1 + self.volume_factor)) + new_trade.sell_amount = trade.sell_amount - fee + else: + raise ValueError(f"Order kind {trade.kind} is invalid.") + return new_trade + + +@dataclass +class SurplusFeePolicy(FeePolicy): + """Surplus based protocol fee""" + + surplus_factor: Fraction + surplus_max_volume_factor: Fraction + + def reverse_protocol_fee(self, trade: Trade) -> Trade: + new_trade = deepcopy(trade) + surplus = trade.surplus() + volume = trade.volume() + surplus_fee = round(surplus * self.surplus_factor / (1 - self.surplus_factor)) + if trade.kind == "sell": + volume_fee = round( + volume + * self.surplus_max_volume_factor + / (1 - self.surplus_max_volume_factor) + ) + fee = min(surplus_fee, volume_fee) + new_trade.buy_amount = trade.buy_amount + fee + elif trade.kind == "buy": + volume_fee = round( + volume + * self.surplus_max_volume_factor + / (1 + self.surplus_max_volume_factor) + ) + fee = min(surplus_fee, volume_fee) + new_trade.sell_amount = trade.sell_amount - fee + else: + raise ValueError(f"Order kind {trade.kind} is invalid.") + return new_trade + + +@dataclass +class Quote: + """Class representing quotes""" + + sell_amount: int + buy_amount: int + fee_amount: int + + def effective_sell_amount(self, kind: str) -> int: + if kind == "sell": + return self.sell_amount + if kind == "buy": + return self.sell_amount + self.fee_amount + raise ValueError(f"Order kind {kind} is invalid.") + + def effective_buy_amount(self, kind: str) -> int: + if kind == "sell": + exchange_rate = Fraction(self.buy_amount, self.sell_amount) + return math.ceil((self.sell_amount - self.fee_amount) * exchange_rate) + if kind == "buy": + return self.buy_amount + raise ValueError(f"Order kind {kind} is invalid.") + + +@dataclass +class PriceImprovementFeePolicy(FeePolicy): + """Price improvement based protocol fee""" + + price_improvement_factor: Fraction + price_improvement_max_volume_factor: Fraction + quote: Quote + + def reverse_protocol_fee(self, trade: Trade) -> Trade: + new_trade = deepcopy(trade) + price_improvement = trade.price_improvement(self.quote) + volume = trade.volume() + price_improvement_fee = max( + 0, + round( + price_improvement + * self.price_improvement_factor + / (1 - self.price_improvement_factor) + ), + ) + if trade.kind == "sell": + volume_fee = round( + volume + * self.price_improvement_max_volume_factor + / (1 - self.price_improvement_max_volume_factor) + ) + fee = min(price_improvement_fee, volume_fee) + new_trade.buy_amount = trade.buy_amount + fee + elif trade.kind == "buy": + volume_fee = round( + volume + * self.price_improvement_max_volume_factor + / (1 + self.price_improvement_max_volume_factor) + ) + fee = min(price_improvement_fee, volume_fee) + new_trade.sell_amount = trade.sell_amount - fee + else: + raise ValueError(f"Order kind {trade.kind} is invalid.") + return new_trade + + +@dataclass +class OnchainSettlementData: + """Class to describe onchain info about a settlement.""" + + auction_id: int + tx_hash: HexBytes + solver: HexBytes + call_data: HexBytes + trades: list[Trade] + + +@dataclass +class OffchainSettlementData: + """Class to describe offchain info about a settlement.""" + + # pylint: disable=too-many-instance-attributes + + auction_id: int + solver: HexBytes + call_data: HexBytes + trade_fee_policies: dict[HexBytes, list[FeePolicy]] + score: int + valid_orders: set[HexBytes] + jit_order_addresses: set[HexBytes] + native_prices: dict[HexBytes, int] + + +# fetching data + + +class BlockchainFetcher: + """ + Class to connect to a node and fetch/process onchain data. + """ + + def __init__(self) -> None: + load_dotenv() + + node_url = os.getenv("NODE_URL") + self.node = Web3(Web3.HTTPProvider(node_url)) + + self.contract = self.node.eth.contract( + address=Address(HexBytes(SETTLEMENT_CONTRACT_ADDRESS)), + abi=gpv2_settlement_abi, + ) + + def get_onchain_data(self, tx_hash: HexBytes) -> OnchainSettlementData: + """ + This function can error since nodes are called. + """ + transaction = self.node.eth.get_transaction(tx_hash) + receipt = self.node.eth.wait_for_transaction_receipt(tx_hash) + decoded_data = self.decode_data(transaction, receipt) + return decoded_data + + def decode_data( + self, transaction: TxData, receipt: TxReceipt + ) -> OnchainSettlementData: + """ + Method that decodes a tx and returns a summary of this decoding. + """ + tx_hash = transaction["hash"] + call_data = transaction["input"] + if isinstance(call_data, str): + call_data = bytes.fromhex(call_data[2:]) + auction_id = int.from_bytes(call_data[-8:], byteorder="big") + + settlement_event = self.contract.events.Settlement().process_receipt( + receipt, errors=DISCARD + )[0] + solver = HexBytes(settlement_event["args"]["solver"]) + trade_events: tuple[EventData] = self.contract.events.Trade().process_receipt( + receipt, errors=DISCARD + ) + + settlement = self.contract.decode_function_input(call_data)[1] + + tokens = settlement["tokens"] + prices = settlement["clearingPrices"] + + trades: list[Trade] = [] + for trade_event, settlement_trade in zip(trade_events, settlement["trades"]): + sell_token_address = HexBytes(tokens[settlement_trade["sellTokenIndex"]]) + buy_token_address = HexBytes(tokens[settlement_trade["buyTokenIndex"]]) + sell_token_clearing_price = prices[ + tokens.index(tokens[settlement_trade["sellTokenIndex"]]) + ] + buy_token_clearing_price = prices[ + tokens.index(tokens[settlement_trade["buyTokenIndex"]]) + ] + trade = Trade( + HexBytes(trade_event["args"]["orderUid"]), + trade_event["args"]["sellAmount"], + trade_event["args"]["buyAmount"], + HexBytes(trade_event["args"]["owner"]), + sell_token_address, + buy_token_address, + settlement_trade["sellAmount"], + settlement_trade["buyAmount"], + "sell" if settlement_trade["flags"] % 2 == 0 else "buy", + sell_token_clearing_price, + buy_token_clearing_price, + ) + trades.append(trade) + + return OnchainSettlementData(auction_id, tx_hash, solver, call_data, trades) + + +class OrderbookAWSFetcher: + """ + This is a class for connecting to the db, and contains a few functions that + fetch necessary data to run the checks that we need. + """ + + def __init__(self) -> None: + load_dotenv() + network = os.getenv("NETWORK") + + self.orderbook_urls = { + "prod": f"https://api.cow.fi/{network}/api/v1/", + "barn": f"https://barn.api.cow.fi/{network}/api/v1/", + } + self.aws_urls = { + "prod": "https://solver-instances.s3.eu-central-1.amazonaws.com/" + f"prod/{network}/autopilot/", + "barn": "https://solver-instances.s3.eu-central-1.amazonaws.com/" + f"staging/{network}/autopilot/", + } + + def get_offchain_data( + self, onchain_data: OnchainSettlementData + ) -> OffchainSettlementData: + """ + Method that fetches all necessary data from the API. + """ + solver = onchain_data.solver + auction_id = onchain_data.auction_id + + solution_data, environment = self.get_solution_data(auction_id, solver) + auction_data = self.get_auction_data(auction_id, environment) + + offchain_data = self.convert_to_offchain_data( + onchain_data, + auction_data, + solution_data, + ) + return offchain_data + + def convert_to_offchain_data( + self, + onchain_data: OnchainSettlementData, + auction_data: dict[str, Any], + solution_data: dict[str, Any], + ) -> OffchainSettlementData: + """Turn Row from database query into OffchainSettlementData""" + # pylint: disable=too-many-locals + auction_id = onchain_data.auction_id + + solver = HexBytes(solution_data["solverAddress"]) + call_data = HexBytes(solution_data["callData"]) + + trade_fee_policies: dict[HexBytes, list[FeePolicy]] = {} + onchain_trades_dict = {trade.order_uid: trade for trade in onchain_data.trades} + protocol_fees_dict = { + HexBytes(order["uid"]): order["protocolFees"] + for order in auction_data["orders"] + if HexBytes(order["uid"]) in onchain_trades_dict + } + for order in solution_data["orders"]: + order_uid = HexBytes(order["id"]) + + fee_policies = self.parse_fee_policies( + protocol_fees_dict.get(order_uid, []) + ) + + trade_fee_policies[order_uid] = fee_policies + + score = int(solution_data["score"]) + valid_orders = {HexBytes(order["uid"]) for order in auction_data["orders"]} + native_prices = { + HexBytes(address): int(price) + for address, price in auction_data["prices"].items() + } + jit_order_addresses = { + HexBytes(address) + for address in auction_data["surplusCapturingJitOrderOwners"] + } + + return OffchainSettlementData( + auction_id, + solver, + call_data, + trade_fee_policies, + score, + valid_orders, + jit_order_addresses, + native_prices, + ) + + def get_solution_data( + self, auction_id: int, solver: HexBytes + ) -> tuple[dict[str, Any], str]: + """Fetch competition data from the database""" + for environment, url in self.orderbook_urls.items(): + try: + response = requests.get( + url + f"solver_competition/{auction_id}", + timeout=REQUEST_TIMEOUT, + ) + response.raise_for_status() + solution_data = response.json()["solutions"][-1] + if HexBytes(solution_data["solverAddress"]) == solver: + return solution_data, environment + except requests.exceptions.HTTPError as err: + if err.response.status_code == 404: + pass + raise ConnectionError(f"Error fetching off-chain data for id {auction_id}") + + def get_auction_data( + self, + auction_id: int, + environment: str, + ) -> dict[str, Any]: + """Fetch auction data from AWS.""" + url = self.aws_urls[environment] + f"{auction_id}.json" + response = requests.get(url, timeout=REQUEST_TIMEOUT) + response.raise_for_status() + auction_data: dict[str, Any] = response.json() + return auction_data + + def parse_fee_policies( + self, protocol_fee_datum: list[dict[str, Any]] + ) -> list[FeePolicy]: + """Pase protocol fees into sorted list""" + fee_policies: list[FeePolicy] = [] + for fee_policy in protocol_fee_datum: + if "surplus" in fee_policy: + fee_policies.append( + SurplusFeePolicy( + Fraction(fee_policy["surplus"]["factor"]), + Fraction(fee_policy["surplus"]["maxVolumeFactor"]), + ) + ) + elif "volume" in fee_policy: + fee_policies.append( + VolumeFeePolicy(Fraction(fee_policy["volume"]["factor"])) + ) + elif "priceImprovement" in fee_policy: + quote = Quote( + int(fee_policy["priceImprovement"]["quote"]["sellAmount"]), + int(fee_policy["priceImprovement"]["quote"]["buyAmount"]), + int(fee_policy["priceImprovement"]["quote"]["fee"]), + ) + fee_policies.append( + PriceImprovementFeePolicy( + Fraction(fee_policy["priceImprovement"]["factor"]), + Fraction(fee_policy["priceImprovement"]["maxVolumeFactor"]), + quote, + ) + ) + else: + raise ValueError(f"Fee kind {fee_policy.keys()} is invalid.") + return fee_policies + + +def fetch_settlement_data( + tx_hash: HexBytes, +) -> tuple[OnchainSettlementData, OffchainSettlementData]: + onchain_fetcher = BlockchainFetcher() + offchain_fetcher = OrderbookAWSFetcher() + + onchain_data = onchain_fetcher.get_onchain_data(tx_hash) + offchain_data = offchain_fetcher.get_offchain_data(onchain_data) + + return onchain_data, offchain_data + + +# computing fees + + +def compute_fee_imbalances( + onchain_data: OnchainSettlementData, offchain_data: OffchainSettlementData +) -> tuple[dict[HexBytes, int], dict[HexBytes, int]]: + protocol_fees: dict[HexBytes, int] = {} + network_fees: dict[HexBytes, int] = {} + for trade in onchain_data.trades: + # protocol fees + fee_policies = offchain_data.trade_fee_policies[trade.order_uid] + protocol_fee_amount = trade.protocol_fee(fee_policies) + protocol_fee_token = trade.surplus_token() + protocol_fees[protocol_fee_token] = protocol_fee_amount + + # network fees + surplus_fee = trade.compute_surplus_fee() # in the surplus token + network_fee = surplus_fee - protocol_fee_amount + if trade.kind == "sell": + network_fee_sell = int( + network_fee + * Fraction( + trade.buy_token_clearing_price, trade.sell_token_clearing_price + ) + ) + else: + network_fee_sell = network_fee + + network_fees[trade.sell_token] = network_fee_sell + + return protocol_fees, network_fees + + +# combined function + + +def batch_fee_imbalances( + tx_hash: HexBytes, +) -> tuple[dict[HexBytes, int], dict[HexBytes, int]]: + onchain_data, offchain_data = fetch_settlement_data(tx_hash) + protocol_fees, network_fees = compute_fee_imbalances(onchain_data, offchain_data) + protocol_fees = { + Web3.to_checksum_address(token.hex()): fee + for token, fee in protocol_fees.items() + } + network_fees = { + Web3.to_checksum_address(token.hex()): fee + for token, fee in network_fees.items() + } + return protocol_fees, network_fees + + +# if __name__ == "__main__": +# tx_hash = HexBytes( +# "0xbd8cf4a21ad811cc3b9e49cff5e95563c3c2651b0ea41e0f8a7987818205c984" +# ) +# protocol_fees, network_fees = batch_fee_imbalances(tx_hash) +# print(protocol_fees) diff --git a/src/helpers/database.py b/src/helpers/database.py index d6c9911..9629ec3 100644 --- a/src/helpers/database.py +++ b/src/helpers/database.py @@ -92,3 +92,31 @@ def write_prices( "price": price, }, ) + + def write_fees( + self, + chain_name: str, + auction_id: int, + block_number: int, + tx_hash: str, + token_address: str, + fee_amount: float, + fee_type: str, + ): + """Function attempts to write price data to the table.""" + tx_hash_bytes = bytes.fromhex(tx_hash[2:]) + token_address_bytes = bytes.fromhex(token_address[2:]) + + query = read_sql_file("src/sql/insert_fee.sql") + self.execute_and_commit( + query, + { + "chain_name": self.chain_name, + "auction_id": auction_id, + "block_number": block_number, + "tx_hash": tx_hash_bytes, + "token_address": token_address_bytes, + "fee_amount": fee_amount, + "fee_type": fee_type, + }, + ) diff --git a/src/sql/insert_coingecko_price.sql b/src/sql/insert_coingecko_price.sql deleted file mode 100644 index 8aa1990..0000000 --- a/src/sql/insert_coingecko_price.sql +++ /dev/null @@ -1,4 +0,0 @@ -INSERT INTO coingecko_prices -(chain_name, block_number, tx_hash, token_address, coingecko_price) -VALUES -(:chain_name, :block_number, :tx_hash, :token_address, :coingecko_price); diff --git a/src/sql/insert_fee.sql b/src/sql/insert_fee.sql new file mode 100644 index 0000000..9f57b22 --- /dev/null +++ b/src/sql/insert_fee.sql @@ -0,0 +1,4 @@ +INSERT INTO fees ( + chain_name, auction_id, block_number, tx_hash, token_address, fee_amount,fee_type +) VALUES ( :chain_name, :auction_id, :block_number, :tx_hash, :token_address, :fee_amount, :fee_type +); diff --git a/src/test_single_hash.py b/src/test_single_hash.py new file mode 100644 index 0000000..93451e2 --- /dev/null +++ b/src/test_single_hash.py @@ -0,0 +1,76 @@ +from hexbytes import HexBytes +from src.imbalances_script import RawTokenImbalances +from src.price_providers.price_feed import PriceFeed +from src.fees.compute_fees import batch_fee_imbalances +from src.transaction_processor import calculate_slippage +from src.helpers.config import get_web3_instance, logger +from contracts.erc20_abi import erc20_abi + + +def log_token_data(title: str, data: dict, name: str): + logger.info(title) + for token, value in data.items(): + logger.info(f"Token Address: {token}, {name}: {value}") + + +class Compute: + def __init__(self): + self.web3 = get_web3_instance() + self.imbalances = RawTokenImbalances(self.web3, "mainnet") + self.price_providers = PriceFeed() + + def compute_data(self, tx_hash: str): + token_imbalances = self.imbalances.compute_imbalances(tx_hash) + protocol_fees, network_fees = batch_fee_imbalances(HexBytes(tx_hash)) + slippage = calculate_slippage(token_imbalances, protocol_fees, network_fees) + eth_slippage = self.calculate_slippage_in_eth(slippage, tx_hash) + + self.log_results( + token_imbalances, protocol_fees, network_fees, slippage, eth_slippage + ) + + def calculate_slippage_in_eth(self, slippage: dict, tx_hash: str) -> dict: + """Calculate slippage in ETH.""" + eth_slippage = {} + receipt = self.web3.eth.get_transaction_receipt(tx_hash) + if receipt: + block_number = receipt.blockNumber + for token_address, amt in slippage.items(): + if amt != 0: + price_data = self.price_providers.get_price( + block_number, token_address + ) + if price_data: + price, _ = price_data + decimals = self._get_token_decimals(token_address) + slippage_in_eth = price * (amt / (10**decimals)) + eth_slippage[token_address] = slippage_in_eth + return eth_slippage + + def _get_token_decimals(self, token_address: str) -> int: + contract = self.web3.eth.contract(address=token_address, abi=erc20_abi) + return contract.functions.decimals().call() + + def log_results( + self, + token_imbalances: dict, + protocol_fees: dict, + network_fees: dict, + slippage: dict, + eth_slippage: dict, + ): + log_token_data("Raw Imbalances:", token_imbalances, "Raw Imbalance") + log_token_data("Protocol Fees:", protocol_fees, "Protocol Fee") + log_token_data("Network Fees:", network_fees, "Network Fee") + log_token_data("Raw Slippage Calculation:", slippage, "Raw Slippage") + log_token_data("Slippage in ETH", eth_slippage, "Slippage") + + +def main(): + compute = Compute() + # e.g. input: 0x980fa3f8ff95c504ba61e054e5c3e50ea36b892f865703b8a665564ac0beb1f4 + compute.compute_data(input("tx hash: ")) + + +if __name__ == "__main__": + main() diff --git a/src/transaction_processor.py b/src/transaction_processor.py index 992563d..e11476f 100644 --- a/src/transaction_processor.py +++ b/src/transaction_processor.py @@ -1,9 +1,12 @@ +from hexbytes import HexBytes +from web3 import Web3 from src.helpers.blockchain_data import BlockchainData from src.helpers.database import Database from src.imbalances_script import RawTokenImbalances from src.price_providers.price_feed import PriceFeed from src.helpers.helper_functions import read_sql_file from src.helpers.config import CHAIN_SLEEP_TIME, logger +from src.fees.compute_fees import batch_fee_imbalances import time @@ -99,9 +102,14 @@ def process_single_transaction( tx_hash, auction_id, block_number, token_address, imbalance ) log_message.append(f"Token: {token_address}, Imbalance: {imbalance}") - for token_address in token_imbalances.keys(): + + protocol_fees, network_fees = batch_fee_imbalances(HexBytes(tx_hash)) + self.handle_fees(protocol_fees, network_fees, auction_id, block_number, tx_hash) + slippage = calculate_slippage(token_imbalances, protocol_fees, network_fees) + + for token_address in slippage.keys(): # fetch price for tokens with non-zero imbalance and write to table - if token_imbalances[token_address] != 0: + if slippage[token_address] != 0: price_data = self.price_providers.get_price(block_number, token_address) if price_data: price, source = price_data @@ -111,3 +119,57 @@ def process_single_transaction( log_message.append(f"Token: {token_address}, Price: {price} ETH") logger.info("\n".join(log_message)) + + def handle_fees( + self, protocol_fees, network_fees, auction_id, block_number, tx_hash + ): + """This function loops over (token, fee) and calls write_fees to write to table.""" + # Write protocol fees + for token_address, fee_amount in protocol_fees.items(): + self.db.write_fees( + chain_name=self.chain_name, + auction_id=auction_id, + block_number=block_number, + tx_hash=tx_hash, + token_address=token_address, + fee_amount=float(fee_amount), + fee_type="protocol", + ) + + # Write network fees + for token_address, fee_amount in network_fees.items(): + self.db.write_fees( + chain_name=self.chain_name, + auction_id=auction_id, + block_number=block_number, + tx_hash=tx_hash, + token_address=token_address, + fee_amount=float(fee_amount), + fee_type="network", + ) + + +def calculate_slippage( + token_imbalances: dict[str, int], + protocol_fees: dict[str, int], + network_fees: dict[str, int], +) -> dict[str, int]: + """Function calculates net slippage for each token per tx.""" + # set of all tokens from all three dicts + all_tokens = ( + set(token_imbalances.keys()) + .union(protocol_fees.keys()) + .union(network_fees.keys()) + ) + slippage = {} + + # calculate net slippage per token + for token in all_tokens: + imbalance = token_imbalances.get(token, 0) + protocol_fee = protocol_fees.get(token, 0) + network_fee = network_fees.get(token, 0) + + total = imbalance - protocol_fee - network_fee + slippage[token] = total + + return slippage From bdbb0c3da7fe8aa8737e78a005db5608687d7fbe Mon Sep 17 00:00:00 2001 From: Shubh Agarwal Date: Thu, 15 Aug 2024 10:01:34 -0400 Subject: [PATCH 02/19] edit env var name --- src/fees/compute_fees.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/fees/compute_fees.py b/src/fees/compute_fees.py index e275e2d..a26cdfb 100644 --- a/src/fees/compute_fees.py +++ b/src/fees/compute_fees.py @@ -385,17 +385,17 @@ class OrderbookAWSFetcher: def __init__(self) -> None: load_dotenv() - network = os.getenv("NETWORK") + chain_name = os.getenv("CHAIN_NAME") self.orderbook_urls = { - "prod": f"https://api.cow.fi/{network}/api/v1/", - "barn": f"https://barn.api.cow.fi/{network}/api/v1/", + "prod": f"https://api.cow.fi/{chain_name}/api/v1/", + "barn": f"https://barn.api.cow.fi/{chain_name}/api/v1/", } self.aws_urls = { "prod": "https://solver-instances.s3.eu-central-1.amazonaws.com/" - f"prod/{network}/autopilot/", + f"prod/{chain_name}/autopilot/", "barn": "https://solver-instances.s3.eu-central-1.amazonaws.com/" - f"staging/{network}/autopilot/", + f"staging/{chain_name}/autopilot/", } def get_offchain_data( From 147b08e27515484298d555c327de9bc70b368236 Mon Sep 17 00:00:00 2001 From: Shubh Agarwal Date: Tue, 20 Aug 2024 13:03:15 -0400 Subject: [PATCH 03/19] black + type fix --- contracts/gpv2_settlement_abi.py | 2 +- src/fees/compute_fees.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contracts/gpv2_settlement_abi.py b/contracts/gpv2_settlement_abi.py index f09f251..c64e82a 100644 --- a/contracts/gpv2_settlement_abi.py +++ b/contracts/gpv2_settlement_abi.py @@ -386,4 +386,4 @@ "type": "function", }, {"stateMutability": "payable", "type": "receive"}, -] \ No newline at end of file +] diff --git a/src/fees/compute_fees.py b/src/fees/compute_fees.py index a26cdfb..c084b3f 100644 --- a/src/fees/compute_fees.py +++ b/src/fees/compute_fees.py @@ -584,7 +584,7 @@ def compute_fee_imbalances( def batch_fee_imbalances( tx_hash: HexBytes, -) -> tuple[dict[HexBytes, int], dict[HexBytes, int]]: +) -> tuple[dict[str, int], dict[str, int]]: onchain_data, offchain_data = fetch_settlement_data(tx_hash) protocol_fees, network_fees = compute_fee_imbalances(onchain_data, offchain_data) protocol_fees = { From d0b1dda5a7f059b02dfc6700fd5b108ebab78b26 Mon Sep 17 00:00:00 2001 From: Shubh Agarwal Date: Tue, 20 Aug 2024 19:23:25 -0400 Subject: [PATCH 04/19] minor fix --- src/test_single_hash.py | 4 ++++ src/transaction_processor.py | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/test_single_hash.py b/src/test_single_hash.py index 93451e2..da0efc3 100644 --- a/src/test_single_hash.py +++ b/src/test_single_hash.py @@ -14,6 +14,10 @@ def log_token_data(title: str, data: dict, name: str): class Compute: + """ + Class that allows one to fetch imbalances, fees, final slippage via a tx hash. + """ + def __init__(self): self.web3 = get_web3_instance() self.imbalances = RawTokenImbalances(self.web3, "mainnet") diff --git a/src/transaction_processor.py b/src/transaction_processor.py index 5d8177a..6a3f53d 100644 --- a/src/transaction_processor.py +++ b/src/transaction_processor.py @@ -1,5 +1,4 @@ from hexbytes import HexBytes -from web3 import Web3 from src.helpers.blockchain_data import BlockchainData from src.helpers.database import Database from src.imbalances_script import RawTokenImbalances From ac7674975617f81d3e7b7fcd38f33454f0a9c970 Mon Sep 17 00:00:00 2001 From: harisang Date: Thu, 22 Aug 2024 11:45:00 +0300 Subject: [PATCH 05/19] fix mypy issue and address review comment --- src/fees/compute_fees.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/fees/compute_fees.py b/src/fees/compute_fees.py index c084b3f..a811eac 100644 --- a/src/fees/compute_fees.py +++ b/src/fees/compute_fees.py @@ -588,19 +588,12 @@ def batch_fee_imbalances( onchain_data, offchain_data = fetch_settlement_data(tx_hash) protocol_fees, network_fees = compute_fee_imbalances(onchain_data, offchain_data) protocol_fees = { - Web3.to_checksum_address(token.hex()): fee + token.hex(): fee for token, fee in protocol_fees.items() } network_fees = { - Web3.to_checksum_address(token.hex()): fee + token.hex(): fee for token, fee in network_fees.items() } return protocol_fees, network_fees - -# if __name__ == "__main__": -# tx_hash = HexBytes( -# "0xbd8cf4a21ad811cc3b9e49cff5e95563c3c2651b0ea41e0f8a7987818205c984" -# ) -# protocol_fees, network_fees = batch_fee_imbalances(tx_hash) -# print(protocol_fees) From 53125f4d996ce5f5d7d93ee59649cac1a9fcf7cf Mon Sep 17 00:00:00 2001 From: harisang Date: Thu, 22 Aug 2024 11:46:41 +0300 Subject: [PATCH 06/19] black fix --- src/fees/compute_fees.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/fees/compute_fees.py b/src/fees/compute_fees.py index a811eac..b2eef36 100644 --- a/src/fees/compute_fees.py +++ b/src/fees/compute_fees.py @@ -587,13 +587,6 @@ def batch_fee_imbalances( ) -> tuple[dict[str, int], dict[str, int]]: onchain_data, offchain_data = fetch_settlement_data(tx_hash) protocol_fees, network_fees = compute_fee_imbalances(onchain_data, offchain_data) - protocol_fees = { - token.hex(): fee - for token, fee in protocol_fees.items() - } - network_fees = { - token.hex(): fee - for token, fee in network_fees.items() - } + protocol_fees = {token.hex(): fee for token, fee in protocol_fees.items()} + network_fees = {token.hex(): fee for token, fee in network_fees.items()} return protocol_fees, network_fees - From cba6ab62f7524e8c74822980a8b69157a470ba27 Mon Sep 17 00:00:00 2001 From: harisang Date: Thu, 22 Aug 2024 11:50:19 +0300 Subject: [PATCH 07/19] mypy fix --- src/fees/compute_fees.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fees/compute_fees.py b/src/fees/compute_fees.py index b2eef36..72cb13d 100644 --- a/src/fees/compute_fees.py +++ b/src/fees/compute_fees.py @@ -587,6 +587,6 @@ def batch_fee_imbalances( ) -> tuple[dict[str, int], dict[str, int]]: onchain_data, offchain_data = fetch_settlement_data(tx_hash) protocol_fees, network_fees = compute_fee_imbalances(onchain_data, offchain_data) - protocol_fees = {token.hex(): fee for token, fee in protocol_fees.items()} - network_fees = {token.hex(): fee for token, fee in network_fees.items()} + protocol_fees = {token: fee for token, fee in protocol_fees.items()} + network_fees = {token: fee for token, fee in network_fees.items()} return protocol_fees, network_fees From cb6e33e0f13ae715cfd7b1102b5290fe7a8309e6 Mon Sep 17 00:00:00 2001 From: harisang Date: Thu, 22 Aug 2024 11:54:10 +0300 Subject: [PATCH 08/19] fix types in batch_fee_imbalances --- src/fees/compute_fees.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fees/compute_fees.py b/src/fees/compute_fees.py index 72cb13d..f7c69ad 100644 --- a/src/fees/compute_fees.py +++ b/src/fees/compute_fees.py @@ -584,7 +584,7 @@ def compute_fee_imbalances( def batch_fee_imbalances( tx_hash: HexBytes, -) -> tuple[dict[str, int], dict[str, int]]: +) -> tuple[dict[HexBytes, int], dict[HexBytes, int]]: onchain_data, offchain_data = fetch_settlement_data(tx_hash) protocol_fees, network_fees = compute_fee_imbalances(onchain_data, offchain_data) protocol_fees = {token: fee for token, fee in protocol_fees.items()} From 8b43e8cbe58d875fda6a146bac76fc853ff1f529 Mon Sep 17 00:00:00 2001 From: harisang Date: Thu, 22 Aug 2024 12:01:22 +0300 Subject: [PATCH 09/19] Revert "minor fix" This reverts commit d0b1dda5a7f059b02dfc6700fd5b108ebab78b26. --- src/test_single_hash.py | 4 ---- src/transaction_processor.py | 1 + 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/test_single_hash.py b/src/test_single_hash.py index da0efc3..93451e2 100644 --- a/src/test_single_hash.py +++ b/src/test_single_hash.py @@ -14,10 +14,6 @@ def log_token_data(title: str, data: dict, name: str): class Compute: - """ - Class that allows one to fetch imbalances, fees, final slippage via a tx hash. - """ - def __init__(self): self.web3 = get_web3_instance() self.imbalances = RawTokenImbalances(self.web3, "mainnet") diff --git a/src/transaction_processor.py b/src/transaction_processor.py index 6a3f53d..5d8177a 100644 --- a/src/transaction_processor.py +++ b/src/transaction_processor.py @@ -1,4 +1,5 @@ from hexbytes import HexBytes +from web3 import Web3 from src.helpers.blockchain_data import BlockchainData from src.helpers.database import Database from src.imbalances_script import RawTokenImbalances From 97772e13d57274444027bb9d97e63845ea1763b2 Mon Sep 17 00:00:00 2001 From: harisang Date: Thu, 22 Aug 2024 12:14:05 +0300 Subject: [PATCH 10/19] Revert "Revert "minor fix"" This reverts commit 8b43e8cbe58d875fda6a146bac76fc853ff1f529. --- src/test_single_hash.py | 4 ++++ src/transaction_processor.py | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/test_single_hash.py b/src/test_single_hash.py index 93451e2..da0efc3 100644 --- a/src/test_single_hash.py +++ b/src/test_single_hash.py @@ -14,6 +14,10 @@ def log_token_data(title: str, data: dict, name: str): class Compute: + """ + Class that allows one to fetch imbalances, fees, final slippage via a tx hash. + """ + def __init__(self): self.web3 = get_web3_instance() self.imbalances = RawTokenImbalances(self.web3, "mainnet") diff --git a/src/transaction_processor.py b/src/transaction_processor.py index 5d8177a..6a3f53d 100644 --- a/src/transaction_processor.py +++ b/src/transaction_processor.py @@ -1,5 +1,4 @@ from hexbytes import HexBytes -from web3 import Web3 from src.helpers.blockchain_data import BlockchainData from src.helpers.database import Database from src.imbalances_script import RawTokenImbalances From 80cef616aa9135c59d37ca0b2cd615cb61a27c26 Mon Sep 17 00:00:00 2001 From: harisang Date: Thu, 22 Aug 2024 12:15:56 +0300 Subject: [PATCH 11/19] Revert "Revert "Revert "minor fix""" This reverts commit 97772e13d57274444027bb9d97e63845ea1763b2. --- src/test_single_hash.py | 4 ---- src/transaction_processor.py | 1 + 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/test_single_hash.py b/src/test_single_hash.py index da0efc3..93451e2 100644 --- a/src/test_single_hash.py +++ b/src/test_single_hash.py @@ -14,10 +14,6 @@ def log_token_data(title: str, data: dict, name: str): class Compute: - """ - Class that allows one to fetch imbalances, fees, final slippage via a tx hash. - """ - def __init__(self): self.web3 = get_web3_instance() self.imbalances = RawTokenImbalances(self.web3, "mainnet") diff --git a/src/transaction_processor.py b/src/transaction_processor.py index 6a3f53d..5d8177a 100644 --- a/src/transaction_processor.py +++ b/src/transaction_processor.py @@ -1,4 +1,5 @@ from hexbytes import HexBytes +from web3 import Web3 from src.helpers.blockchain_data import BlockchainData from src.helpers.database import Database from src.imbalances_script import RawTokenImbalances From bba426c54a2b8801af51397f2512bd9fc810527d Mon Sep 17 00:00:00 2001 From: harisang Date: Thu, 22 Aug 2024 12:15:58 +0300 Subject: [PATCH 12/19] t Revert "Revert "minor fix"" This reverts commit 8b43e8cbe58d875fda6a146bac76fc853ff1f529. --- src/test_single_hash.py | 4 ++++ src/transaction_processor.py | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/test_single_hash.py b/src/test_single_hash.py index 93451e2..da0efc3 100644 --- a/src/test_single_hash.py +++ b/src/test_single_hash.py @@ -14,6 +14,10 @@ def log_token_data(title: str, data: dict, name: str): class Compute: + """ + Class that allows one to fetch imbalances, fees, final slippage via a tx hash. + """ + def __init__(self): self.web3 = get_web3_instance() self.imbalances = RawTokenImbalances(self.web3, "mainnet") diff --git a/src/transaction_processor.py b/src/transaction_processor.py index 5d8177a..6a3f53d 100644 --- a/src/transaction_processor.py +++ b/src/transaction_processor.py @@ -1,5 +1,4 @@ from hexbytes import HexBytes -from web3 import Web3 from src.helpers.blockchain_data import BlockchainData from src.helpers.database import Database from src.imbalances_script import RawTokenImbalances From 19a82db5b4e13f9bbf99d1e1a0ee57b43e068993 Mon Sep 17 00:00:00 2001 From: harisang Date: Thu, 22 Aug 2024 12:16:02 +0300 Subject: [PATCH 13/19] vert "fix types in batch_fee_imbalances" This reverts commit cb6e33e0f13ae715cfd7b1102b5290fe7a8309e6. --- src/fees/compute_fees.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fees/compute_fees.py b/src/fees/compute_fees.py index f7c69ad..72cb13d 100644 --- a/src/fees/compute_fees.py +++ b/src/fees/compute_fees.py @@ -584,7 +584,7 @@ def compute_fee_imbalances( def batch_fee_imbalances( tx_hash: HexBytes, -) -> tuple[dict[HexBytes, int], dict[HexBytes, int]]: +) -> tuple[dict[str, int], dict[str, int]]: onchain_data, offchain_data = fetch_settlement_data(tx_hash) protocol_fees, network_fees = compute_fee_imbalances(onchain_data, offchain_data) protocol_fees = {token: fee for token, fee in protocol_fees.items()} From c6f36efbf4edeb92e891eafd33dfbd6e6e30695f Mon Sep 17 00:00:00 2001 From: harisang Date: Thu, 22 Aug 2024 12:16:05 +0300 Subject: [PATCH 14/19] Revert "mypy fix" This reverts commit cba6ab62f7524e8c74822980a8b69157a470ba27. --- src/fees/compute_fees.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fees/compute_fees.py b/src/fees/compute_fees.py index 72cb13d..b2eef36 100644 --- a/src/fees/compute_fees.py +++ b/src/fees/compute_fees.py @@ -587,6 +587,6 @@ def batch_fee_imbalances( ) -> tuple[dict[str, int], dict[str, int]]: onchain_data, offchain_data = fetch_settlement_data(tx_hash) protocol_fees, network_fees = compute_fee_imbalances(onchain_data, offchain_data) - protocol_fees = {token: fee for token, fee in protocol_fees.items()} - network_fees = {token: fee for token, fee in network_fees.items()} + protocol_fees = {token.hex(): fee for token, fee in protocol_fees.items()} + network_fees = {token.hex(): fee for token, fee in network_fees.items()} return protocol_fees, network_fees From 320acf24c881e2b630cdd193c0160dad44aa74bf Mon Sep 17 00:00:00 2001 From: harisang Date: Thu, 22 Aug 2024 12:16:07 +0300 Subject: [PATCH 15/19] Revert "black fix" This reverts commit 53125f4d996ce5f5d7d93ee59649cac1a9fcf7cf. --- src/fees/compute_fees.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/fees/compute_fees.py b/src/fees/compute_fees.py index b2eef36..a811eac 100644 --- a/src/fees/compute_fees.py +++ b/src/fees/compute_fees.py @@ -587,6 +587,13 @@ def batch_fee_imbalances( ) -> tuple[dict[str, int], dict[str, int]]: onchain_data, offchain_data = fetch_settlement_data(tx_hash) protocol_fees, network_fees = compute_fee_imbalances(onchain_data, offchain_data) - protocol_fees = {token.hex(): fee for token, fee in protocol_fees.items()} - network_fees = {token.hex(): fee for token, fee in network_fees.items()} + protocol_fees = { + token.hex(): fee + for token, fee in protocol_fees.items() + } + network_fees = { + token.hex(): fee + for token, fee in network_fees.items() + } return protocol_fees, network_fees + From 1e1ea53e1baa614574724c9a6a6c398284cbe4f3 Mon Sep 17 00:00:00 2001 From: Shubh Agarwal Date: Thu, 22 Aug 2024 23:26:41 -0400 Subject: [PATCH 16/19] address comments --- src/fees/compute_fees.py | 32 +++++--------------------------- src/test_single_hash.py | 18 ++++++++++++------ 2 files changed, 17 insertions(+), 33 deletions(-) diff --git a/src/fees/compute_fees.py b/src/fees/compute_fees.py index c084b3f..89dcd9d 100644 --- a/src/fees/compute_fees.py +++ b/src/fees/compute_fees.py @@ -546,21 +546,17 @@ def fetch_settlement_data( return onchain_data, offchain_data -# computing fees - - def compute_fee_imbalances( onchain_data: OnchainSettlementData, offchain_data: OffchainSettlementData -) -> tuple[dict[HexBytes, int], dict[HexBytes, int]]: - protocol_fees: dict[HexBytes, int] = {} - network_fees: dict[HexBytes, int] = {} +) -> tuple[dict[str, int], dict[str, int]]: + protocol_fees: dict[str, int] = {} + network_fees: dict[str, int] = {} for trade in onchain_data.trades: # protocol fees fee_policies = offchain_data.trade_fee_policies[trade.order_uid] protocol_fee_amount = trade.protocol_fee(fee_policies) protocol_fee_token = trade.surplus_token() - protocol_fees[protocol_fee_token] = protocol_fee_amount - + protocol_fees[protocol_fee_token.to_0x_hex()] = protocol_fee_amount # network fees surplus_fee = trade.compute_surplus_fee() # in the surplus token network_fee = surplus_fee - protocol_fee_amount @@ -573,9 +569,7 @@ def compute_fee_imbalances( ) else: network_fee_sell = network_fee - - network_fees[trade.sell_token] = network_fee_sell - + network_fees[trade.sell_token.to_0x_hex()] = network_fee_sell return protocol_fees, network_fees @@ -587,20 +581,4 @@ def batch_fee_imbalances( ) -> tuple[dict[str, int], dict[str, int]]: onchain_data, offchain_data = fetch_settlement_data(tx_hash) protocol_fees, network_fees = compute_fee_imbalances(onchain_data, offchain_data) - protocol_fees = { - Web3.to_checksum_address(token.hex()): fee - for token, fee in protocol_fees.items() - } - network_fees = { - Web3.to_checksum_address(token.hex()): fee - for token, fee in network_fees.items() - } return protocol_fees, network_fees - - -# if __name__ == "__main__": -# tx_hash = HexBytes( -# "0xbd8cf4a21ad811cc3b9e49cff5e95563c3c2651b0ea41e0f8a7987818205c984" -# ) -# protocol_fees, network_fees = batch_fee_imbalances(tx_hash) -# print(protocol_fees) diff --git a/src/test_single_hash.py b/src/test_single_hash.py index da0efc3..351f97a 100644 --- a/src/test_single_hash.py +++ b/src/test_single_hash.py @@ -1,4 +1,5 @@ from hexbytes import HexBytes +from web3 import Web3 from src.imbalances_script import RawTokenImbalances from src.price_providers.price_feed import PriceFeed from src.fees.compute_fees import batch_fee_imbalances @@ -41,18 +42,23 @@ def calculate_slippage_in_eth(self, slippage: dict, tx_hash: str) -> dict: block_number = receipt.blockNumber for token_address, amt in slippage.items(): if amt != 0: - price_data = self.price_providers.get_price( - block_number, token_address - ) + price_params = { + "block_number": block_number, + "token_address": token_address, + "tx_hash": tx_hash, + } + price_data = self.price_providers.get_price(price_params) if price_data: price, _ = price_data - decimals = self._get_token_decimals(token_address) + decimals = self.get_token_decimals(token_address) slippage_in_eth = price * (amt / (10**decimals)) eth_slippage[token_address] = slippage_in_eth return eth_slippage - def _get_token_decimals(self, token_address: str) -> int: - contract = self.web3.eth.contract(address=token_address, abi=erc20_abi) + def get_token_decimals(self, token_address: str) -> int: + contract = self.web3.eth.contract( + address=Web3.to_checksum_address(token_address), abi=erc20_abi + ) return contract.functions.decimals().call() def log_results( From 7b5477757d1f9dcb8035fe5a860911b1d74a1890 Mon Sep 17 00:00:00 2001 From: harisang Date: Fri, 23 Aug 2024 16:23:59 +0300 Subject: [PATCH 17/19] use competition endpoint to compute fees --- src/fees/compute_fees.py | 339 ++++++++++++++------------------------- 1 file changed, 117 insertions(+), 222 deletions(-) diff --git a/src/fees/compute_fees.py b/src/fees/compute_fees.py index 89dcd9d..1518c3a 100644 --- a/src/fees/compute_fees.py +++ b/src/fees/compute_fees.py @@ -5,21 +5,13 @@ import math import os from typing import Any - - from dotenv import load_dotenv from eth_typing import Address from hexbytes import HexBytes -from web3 import Web3 -from web3.logs import DISCARD -from web3.types import TxData, TxReceipt, EventData from src.constants import ( - SETTLEMENT_CONTRACT_ADDRESS, REQUEST_TIMEOUT, ) -from contracts.gpv2_settlement_abi import gpv2_settlement_abi - import requests # types for trades @@ -32,7 +24,6 @@ class Trade: order_uid: HexBytes sell_amount: int buy_amount: int - owner: HexBytes sell_token: HexBytes buy_token: HexBytes limit_sell_amount: int @@ -40,6 +31,7 @@ class Trade: kind: str sell_token_clearing_price: int buy_token_clearing_price: int + fee_policies: list["FeePolicy"] def volume(self) -> int: """Compute volume of a trade in the surplus token""" @@ -70,20 +62,20 @@ def surplus(self) -> int: return current_limit_sell_amount - self.sell_amount raise ValueError(f"Order kind {self.kind} is invalid.") - def raw_surplus(self, fee_policies: list["FeePolicy"]) -> int: + def raw_surplus(self) -> int: """Compute raw surplus of a trade in the surplus token First, the application of protocol fees is reversed. Then, surplus of the resulting trade is computed.""" raw_trade = deepcopy(self) - for fee_policy in reversed(fee_policies): + for fee_policy in reversed(self.fee_policies): raw_trade = fee_policy.reverse_protocol_fee(raw_trade) return raw_trade.surplus() - def protocol_fee(self, fee_policies): + def protocol_fee(self): """Compute protocol fees of a trade in the surplus token Protocol fees are computed as the difference of raw surplus and surplus.""" - return self.raw_surplus(fee_policies) - self.surplus() + return self.raw_surplus() - self.surplus() def surplus_token(self) -> HexBytes: """Returns the surplus token""" @@ -270,116 +262,24 @@ def reverse_protocol_fee(self, trade: Trade) -> Trade: @dataclass -class OnchainSettlementData: - """Class to describe onchain info about a settlement.""" - - auction_id: int - tx_hash: HexBytes - solver: HexBytes - call_data: HexBytes - trades: list[Trade] - - -@dataclass -class OffchainSettlementData: - """Class to describe offchain info about a settlement.""" +class SettlementData: + """Class to describe info about a settlement.""" # pylint: disable=too-many-instance-attributes auction_id: int + tx_hash: HexBytes solver: HexBytes - call_data: HexBytes - trade_fee_policies: dict[HexBytes, list[FeePolicy]] - score: int - valid_orders: set[HexBytes] - jit_order_addresses: set[HexBytes] + trades: list[Trade] native_prices: dict[HexBytes, int] # fetching data -class BlockchainFetcher: - """ - Class to connect to a node and fetch/process onchain data. +class OrderbookFetcher: """ - - def __init__(self) -> None: - load_dotenv() - - node_url = os.getenv("NODE_URL") - self.node = Web3(Web3.HTTPProvider(node_url)) - - self.contract = self.node.eth.contract( - address=Address(HexBytes(SETTLEMENT_CONTRACT_ADDRESS)), - abi=gpv2_settlement_abi, - ) - - def get_onchain_data(self, tx_hash: HexBytes) -> OnchainSettlementData: - """ - This function can error since nodes are called. - """ - transaction = self.node.eth.get_transaction(tx_hash) - receipt = self.node.eth.wait_for_transaction_receipt(tx_hash) - decoded_data = self.decode_data(transaction, receipt) - return decoded_data - - def decode_data( - self, transaction: TxData, receipt: TxReceipt - ) -> OnchainSettlementData: - """ - Method that decodes a tx and returns a summary of this decoding. - """ - tx_hash = transaction["hash"] - call_data = transaction["input"] - if isinstance(call_data, str): - call_data = bytes.fromhex(call_data[2:]) - auction_id = int.from_bytes(call_data[-8:], byteorder="big") - - settlement_event = self.contract.events.Settlement().process_receipt( - receipt, errors=DISCARD - )[0] - solver = HexBytes(settlement_event["args"]["solver"]) - trade_events: tuple[EventData] = self.contract.events.Trade().process_receipt( - receipt, errors=DISCARD - ) - - settlement = self.contract.decode_function_input(call_data)[1] - - tokens = settlement["tokens"] - prices = settlement["clearingPrices"] - - trades: list[Trade] = [] - for trade_event, settlement_trade in zip(trade_events, settlement["trades"]): - sell_token_address = HexBytes(tokens[settlement_trade["sellTokenIndex"]]) - buy_token_address = HexBytes(tokens[settlement_trade["buyTokenIndex"]]) - sell_token_clearing_price = prices[ - tokens.index(tokens[settlement_trade["sellTokenIndex"]]) - ] - buy_token_clearing_price = prices[ - tokens.index(tokens[settlement_trade["buyTokenIndex"]]) - ] - trade = Trade( - HexBytes(trade_event["args"]["orderUid"]), - trade_event["args"]["sellAmount"], - trade_event["args"]["buyAmount"], - HexBytes(trade_event["args"]["owner"]), - sell_token_address, - buy_token_address, - settlement_trade["sellAmount"], - settlement_trade["buyAmount"], - "sell" if settlement_trade["flags"] % 2 == 0 else "buy", - sell_token_clearing_price, - buy_token_clearing_price, - ) - trades.append(trade) - - return OnchainSettlementData(auction_id, tx_hash, solver, call_data, trades) - - -class OrderbookAWSFetcher: - """ - This is a class for connecting to the db, and contains a few functions that + This is a class for connecting to the orderbook api, and contains a few functions that fetch necessary data to run the checks that we need. """ @@ -391,113 +291,113 @@ def __init__(self) -> None: "prod": f"https://api.cow.fi/{chain_name}/api/v1/", "barn": f"https://barn.api.cow.fi/{chain_name}/api/v1/", } - self.aws_urls = { - "prod": "https://solver-instances.s3.eu-central-1.amazonaws.com/" - f"prod/{chain_name}/autopilot/", - "barn": "https://solver-instances.s3.eu-central-1.amazonaws.com/" - f"staging/{chain_name}/autopilot/", - } - def get_offchain_data( - self, onchain_data: OnchainSettlementData - ) -> OffchainSettlementData: + def get_all_data(self, tx_hash: HexBytes) -> SettlementData: """ Method that fetches all necessary data from the API. """ - solver = onchain_data.solver - auction_id = onchain_data.auction_id - - solution_data, environment = self.get_solution_data(auction_id, solver) - auction_data = self.get_auction_data(auction_id, environment) - - offchain_data = self.convert_to_offchain_data( - onchain_data, - auction_data, - solution_data, - ) - return offchain_data - - def convert_to_offchain_data( - self, - onchain_data: OnchainSettlementData, - auction_data: dict[str, Any], - solution_data: dict[str, Any], - ) -> OffchainSettlementData: - """Turn Row from database query into OffchainSettlementData""" - # pylint: disable=too-many-locals - auction_id = onchain_data.auction_id - - solver = HexBytes(solution_data["solverAddress"]) - call_data = HexBytes(solution_data["callData"]) - - trade_fee_policies: dict[HexBytes, list[FeePolicy]] = {} - onchain_trades_dict = {trade.order_uid: trade for trade in onchain_data.trades} - protocol_fees_dict = { - HexBytes(order["uid"]): order["protocolFees"] - for order in auction_data["orders"] - if HexBytes(order["uid"]) in onchain_trades_dict - } - for order in solution_data["orders"]: - order_uid = HexBytes(order["id"]) - - fee_policies = self.parse_fee_policies( - protocol_fees_dict.get(order_uid, []) - ) - - trade_fee_policies[order_uid] = fee_policies - - score = int(solution_data["score"]) - valid_orders = {HexBytes(order["uid"]) for order in auction_data["orders"]} - native_prices = { + endpoint_data, environment = self.get_auction_data(tx_hash) + + solutions = endpoint_data["solutions"] + # here we detect the winning solution + for sol in solutions: + if sol["ranking"] == 1: + winning_sol = sol + + auction_id = endpoint_data["auctionId"] + solver = HexBytes(winning_sol["solverAddress"]) + + executed_orders = [ + (HexBytes(order["id"]), int(order["sellAmount"]), int(order["buyAmount"])) + for order in winning_sol["orders"] + ] + clearing_prices = { HexBytes(address): int(price) - for address, price in auction_data["prices"].items() + for address, price in winning_sol["clearingPrices"].items() } - jit_order_addresses = { - HexBytes(address) - for address in auction_data["surplusCapturingJitOrderOwners"] + native_prices = { + address: int(endpoint_data["auction"]["prices"][address.hex()]) + for address, _ in clearing_prices.items() } + trades = [] + for uid, executed_sell_amount, executed_buy_amount in executed_orders: + order_data = self.get_order_data(uid, environment) + if order_data == None: + # this can only happen for now if the order is a jit CoW AMM order + continue + trade_data = self.get_trade_data(uid, tx_hash, environment) + + kind = order_data["kind"] + sell_token = HexBytes(order_data["sellToken"]) + buy_token = HexBytes(order_data["buyToken"]) + limit_sell_amount = int(order_data["sellAmount"]) + limit_buy_amount = int(order_data["buyAmount"]) + sell_token_clearing_price = clearing_prices[sell_token] + buy_token_clearing_price = clearing_prices[buy_token] + fee_policies = self.parse_fee_policies(trade_data["feePolicies"]) - return OffchainSettlementData( - auction_id, - solver, - call_data, - trade_fee_policies, - score, - valid_orders, - jit_order_addresses, - native_prices, + trade = Trade( + order_uid=uid, + sell_amount=executed_sell_amount, + buy_amount=executed_buy_amount, + sell_token=sell_token, + buy_token=buy_token, + limit_sell_amount=limit_sell_amount, + limit_buy_amount=limit_buy_amount, + kind=kind, + sell_token_clearing_price=sell_token_clearing_price, + buy_token_clearing_price=buy_token_clearing_price, + fee_policies=fee_policies, + ) + trades.append(trade) + + settlement_data = SettlementData( + auction_id=auction_id, + tx_hash=tx_hash, + solver=solver, + trades=trades, + native_prices=native_prices, ) + return settlement_data - def get_solution_data( - self, auction_id: int, solver: HexBytes - ) -> tuple[dict[str, Any], str]: - """Fetch competition data from the database""" + def get_auction_data(self, tx_hash: HexBytes): for environment, url in self.orderbook_urls.items(): try: response = requests.get( - url + f"solver_competition/{auction_id}", + url + f"solver_competition/by_tx_hash/{tx_hash.hex()}", timeout=REQUEST_TIMEOUT, ) response.raise_for_status() - solution_data = response.json()["solutions"][-1] - if HexBytes(solution_data["solverAddress"]) == solver: - return solution_data, environment + auction_data = response.json() + return auction_data, environment except requests.exceptions.HTTPError as err: if err.response.status_code == 404: pass - raise ConnectionError(f"Error fetching off-chain data for id {auction_id}") - - def get_auction_data( - self, - auction_id: int, - environment: str, - ) -> dict[str, Any]: - """Fetch auction data from AWS.""" - url = self.aws_urls[environment] + f"{auction_id}.json" - response = requests.get(url, timeout=REQUEST_TIMEOUT) - response.raise_for_status() - auction_data: dict[str, Any] = response.json() - return auction_data + raise ConnectionError(f"Error fetching off-chain data for tx {tx_hash.hex()}") + + def get_order_data(self, uid: HexBytes, environment: str): + prefix = self.orderbook_urls[environment] + url = prefix + f"orders/{uid.hex()}" + response = requests.get( + url, + timeout=REQUEST_TIMEOUT, + ) + if response.ok == False: + # jit CoW AMM detected + return None + order_data = response.json() + return order_data + + def get_trade_data(self, uid: HexBytes, tx_hash: HexBytes, environment: str): + prefix = self.orderbook_urls[environment] + url = prefix + f"trades?orderUid={uid.hex()}" + response = requests.get(url) + trade_data_temp = response.json() + for t in trade_data_temp: + if HexBytes(t["txHash"]) == tx_hash: + trade_data = t + break + return trade_data def parse_fee_policies( self, protocol_fee_datum: list[dict[str, Any]] @@ -534,29 +434,19 @@ def parse_fee_policies( return fee_policies -def fetch_settlement_data( - tx_hash: HexBytes, -) -> tuple[OnchainSettlementData, OffchainSettlementData]: - onchain_fetcher = BlockchainFetcher() - offchain_fetcher = OrderbookAWSFetcher() - - onchain_data = onchain_fetcher.get_onchain_data(tx_hash) - offchain_data = offchain_fetcher.get_offchain_data(onchain_data) - - return onchain_data, offchain_data - - +# computing fees def compute_fee_imbalances( - onchain_data: OnchainSettlementData, offchain_data: OffchainSettlementData -) -> tuple[dict[str, int], dict[str, int]]: - protocol_fees: dict[str, int] = {} - network_fees: dict[str, int] = {} - for trade in onchain_data.trades: + settlement_data: SettlementData, +) -> tuple[dict[HexBytes, int], dict[HexBytes, int]]: + protocol_fees: dict[HexBytes, int] = {} + network_fees: dict[HexBytes, int] = {} + for trade in settlement_data.trades: # protocol fees - fee_policies = offchain_data.trade_fee_policies[trade.order_uid] - protocol_fee_amount = trade.protocol_fee(fee_policies) + fee_policies = trade.fee_policies + protocol_fee_amount = trade.protocol_fee() protocol_fee_token = trade.surplus_token() - protocol_fees[protocol_fee_token.to_0x_hex()] = protocol_fee_amount + protocol_fees[protocol_fee_token] = protocol_fee_amount + # network fees surplus_fee = trade.compute_surplus_fee() # in the surplus token network_fee = surplus_fee - protocol_fee_amount @@ -569,7 +459,9 @@ def compute_fee_imbalances( ) else: network_fee_sell = network_fee - network_fees[trade.sell_token.to_0x_hex()] = network_fee_sell + + network_fees[trade.sell_token] = network_fee_sell + return protocol_fees, network_fees @@ -579,6 +471,9 @@ def compute_fee_imbalances( def batch_fee_imbalances( tx_hash: HexBytes, ) -> tuple[dict[str, int], dict[str, int]]: - onchain_data, offchain_data = fetch_settlement_data(tx_hash) - protocol_fees, network_fees = compute_fee_imbalances(onchain_data, offchain_data) + orderbook_api = OrderbookFetcher() + settlement_data = orderbook_api.get_all_data(tx_hash) + protocol_fees, network_fees = compute_fee_imbalances(settlement_data) + protocol_fees = {token.hex(): fee for token, fee in protocol_fees.items()} + network_fees = {token.hex(): fee for token, fee in network_fees.items()} return protocol_fees, network_fees From 7dae420fcadfdb68c013e5ae2df6b1dacbb4cda2 Mon Sep 17 00:00:00 2001 From: harisang Date: Fri, 23 Aug 2024 16:33:35 +0300 Subject: [PATCH 18/19] revert some accidental changes --- src/fees/compute_fees.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/fees/compute_fees.py b/src/fees/compute_fees.py index 1518c3a..14bccad 100644 --- a/src/fees/compute_fees.py +++ b/src/fees/compute_fees.py @@ -437,16 +437,14 @@ def parse_fee_policies( # computing fees def compute_fee_imbalances( settlement_data: SettlementData, -) -> tuple[dict[HexBytes, int], dict[HexBytes, int]]: - protocol_fees: dict[HexBytes, int] = {} - network_fees: dict[HexBytes, int] = {} +) -> tuple[dict[str, int], dict[str, int]]: + protocol_fees: dict[str, int] = {} + network_fees: dict[str, int] = {} for trade in settlement_data.trades: # protocol fees - fee_policies = trade.fee_policies protocol_fee_amount = trade.protocol_fee() protocol_fee_token = trade.surplus_token() - protocol_fees[protocol_fee_token] = protocol_fee_amount - + protocol_fees[protocol_fee_token.to_0x_hex()] = protocol_fee_amount # network fees surplus_fee = trade.compute_surplus_fee() # in the surplus token network_fee = surplus_fee - protocol_fee_amount @@ -460,7 +458,7 @@ def compute_fee_imbalances( else: network_fee_sell = network_fee - network_fees[trade.sell_token] = network_fee_sell + network_fees[trade.sell_token.to_0x.hex()] = network_fee_sell return protocol_fees, network_fees @@ -474,6 +472,4 @@ def batch_fee_imbalances( orderbook_api = OrderbookFetcher() settlement_data = orderbook_api.get_all_data(tx_hash) protocol_fees, network_fees = compute_fee_imbalances(settlement_data) - protocol_fees = {token.hex(): fee for token, fee in protocol_fees.items()} - network_fees = {token.hex(): fee for token, fee in network_fees.items()} return protocol_fees, network_fees From 852b699a3e7ebfbe9056042c7d7b07fcb291f7b3 Mon Sep 17 00:00:00 2001 From: harisang Date: Fri, 23 Aug 2024 16:39:11 +0300 Subject: [PATCH 19/19] fix typo --- src/fees/compute_fees.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fees/compute_fees.py b/src/fees/compute_fees.py index 14bccad..3a8de0e 100644 --- a/src/fees/compute_fees.py +++ b/src/fees/compute_fees.py @@ -458,7 +458,7 @@ def compute_fee_imbalances( else: network_fee_sell = network_fee - network_fees[trade.sell_token.to_0x.hex()] = network_fee_sell + network_fees[trade.sell_token.to_0x_hex()] = network_fee_sell return protocol_fees, network_fees