From 6003ca28f4f9f8c57a4684fe0f78a5dc33582494 Mon Sep 17 00:00:00 2001 From: dbeal Date: Fri, 27 Sep 2024 07:21:09 +0900 Subject: [PATCH] batch feed_ids to pyth api (#65) * batch feed_ids to pyth api another optimization that was forgotten about in the first pass * update batching * improve batching * version bump --------- Co-authored-by: Troy --- src/setup.py | 2 +- src/synthetix/perps/perps.py | 11 +- src/synthetix/pyth/pyth.py | 16 ++- src/synthetix/utils/multicall.py | 181 ++++++++++++++++++++++--------- 4 files changed, 147 insertions(+), 63 deletions(-) diff --git a/src/setup.py b/src/setup.py index 6c57068..f86265d 100644 --- a/src/setup.py +++ b/src/setup.py @@ -2,7 +2,7 @@ setup( name="synthetix", - version="0.1.19", + version="0.1.20", description="Synthetix protocol SDK", long_description=open("README.md").read(), long_description_content_type="text/markdown", diff --git a/src/synthetix/perps/perps.py b/src/synthetix/perps/perps.py index 8d08f29..a9aa934 100644 --- a/src/synthetix/perps/perps.py +++ b/src/synthetix/perps/perps.py @@ -7,7 +7,7 @@ call_erc7412, multicall_erc7412, write_erc7412, - make_fulfillment_request, + make_pyth_fulfillment_request, ) from .constants import DISABLED_MARKETS from .perps_utils import unpack_bfp_configuration, unpack_bfp_configuration_by_id @@ -230,15 +230,14 @@ def _prepare_oracle_call(self, market_names: [str] = []): price_metadata = pyth_data["meta"] # prepare the oracle call - raw_feed_ids = [decode_hex(feed_id) for feed_id in feed_ids] - args = (1, 30, raw_feed_ids) - - to, data, _ = make_fulfillment_request( + to, data, _ = make_pyth_fulfillment_request( self.snx, self.snx.contracts["pyth_erc7412_wrapper"]["PythERC7412Wrapper"]["address"], + 1, # update type + feed_ids, price_update_data, + 30, # staleness tolerance 0, - args, ) value = len(market_names) diff --git a/src/synthetix/pyth/pyth.py b/src/synthetix/pyth/pyth.py index 1b12988..afe3931 100644 --- a/src/synthetix/pyth/pyth.py +++ b/src/synthetix/pyth/pyth.py @@ -96,7 +96,17 @@ def _fetch_prices(self, feed_ids: [str], publish_time: int | None = None): :return: List of price update data :rtype: [bytes] | None """ - self.logger.info(f"Fetching Pyth data for {len(feed_ids)} markets") + market_names = ",".join( + [ + self.symbol_lookup[feed_id] + for feed_id in feed_ids + if feed_id in self.symbol_lookup + ] + ) + self.logger.info( + f"Fetching Pyth data for {len(feed_ids)} markets ({market_names}) @ {publish_time if publish_time else 'latest'}" + ) + self.logger.debug(f"Fetching data for feed ids: {feed_ids}") params = {"ids[]": feed_ids, "encoding": "hex"} @@ -113,9 +123,7 @@ def _fetch_prices(self, feed_ids: [str], publish_time: int | None = None): if response.text and "Price ids not found" in response.text: self.logger.info(f"Removing missing price feeds: {response.text}") feed_ids = [ - feed_id - for feed_id in feed_ids - if feed_id not in response.text + feed_id for feed_id in feed_ids if feed_id not in response.text ] return self._fetch_prices(feed_ids, publish_time=publish_time) diff --git a/src/synthetix/utils/multicall.py b/src/synthetix/utils/multicall.py index 7019395..9bfbdd2 100644 --- a/src/synthetix/utils/multicall.py +++ b/src/synthetix/utils/multicall.py @@ -1,3 +1,4 @@ +from eth_typing import HexStr import requests import base64 from web3.exceptions import ContractCustomError @@ -79,20 +80,30 @@ def decode_erc7412_oracle_data_required_error(snx, error): raise Exception("Error data can not be decoded") -def make_fulfillment_request(snx, address, price_update_data, fee, args): +def make_pyth_fulfillment_request( + snx, + address, + update_type, + feed_ids, + price_update_data, + publish_time_or_staleness, + fee, +): + # log all of the inputs erc_contract = snx.web3.eth.contract( address=address, abi=snx.contracts["pyth_erc7412_wrapper"]["PythERC7412Wrapper"]["abi"], ) - update_type, publish_time_or_staleness, feed_ids = args + # update_type, publish_time_or_staleness, feed_ids = args + feed_ids = [decode_hex(f) for f in feed_ids] encoded_args = encode( ["uint8", "uint64", "bytes32[]", "bytes[]"], [update_type, publish_time_or_staleness, feed_ids, price_update_data], ) # assume 1 wei per price update - value = fee if fee > 0 else len(price_update_data) * 1 + value = fee if fee > 0 else len(feed_ids) * 1 update_tx = erc_contract.functions.fulfillOracleQuery( encoded_args @@ -100,63 +111,65 @@ def make_fulfillment_request(snx, address, price_update_data, fee, args): return update_tx["to"], update_tx["data"], update_tx["value"] -def handle_erc7412_error(snx, error, calls): - "When receiving a ERC7412 error, will return an updated list of calls with the required price updates" +class PythVaaRequest: + feed_ids: list[HexStr] = [] + publish_time = 0 + fee = 0 + + +class ERC7412Requests: + pyth_address = "" + pyth_latest: list[HexStr] = [] + pyth_latest_fee = 0 + pyth_vaa: list[PythVaaRequest] = [] + + +def aggregate_erc7412_price_requests(snx, error, requests=None): + "Figures out all the prices that have been requested by an ERC7412 error and puts them all in aggregated requests" + if not requests: + requests = ERC7412Requests() if type(error) is ContractCustomError and error.data.startswith(SELECTOR_ERRORS): errors = decode_erc7412_errors_error(error.data) # TODO: execute in parallel for sub_error in errors: - sub_calls = handle_erc7412_error(snx, sub_error, []) - calls = sub_calls + calls + requests = aggregate_erc7412_price_requests(snx, sub_error, requests) - return calls + return requests if type(error) is ContractCustomError and ( error.data.startswith(SELECTOR_ORACLE_DATA_REQUIRED) or error.data.startswith(SELECTOR_ORACLE_DATA_REQUIRED_WITH_FEE) ): # decode error data - address, feed_ids, fee, args = decode_erc7412_oracle_data_required_error( - snx, error.data - ) - update_type = args[0] - - if update_type == 1: - # fetch the data from pyth for those feed ids - if not snx.is_fork: - pyth_data = snx.pyth.get_price_from_ids(feed_ids) - price_update_data = pyth_data["price_update_data"] - else: - # if it's a fork, get the price for the latest block - # this avoids providing "future" prices to the contract on a fork - block = snx.web3.eth.get_block("latest") - - # set a manual 60 second staleness - publish_time = block.timestamp - 60 - pyth_data = snx.pyth.get_price_from_ids( - feed_ids, publish_time=publish_time - ) - price_update_data = pyth_data["price_update_data"] - - # create a new request - to, data, value = make_fulfillment_request( - snx, address, price_update_data, fee, args - ) - elif update_type == 2: - # fetch the data from pyth for those feed ids - pyth_data = snx.pyth.get_price_from_ids(feed_ids, publish_time=args[1]) - price_update_data = pyth_data["price_update_data"] - - # create a new request - to, data, value = make_fulfillment_request( - snx, address, price_update_data, fee, args + update_type = None + address = "" + feed_ids = [] + fee = 0 + args = [] + try: + address, feed_ids, fee, args = decode_erc7412_oracle_data_required_error( + snx, error.data ) - else: - snx.logger.error(f"Unknown update type: {update_type}") - raise error - - calls = [(to, True, value, data)] + calls - return calls + update_type = args[0] + except: + pass + + if update_type: + requests.pyth_address = address + if update_type == 1: + # fetch the data from pyth for those feed ids + requests.pyth_latest = requests.pyth_latest + feed_ids + requests.pyth_latest_fee = requests.pyth_latest_fee + fee + elif update_type == 2: + # fetch the data from pyth for those feed ids + vaa_request = PythVaaRequest() + vaa_request.feed_ids = feed_ids + vaa_request.publish_time = args[1] + vaa_request.fee = fee + requests.pyth_vaa = requests.pyth_vaa + [vaa_request] + else: + snx.logger.error(f"Unknown update type: {update_type}") + raise error else: try: is_nonce_error = ( @@ -169,11 +182,75 @@ def handle_erc7412_error(snx, error, calls): if is_nonce_error: snx.logger.debug(f"Error is related to nonce, resetting nonce") snx.nonce = snx.web3.eth.get_transaction_count(snx.address) - return calls + return requests else: snx.logger.debug(f"Error is not related to oracle data") raise error + return requests + + +def handle_erc7412_error(snx, error): + "When receiving a ERC7412 error, will return an updated list of calls with the required price updates" + requests = aggregate_erc7412_price_requests(snx, error) + calls = [] + + if len(requests.pyth_latest) > 0: + # fetch the data from pyth for those feed ids + if not snx.is_fork: + pyth_data = snx.pyth.get_price_from_ids(requests.pyth_latest) + price_update_data = pyth_data["price_update_data"] + else: + # if it's a fork, get the price for the latest block + # this avoids providing "future" prices to the contract on a fork + block = snx.web3.eth.get_block("latest") + + # set a manual 60 second staleness + publish_time = block.timestamp - 60 + pyth_data = snx.pyth.get_price_from_ids( + requests.pyth_latest, publish_time=publish_time + ) + price_update_data = pyth_data["price_update_data"] + + # create a new request + # TODO: the actual number should go here for staleness + to, data, value = make_pyth_fulfillment_request( + snx, + requests.pyth_address, + 1, + requests.pyth_latest, + price_update_data, + 3600, + requests.pyth_latest_fee, + ) + + calls.append((to, True, value, data)) + + if len(requests.pyth_vaa) > 0: + for r in requests.pyth_vaa: + # fetch the data from pyth for those feed ids + pyth_data = snx.pyth.get_price_from_ids( + r.feed_ids, publish_time=r.publish_time + ) + price_update_data = pyth_data["price_update_data"] + + # create a new request + to, data, value = make_pyth_fulfillment_request( + snx, + requests.pyth_address, + 2, + r.feed_ids, + price_update_data, + r.publish_time, + r.fee, + ) + + calls.append((to, True, value, data)) + + # note: more calls (ex. new oracle providers) can be added here in the future + + return calls + def write_erc7412(snx, contract, function_name, args, tx_params={}, calls=[]): # prepare the initial call @@ -209,7 +286,7 @@ def write_erc7412(snx, contract, function_name, args, tx_params={}, calls=[]): snx.logger.debug(f"Simulation failed, decoding the error {e}") # handle the error by appending calls - calls = handle_erc7412_error(snx, e, calls) + calls = handle_erc7412_error(snx, e) + calls def call_erc7412(snx, contract, function_name, args, calls=[], block="latest"): @@ -244,7 +321,7 @@ def call_erc7412(snx, contract, function_name, args, calls=[], block="latest"): snx.logger.debug(f"Simulation failed, decoding the error {e}") # handle the error by appending calls - calls = handle_erc7412_error(snx, e, calls) + calls = handle_erc7412_error(snx, e) + calls def multicall_erc7412( @@ -297,4 +374,4 @@ def multicall_erc7412( snx.logger.debug(f"Simulation failed, decoding the error {e}") # handle the error by appending calls - calls = handle_erc7412_error(snx, e, calls) + calls = handle_erc7412_error(snx, e) + calls