batch feed_ids to pyth api (#65)

* batch feed_ids to pyth api

another optimization that was forgotten about in the first pass

* update batching

* improve batching

* version bump

---------

Co-authored-by: Troy <[email protected]>

src/setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name="synthetix",
-    version="0.1.19",
+    version="0.1.20",
     description="Synthetix protocol SDK",
     long_description=open("README.md").read(),
     long_description_content_type="text/markdown",

src/synthetix/perps/perps.py

@@ -7,7 +7,7 @@
     call_erc7412,
     multicall_erc7412,
     write_erc7412,
-    make_fulfillment_request,
+    make_pyth_fulfillment_request,
 )
 from .constants import DISABLED_MARKETS
 from .perps_utils import unpack_bfp_configuration, unpack_bfp_configuration_by_id
@@ -230,15 +230,14 @@ def _prepare_oracle_call(self, market_names: [str] = []):
             price_metadata = pyth_data["meta"]
 
         # prepare the oracle call
-        raw_feed_ids = [decode_hex(feed_id) for feed_id in feed_ids]
-        args = (1, 30, raw_feed_ids)
-
-        to, data, _ = make_fulfillment_request(
+        to, data, _ = make_pyth_fulfillment_request(
             self.snx,
             self.snx.contracts["pyth_erc7412_wrapper"]["PythERC7412Wrapper"]["address"],
+            1, # update type
+            feed_ids,
             price_update_data,
+            30, # staleness tolerance
             0,
-            args,
         )
         value = len(market_names)
 

src/synthetix/pyth/pyth.py

@@ -96,7 +96,17 @@ def _fetch_prices(self, feed_ids: [str], publish_time: int | None = None):
         :return: List of price update data
         :rtype: [bytes] | None
         """
-        self.logger.info(f"Fetching Pyth data for {len(feed_ids)} markets")
+        market_names = ",".join(
+            [
+                self.symbol_lookup[feed_id]
+                for feed_id in feed_ids
+                if feed_id in self.symbol_lookup
+            ]
+        )
+        self.logger.info(
+            f"Fetching Pyth data for {len(feed_ids)} markets ({market_names}) @ {publish_time if publish_time else 'latest'}"
+        )
+
         self.logger.debug(f"Fetching data for feed ids: {feed_ids}")
 
         params = {"ids[]": feed_ids, "encoding": "hex"}
@@ -113,9 +123,7 @@ def _fetch_prices(self, feed_ids: [str], publish_time: int | None = None):
                 if response.text and "Price ids not found" in response.text:
                     self.logger.info(f"Removing missing price feeds: {response.text}")
                     feed_ids = [
-                        feed_id
-                        for feed_id in feed_ids
-                        if feed_id not in response.text
+                        feed_id for feed_id in feed_ids if feed_id not in response.text
                     ]
                     return self._fetch_prices(feed_ids, publish_time=publish_time)
 

src/synthetix/utils/multicall.py

@@ -1,3 +1,4 @@
+from eth_typing import HexStr
 import requests
 import base64
 from web3.exceptions import ContractCustomError
@@ -79,84 +80,96 @@ def decode_erc7412_oracle_data_required_error(snx, error):
     raise Exception("Error data can not be decoded")
 
 
-def make_fulfillment_request(snx, address, price_update_data, fee, args):
+def make_pyth_fulfillment_request(
+    snx,
+    address,
+    update_type,
+    feed_ids,
+    price_update_data,
+    publish_time_or_staleness,
+    fee,
+):
+    # log all of the inputs
     erc_contract = snx.web3.eth.contract(
         address=address,
         abi=snx.contracts["pyth_erc7412_wrapper"]["PythERC7412Wrapper"]["abi"],
     )
 
-    update_type, publish_time_or_staleness, feed_ids = args
+    # update_type, publish_time_or_staleness, feed_ids = args
+    feed_ids = [decode_hex(f) for f in feed_ids]
     encoded_args = encode(
         ["uint8", "uint64", "bytes32[]", "bytes[]"],
         [update_type, publish_time_or_staleness, feed_ids, price_update_data],
     )
 
     # assume 1 wei per price update
-    value = fee if fee > 0 else len(price_update_data) * 1
+    value = fee if fee > 0 else len(feed_ids) * 1
 
     update_tx = erc_contract.functions.fulfillOracleQuery(
         encoded_args
     ).build_transaction({"value": value, "gas": None})
     return update_tx["to"], update_tx["data"], update_tx["value"]
 
 
-def handle_erc7412_error(snx, error, calls):
-    "When receiving a ERC7412 error, will return an updated list of calls with the required price updates"
+class PythVaaRequest:
+    feed_ids: list[HexStr] = []
+    publish_time = 0
+    fee = 0
+
+
+class ERC7412Requests:
+    pyth_address = ""
+    pyth_latest: list[HexStr] = []
+    pyth_latest_fee = 0
+    pyth_vaa: list[PythVaaRequest] = []
+
+
+def aggregate_erc7412_price_requests(snx, error, requests=None):
+    "Figures out all the prices that have been requested by an ERC7412 error and puts them all in aggregated requests"
+    if not requests:
+        requests = ERC7412Requests()
     if type(error) is ContractCustomError and error.data.startswith(SELECTOR_ERRORS):
         errors = decode_erc7412_errors_error(error.data)
 
         # TODO: execute in parallel
         for sub_error in errors:
-            sub_calls = handle_erc7412_error(snx, sub_error, [])
-            calls = sub_calls + calls
+            requests = aggregate_erc7412_price_requests(snx, sub_error, requests)
 
-        return calls
+        return requests
     if type(error) is ContractCustomError and (
         error.data.startswith(SELECTOR_ORACLE_DATA_REQUIRED)
         or error.data.startswith(SELECTOR_ORACLE_DATA_REQUIRED_WITH_FEE)
     ):
         # decode error data
-        address, feed_ids, fee, args = decode_erc7412_oracle_data_required_error(
-            snx, error.data
-        )
-        update_type = args[0]
-
-        if update_type == 1:
-            # fetch the data from pyth for those feed ids
-            if not snx.is_fork:
-                pyth_data = snx.pyth.get_price_from_ids(feed_ids)
-                price_update_data = pyth_data["price_update_data"]
-            else:
-                # if it's a fork, get the price for the latest block
-                # this avoids providing "future" prices to the contract on a fork
-                block = snx.web3.eth.get_block("latest")
-
-                # set a manual 60 second staleness
-                publish_time = block.timestamp - 60
-                pyth_data = snx.pyth.get_price_from_ids(
-                    feed_ids, publish_time=publish_time
-                )
-                price_update_data = pyth_data["price_update_data"]
-
-            # create a new request
-            to, data, value = make_fulfillment_request(
-                snx, address, price_update_data, fee, args
-            )
-        elif update_type == 2:
-            # fetch the data from pyth for those feed ids
-            pyth_data = snx.pyth.get_price_from_ids(feed_ids, publish_time=args[1])
-            price_update_data = pyth_data["price_update_data"]
-
-            # create a new request
-            to, data, value = make_fulfillment_request(
-                snx, address, price_update_data, fee, args
+        update_type = None
+        address = ""
+        feed_ids = []
+        fee = 0
+        args = []
+        try:
+            address, feed_ids, fee, args = decode_erc7412_oracle_data_required_error(
+                snx, error.data
             )
-        else:
-            snx.logger.error(f"Unknown update type: {update_type}")
-            raise error
-
-        calls = [(to, True, value, data)] + calls
-        return calls
+            update_type = args[0]
+        except:
+            pass
+
+        if update_type:
+            requests.pyth_address = address
+            if update_type == 1:
+                # fetch the data from pyth for those feed ids
+                requests.pyth_latest = requests.pyth_latest + feed_ids
+                requests.pyth_latest_fee = requests.pyth_latest_fee + fee
+            elif update_type == 2:
+                # fetch the data from pyth for those feed ids
+                vaa_request = PythVaaRequest()
+                vaa_request.feed_ids = feed_ids
+                vaa_request.publish_time = args[1]
+                vaa_request.fee = fee
+                requests.pyth_vaa = requests.pyth_vaa + [vaa_request]
+            else:
+                snx.logger.error(f"Unknown update type: {update_type}")
+                raise error
     else:
         try:
             is_nonce_error = (
@@ -169,11 +182,75 @@ def handle_erc7412_error(snx, error, calls):
         if is_nonce_error:
             snx.logger.debug(f"Error is related to nonce, resetting nonce")
             snx.nonce = snx.web3.eth.get_transaction_count(snx.address)
-            return calls
+            return requests
         else:
             snx.logger.debug(f"Error is not related to oracle data")
             raise error
 
+    return requests
+
+
+def handle_erc7412_error(snx, error):
+    "When receiving a ERC7412 error, will return an updated list of calls with the required price updates"
+    requests = aggregate_erc7412_price_requests(snx, error)
+    calls = []
+
+    if len(requests.pyth_latest) > 0:
+        # fetch the data from pyth for those feed ids
+        if not snx.is_fork:
+            pyth_data = snx.pyth.get_price_from_ids(requests.pyth_latest)
+            price_update_data = pyth_data["price_update_data"]
+        else:
+            # if it's a fork, get the price for the latest block
+            # this avoids providing "future" prices to the contract on a fork
+            block = snx.web3.eth.get_block("latest")
+
+            # set a manual 60 second staleness
+            publish_time = block.timestamp - 60
+            pyth_data = snx.pyth.get_price_from_ids(
+                requests.pyth_latest, publish_time=publish_time
+            )
+            price_update_data = pyth_data["price_update_data"]
+
+        # create a new request
+        # TODO: the actual number should go here for staleness
+        to, data, value = make_pyth_fulfillment_request(
+            snx,
+            requests.pyth_address,
+            1,
+            requests.pyth_latest,
+            price_update_data,
+            3600,
+            requests.pyth_latest_fee,
+        )
+
+        calls.append((to, True, value, data))
+
+    if len(requests.pyth_vaa) > 0:
+        for r in requests.pyth_vaa:
+            # fetch the data from pyth for those feed ids
+            pyth_data = snx.pyth.get_price_from_ids(
+                r.feed_ids, publish_time=r.publish_time
+            )
+            price_update_data = pyth_data["price_update_data"]
+
+            # create a new request
+            to, data, value = make_pyth_fulfillment_request(
+                snx,
+                requests.pyth_address,
+                2,
+                r.feed_ids,
+                price_update_data,
+                r.publish_time,
+                r.fee,
+            )
+
+            calls.append((to, True, value, data))
+
+    # note: more calls (ex. new oracle providers) can be added here in the future
+
+    return calls
+
 
 def write_erc7412(snx, contract, function_name, args, tx_params={}, calls=[]):
     # prepare the initial call
@@ -209,7 +286,7 @@ def write_erc7412(snx, contract, function_name, args, tx_params={}, calls=[]):
             snx.logger.debug(f"Simulation failed, decoding the error {e}")
 
             # handle the error by appending calls
-            calls = handle_erc7412_error(snx, e, calls)
+            calls = handle_erc7412_error(snx, e) + calls
 
 
 def call_erc7412(snx, contract, function_name, args, calls=[], block="latest"):
@@ -244,7 +321,7 @@ def call_erc7412(snx, contract, function_name, args, calls=[], block="latest"):
             snx.logger.debug(f"Simulation failed, decoding the error {e}")
 
             # handle the error by appending calls
-            calls = handle_erc7412_error(snx, e, calls)
+            calls = handle_erc7412_error(snx, e) + calls
 
 
 def multicall_erc7412(
@@ -297,4 +374,4 @@ def multicall_erc7412(
             snx.logger.debug(f"Simulation failed, decoding the error {e}")
 
             # handle the error by appending calls
-            calls = handle_erc7412_error(snx, e, calls)
+            calls = handle_erc7412_error(snx, e) + calls