Fix integration tests (#3420)

* Make ganache listen on both IPv4 and IPv6

* Upgrade requirements

* Additional fixes

* Fix GitHub CI workflows 'Lint' and 'Golang CI'

.github/workflows/go.yml

@@ -6,18 +6,13 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go 1.x
-        uses: actions/setup-go@v2
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v3
         with:
-          go-version: ^1.18
-        id: go
-
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
-
+          go-version: 1.19
+      - uses: actions/checkout@v3
       - name: Build
         run: make install
-
       - name: Test
         run: |
           go test -v ./... -coverprofile=coverage.txt -covermode=atomic -coverpkg $(go list ./... | grep -v test | tr "\n" ",")
@@ -27,7 +22,6 @@ jobs:
             echo "Excluding ${filename} from coverage report..."
             sed -i.bak "/$(echo $filename | sed 's/\//\\\//g')/d" coverage.txt
           done
-
       - name: Upload coverage report
         uses: codecov/codecov-action@v3
         with:

.github/workflows/lint.yml

@@ -7,8 +7,15 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+      - name: Set GOROOT (fix for https://github.com/golangci/golangci-lint/issues/3107)
+        run: echo "GOROOT=$(go env GOROOT)" >> $GITHUB_ENV
+      - uses: actions/checkout@v3
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v2.5.2
+        uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.47.3
+          # TODO Upgrade golangci-lint version and fix errors reported by the new version
+          version: v1.47.3

test/integration/framework/requirements.txt

@@ -1,6 +1,6 @@
-grpcio-tools==1.47.0
-pytest==7.1.2
+grpcio-tools==1.54.0
+pytest==7.3.1
 PyYAML==6.0
 rusty-rlp==0.2.1
 toml==0.10.2
-web3==5.29.2
+web3==6.3.0

test/integration/framework/siftool

@@ -24,12 +24,12 @@ def init_venv(venv_dir, requirements_txt):
     execst([venv_pip, "install", "--upgrade", "pip"])
     execst([venv_pip, "install", "wheel"])
     execst([venv_pip, "install", "-r", requirements_txt])
-    if False and (sys.version_info.major > 3) or ((sys.version_info.major == 3) and (sys.version_info.minor > 9)):
-        # Temporary workaround for https://github.com/aaugustin/websockets/issues/916
-        # Needed only until a newer version of web3py is released.
-        # Due to a bug in web3/websockets siftool currently works with Python versions up to 3.9 only.
-        # You can use pyenv to install a compatible version of Python: https://github.com/pyenv/pyenv
-        execst([venv_pip, "install", "--upgrade", "websockets==10.3"])
+    # if False and (sys.version_info.major > 3) or ((sys.version_info.major == 3) and (sys.version_info.minor > 9)):
+    #     # Temporary workaround for https://github.com/aaugustin/websockets/issues/916
+    #     # Needed only until a newer version of web3py is released.
+    #     # Due to a bug in web3/websockets siftool currently works with Python versions up to 3.9 only.
+    #     # You can use pyenv to install a compatible version of Python: https://github.com/pyenv/pyenv
+    #     execst([venv_pip, "install", "--upgrade", "websockets==10.3"])
 
 
 def ensure_venv(venv_dir, requirements_txt, lock_file=None):

test/integration/framework/src/siftool/eth.py

@@ -61,7 +61,7 @@ def web3_wait_for_connection_up(w3_conn: web3.Web3, polling_time: int = 1, timeo
 def validate_address_and_private_key(addr: Optional[Address], private_key: Optional[PrivateKey]
 ) -> Tuple[Address, Optional[PrivateKey]]:
     a = web3.Web3().eth.account
-    addr = web3.Web3.toChecksumAddress(addr) if addr else None
+    addr = web3.Web3.to_checksum_address(addr) if addr else None
     if private_key:
         match_hex = re.match("^(0x)?([0-9a-fA-F]{64})$", private_key)
         private_key = match_hex[2].lower() if match_hex else _mnemonic_to_private_key(private_key)
@@ -105,14 +105,14 @@ def __init__(self, w3_conn, is_local_node):
         # self.ethereum_network_descriptor = None
 
     def _get_private_key(self, addr):
-        addr = web3.Web3.toChecksumAddress(addr)
+        addr = web3.Web3.to_checksum_address(addr)
         if addr not in self.private_keys:
             raise Exception(f"No private key set for address {addr}")
         return self.private_keys[addr]
 
     def set_private_key(self, addr: Address, private_key: PrivateKey):
         a = web3.Web3().eth.account
-        addr = web3.Web3.toChecksumAddress(addr)
+        addr = web3.Web3.to_checksum_address(addr)
         if private_key is None:
             self.private_keys.pop(addr)  # Remove
         else:
@@ -254,7 +254,7 @@ def _send_raw_transaction(self, smart_contract_call_obj, from_addr, tx_opts=None
 
         if smart_contract_call_obj is not None:
             # With no gas/gasPrice
-            tx = smart_contract_call_obj.buildTransaction(tx)
+            tx = smart_contract_call_obj.build_transaction(tx)
 
         private_key = self._get_private_key(from_addr)
         signed_tx = self.w3_conn.eth.account.sign_transaction(tx, private_key=private_key)

test/integration/framework/src/siftool/hardhat.py

@@ -147,7 +147,7 @@ def default_accounts():
     # Format: [address, private_key]
     # Note: for compatibility with ganache, private keys should be stripped of "0x" prefix
     # (when you pass a private key to ebrelayer via ETHEREUM_PRIVATE_KEY, the key is treated as invalid)
-    return [[web3.Web3.toChecksumAddress(address), private_key] for address, private_key in [[
+    return [[web3.Web3.to_checksum_address(address), private_key] for address, private_key in [[
         "0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266",
         "ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80",
     ], [

test/integration/framework/src/siftool/sifchain.py

@@ -80,7 +80,7 @@ def sifchain_denom_hash_to_token_contract_address(token_hash: str) -> Tuple[int,
     if not m:
         raise Exception("Invalid sifchain denom '{}'".format(token_hash))
     network_descriptor = int(m[1])
-    token_address = web3.Web3.toChecksumAddress(m[2])
+    token_address = web3.Web3.to_checksum_address(m[2])
     return network_descriptor, token_address
 
 # Deprecated
@@ -1348,13 +1348,13 @@ def __init__(self, message = None):
 
 
 def is_min_commission_too_low_exception(e: Exception):
-    patt = re.compile("^validator commission [\d.]+ cannot be lower than minimum of [\d.]+: invalid request$")
+    patt = re.compile("^validator commission [\\d.]+ cannot be lower than minimum of [\\d.]+: invalid request$")
     return (type(e) == SifnodedException) and patt.match(e.message)
 
 
 def is_max_voting_power_limit_exceeded_exception(e: Exception):
-    patt = re.compile("^This validator has a voting power of [\d.]+%. Delegations not allowed to a validator whose "
-        "post-delegation voting power is more than [\d.]+%. Please delegate to a validator with less bonded tokens: "
+    patt = re.compile("^This validator has a voting power of [\\d.]+%. Delegations not allowed to a validator whose "
+        "post-delegation voting power is more than [\\d.]+%. Please delegate to a validator with less bonded tokens: "
         "invalid request$")
     return (type(e) == SifnodedException) and patt.match(e.message)
 

test/integration/framework/src/siftool/test_utils.py

@@ -131,9 +131,9 @@ def get_env_ctx_peggy2():
 
         # TODO We're mixing "OPERATOR" vs. "OWNER"
         # TODO Addressses from dot_env_vars are not in correct EIP55 "checksum" format
-        # operator_address = web3.Web3.toChecksumAddress(dot_env_vars["ETH_ACCOUNT_OPERATOR_ADDRESS"])
+        # operator_address = web3.Web3.to_checksum_address(dot_env_vars["ETH_ACCOUNT_OPERATOR_ADDRESS"])
         # operator_private_key = dot_env_vars["ETH_ACCOUNT_OPERATOR_PRIVATEKEY"][2:]
-        owner_address = web3.Web3.toChecksumAddress(dot_env_vars["ETH_ACCOUNT_OWNER_ADDRESS"])
+        owner_address = web3.Web3.to_checksum_address(dot_env_vars["ETH_ACCOUNT_OWNER_ADDRESS"])
         owner_private_key = dot_env_vars.get("ETH_ACCOUNT_OWNER_PRIVATEKEY")
         if (owner_private_key is not None) and (owner_private_key.startswith("0x")):
             owner_private_key = owner_private_key[2:]  # TODO Remove
@@ -320,7 +320,7 @@ def _get_overrides_for_smart_contract_addresses(env_vars):
         "BridgeToken": "BRIDGE_TOKEN_ADDRESS",  # Peggy1 only
         # Missing "Blocklist" missing (Peggy2 only)
     }
-    return dict(((k, web3.Web3.toChecksumAddress(env_vars[v])) for k, v in mappings.items() if v in env_vars))
+    return dict(((k, web3.Web3.to_checksum_address(env_vars[v])) for k, v in mappings.items() if v in env_vars))
 
 
 def sif_addr_to_evm_arg(sif_address):
@@ -408,7 +408,7 @@ def tx_get_sc_at(self, sc_name, address):
     def smart_contract_get_past_events(self, sc, event_name, from_block=None, to_block=None):
         from_block = from_block if from_block is not None else 1
         to_block = str(to_block) if to_block is not None else "latest"
-        filter = sc.events[event_name].createFilter(fromBlock=from_block, toBlock=to_block)
+        filter = sc.events[event_name].create_filter(fromBlock=from_block, toBlock=to_block)
         try:
             return filter.get_all_entries()
         finally:
@@ -534,7 +534,7 @@ def get_whitelisted_tokens_from_bridge_bank_past_events(self):
         for e in past_events:
             token_addr = e.args["_token"]
             value = e.args["_value"]
-            assert web3.Web3.toChecksumAddress(token_addr) == token_addr
+            assert web3.Web3.to_checksum_address(token_addr) == token_addr
             # Logically the whitelist only consists of entries that have the last value of True.
             # If the data is clean, then for each token_addr we should first see a True event, possibly
             # followed by alternating False and True. The last value is the active one.
@@ -605,7 +605,7 @@ def send_erc20_from_ethereum_to_sifchain(self, from_eth_addr, dest_sichain_addr,
         # # When transfering ERC20, the amount needs to be passed as argument, and the "message.value" should be 0
         # # TODO Error handling
         # #      "web3.exceptions.ContractLogicError: execution reverted: SafeERC20: low-level call failed" in case that amount is more than what is available / what was "approved" to BridgeBank
-        # tx = bridge_bank.functions.lock(recipient, erc20_token_addr, amount).buildTransaction({
+        # tx = bridge_bank.functions.lock(recipient, erc20_token_addr, amount).build_transaction({
         #     "from": self.operator,
         #     "nonce": self.w3_conn.eth.get_transaction_count(self.operator)
         # })
@@ -787,7 +787,7 @@ def send_from_ethereum_to_sifchain(self, from_eth_acct: str, to_sif_acct: str, a
     # Peggy1-specific
     def set_ofac_blocklist_to(self, addrs):
         blocklist_sc = self.get_blocklist_sc()
-        addrs = [web3.Web3.toChecksumAddress(addr) for addr in addrs]
+        addrs = [web3.Web3.to_checksum_address(addr) for addr in addrs]
         existing_entries = blocklist_sc.functions.getFullList().call()
         to_add = [addr for addr in addrs if addr not in existing_entries]
         to_remove = [addr for addr in existing_entries if addr not in addrs]

test/integration/ganache_start.sh

@@ -27,7 +27,7 @@ done
 
 # ganache really hates running in the background.  Put it in a tmux session to keep all its input code happy.
 # If you don't do this, ganache-cli will just exit.
-nohup tmux new-session -d -s my_session "ganache-cli ${block_delay} -h 0.0.0.0 --mnemonic 'candy maple cake sugar pudding cream honey rich smooth crumble sweet treat' --networkId '5777' --port '7545' --db ${GANACHE_DB_DIR} --account_keys_path $GANACHE_KEYS_JSON > $GANACHE_LOG 2>&1"
+nohup tmux new-session -d -s my_session "ganache-cli ${block_delay} -h :: --mnemonic 'candy maple cake sugar pudding cream honey rich smooth crumble sweet treat' --networkId '5777' --port '7545' --db ${GANACHE_DB_DIR} --account_keys_path $GANACHE_KEYS_JSON > $GANACHE_LOG 2>&1"
 
 # wait for ganache to come up
 sleep 5

test/integration/src/py/test_ofac_blocklist.py

@@ -43,7 +43,7 @@ def test_blocklist_eth(ctx):
 
     bridge_bank = ctx.get_bridge_bank_sc()
 
-    filter = bridge_bank.events.LogLock.createFilter(fromBlock="latest")
+    filter = bridge_bank.events.LogLock.create_filter(fromBlock="latest")
 
     # Valid negative test outcome: transaction rejected with the string "Address is blocklisted"
     def assert_blocked(addr):
@@ -123,7 +123,7 @@ def test_blocklist_erc20(ctx):
 
     to_sif_acct = ctx.create_sifchain_addr()
 
-    filter = bridge_bank.events.LogLock.createFilter(fromBlock="latest")
+    filter = bridge_bank.events.LogLock.create_filter(fromBlock="latest")
 
     def assert_blocked(addr):
         assert len(filter.get_new_entries()) == 0

test/integration/src/py/test_siftool_framework.py

@@ -1,5 +1,5 @@
 import logging
-import web3
+import pytest
 
 import siftool_path
 from siftool import eth
@@ -24,7 +24,7 @@
 def test_sanity_checks(ctx):
     ctx.sanity_check()
 
-def test_eth_fee_functions(ctx):
+def test_eth_fee_functions_1(ctx):
     e = ctx.eth.w3_conn.eth
     null_txn = {"to": eth.NULL_ADDRESS}
     gas_price = None
@@ -34,13 +34,6 @@ def test_eth_fee_functions(ctx):
         assert ctx.eth.is_contract_logic_error_method_not_found(ex)
     if gas_price is not None:
         assert gas_price > 1 * eth.GWEI
-    max_priority_fee = None
-    try:
-        max_priority_fee = e.max_priority_fee
-    except Exception as ex:
-        assert ctx.eth.is_contract_logic_error_method_not_found(ex, "eth_maxPriorityFeePerGas")
-    if max_priority_fee is not None:
-        assert max_priority_fee >= 1 * eth.GWEI
     fee_history = None
     try:
         fee_history = e.fee_history(1, "latest", [25])
@@ -60,17 +53,30 @@ def test_eth_fee_functions(ctx):
         assert e.estimate_gas(null_txn) == eth.MIN_TX_GAS
     assert gas_price is not None
     if ctx.eth.is_local_node:
-        assert max_priority_fee is None
         assert (fee_history is None) == (not is_hardhat)
         assert ctx.eth.gas_estimate_fn is None
         assert bool(ctx.eth.fixed_gas_args)
     else:
-        assert max_priority_fee is not None
         assert fee_history is not None
         assert ctx.eth.gas_estimate_fn is not None
         gas, max_fee_per_gas, max_priority_fee_per_gas, gas_price = ctx.eth.gas_estimate_fn(null_txn)
     return
 
+@pytest.mark.filterwarnings("ignore::UserWarning")
+def test_eth_fee_functions_2(ctx):
+    e = ctx.eth.w3_conn.eth
+    max_priority_fee = None
+    try:
+        max_priority_fee = e.max_priority_fee
+    except Exception as ex:
+        assert ctx.eth.is_contract_logic_error_method_not_found(ex, "eth_maxPriorityFeePerGas")
+    if ctx.eth.is_local_node:
+        assert max_priority_fee is None
+    else:
+        assert max_priority_fee is not None
+        assert max_priority_fee >= 1 * eth.GWEI
+
+
 def test_send_ether(ctx):
     operator = ctx.operator