Problem: integration tests CI spend a lot of time

Solution:
- run in parallel using markers and job matrix

.github/workflows/test.yml

@@ -17,6 +17,11 @@ jobs:
   integration_tests:
     runs-on: ubuntu-latest
     timeout-minutes: 240
+    strategy:
+      matrix:
+        tests: [unmarked ibc ica upgrade]
+    env:
+      TESTS_TO_RUN: ${{ matrix.tests }}
     steps:
       - uses: actions/checkout@v3
         with:

Makefile

@@ -219,6 +219,12 @@ test-sim-profile:
 ###                                Integration Test                         ###
 ###############################################################################
 
+# possible values:
+# - all: run all integration tests
+# - unmarked: run integration tests that are not marked
+# - marker1,marker2: markers separated by comma, run integration tests that are marked with any of the markers
+TESTS_TO_RUN ?= all
+
 run-integration-tests:
 	@make gen-bindings-contracts
 	@nix-shell ./integration_tests/shell.nix --run ./scripts/run-integration-tests

integration_tests/conftest.py

@@ -13,6 +13,15 @@
 def pytest_configure(config):
     config.addinivalue_line("markers", "slow: marks tests as slow")
     config.addinivalue_line("markers", "gravity: gravity bridge test cases")
+    config.addinivalue_line("markers", "ibc: marks ibc tests")
+    config.addinivalue_line("markers", "ica: marks ica tests")
+    config.addinivalue_line("markers", "upgrade: marks upgrade tests")
+
+
+def pytest_collection_modifyitems(items, config):
+    for item in items:
+        if not any(item.iter_markers()):
+            item.add_marker("unmarked")
 
 
 @pytest.fixture(scope="session")

integration_tests/test_ibc.py

@@ -12,6 +12,8 @@
 )
 from .utils import ADDRS, CONTRACTS, deploy_contract, send_transaction, wait_for_fn
 
+pytestmark = pytest.mark.gov
+
 
 @pytest.fixture(scope="module", params=[True, False])
 def ibc(request, tmp_path_factory):

integration_tests/test_ibc_rly.py

@@ -28,6 +28,8 @@
     wait_for_new_blocks,
 )
 
+pytestmark = pytest.mark.gov
+
 CONTRACT = "0x0000000000000000000000000000000000000065"
 contract_info = json.loads(CONTRACT_ABIS["IRelayerModule"].read_text())
 method_map = get_method_map(contract_info)

integration_tests/test_ibc_rly_gas.py

@@ -4,6 +4,8 @@
 from .ibc_utils import log_gas_records, prepare_network, rly_transfer
 from .utils import wait_for_new_blocks
 
+pytestmark = pytest.mark.gov
+
 
 @pytest.fixture(scope="module", params=["ibc_rly", "ibc_rly_evm"])
 def ibc(request, tmp_path_factory):

integration_tests/test_ibc_timeout.py

@@ -9,6 +9,8 @@
 )
 from .utils import ADDRS, eth_to_bech32, wait_for_fn
 
+pytestmark = pytest.mark.gov
+
 
 @pytest.fixture(scope="module")
 def ibc(request, tmp_path_factory):

integration_tests/test_ibc_update_client.py

@@ -6,6 +6,8 @@
 from .ibc_utils import prepare_network
 from .utils import approve_proposal, wait_for_fn
 
+pytestmark = pytest.mark.gov
+
 
 @pytest.fixture(scope="module")
 def ibc(request, tmp_path_factory):

integration_tests/test_ica.py

@@ -17,6 +17,8 @@
 )
 from .utils import CONTRACTS, wait_for_fn
 
+pytestmark = pytest.mark.ica
+
 
 @pytest.fixture(scope="module")
 def ibc(request, tmp_path_factory):

integration_tests/test_ica_precompile.py

@@ -27,6 +27,8 @@
     wait_for_fn,
 )
 
+pytestmark = pytest.mark.ica
+
 CONTRACT = "0x0000000000000000000000000000000000000066"
 connid = "connection-0"
 no_timeout = 300000000000

integration_tests/test_upgrade.py

@@ -24,6 +24,8 @@
     wait_for_port,
 )
 
+pytestmark = pytest.mark.upgrade
+
 
 @pytest.fixture(scope="module")
 def testnet(tmp_path_factory):

scripts/run-integration-tests

@@ -7,7 +7,17 @@ export TMPDIR=/tmp
 
 echo "build test contracts"
 cd ../integration_tests/contracts
+( flock -w 60 -n 9 || exit 1;
 HUSKY_SKIP_INSTALL=1 npm install
 npm run typechain
+) 9> lockfile
 cd ..
-nix-shell --run "pytest -vv -s"
+TESTS_TO_RUN="${TESTS_TO_RUN:-all}"
+if [[ "$TESTS_TO_RUN" == "all" ]]; then
+  echo "run all tests"
+  cmd="pytest -vv -s"
+else
+  echo "run tests matching $TESTS_TO_RUN"
+  cmd="pytest -vv -s -m '$TESTS_TO_RUN'"
+fi
+nix-shell --run "$cmd"