Problem: integration tests CI spend a lot of time (crypto-org-chain#1271

) * Problem: integration tests CI spend a lot of time Solution: - run in parallel using markers and job matrix * fix matrix * flock is not nesserary * fix marker * break up ibc test cases * separate gov test cases * split out some slow test cases * split more ibc test * Update .github/workflows/test.yml Signed-off-by: yihuang <[email protected]> * fix pytest lint * more balance --------- Signed-off-by: yihuang <[email protected]>
mmsqe · Dec 22, 2023 · 93bf808 · 93bf808
1 parent 28b116b
commit 93bf808
Show file tree

Hide file tree

Showing 12 changed files with 48 additions and 3 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -16,6 +16,11 @@ jobs:
   integration_tests:
     runs-on: ubuntu-latest
     timeout-minutes: 240
+    strategy:
+      matrix:
+        tests: [unmarked, ibc, ibc_timeout, ibc_update_client, upgrade, slow]
+    env:
+      TESTS_TO_RUN: ${{ matrix.tests }}
     steps:
       - uses: actions/checkout@v3
         with:

diff --git a/Makefile b/Makefile
@@ -216,6 +216,12 @@ test-sim-profile:
 ###                                Integration Test                         ###
 ###############################################################################
 
+# possible values:
+# - all: run all integration tests
+# - unmarked: run integration tests that are not marked
+# - marker1,marker2: markers separated by comma, run integration tests that are marked with any of the markers
+TESTS_TO_RUN ?= all
+
 run-integration-tests:
 	@nix-shell ./integration_tests/shell.nix --run ./scripts/run-integration-tests
 

diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py
@@ -13,6 +13,16 @@
 def pytest_configure(config):
     config.addinivalue_line("markers", "slow: marks tests as slow")
     config.addinivalue_line("markers", "gravity: gravity bridge test cases")
+    config.addinivalue_line("markers", "upgrade: marks upgrade tests")
+    config.addinivalue_line("markers", "ibc: marks default ibc tests")
+    config.addinivalue_line("markers", "ibc_timeout: marks ibc timeout tests")
+    config.addinivalue_line("markers", "ibc_update_client: marks ibc updateclient test")
+
+
+def pytest_collection_modifyitems(items, config):
+    for item in items:
+        if not any(item.iter_markers()):
+            item.add_marker("unmarked")
 
 
 @pytest.fixture(scope="session")

diff --git a/integration_tests/test_exported_genesis.py b/integration_tests/test_exported_genesis.py
@@ -6,6 +6,8 @@
 from .network import setup_custom_cronos
 from .utils import ADDRS, CONTRACTS
 
+pytestmark = pytest.mark.slow
+
 
 @pytest.fixture(scope="module")
 def custom_cronos(tmp_path_factory):

diff --git a/integration_tests/test_ibc.py b/integration_tests/test_ibc.py
@@ -19,6 +19,8 @@
     wait_for_new_blocks,
 )
 
+pytestmark = pytest.mark.ibc
+
 
 @pytest.fixture(scope="module", params=[True, False])
 def ibc(request, tmp_path_factory):

diff --git a/integration_tests/test_ibc_timeout.py b/integration_tests/test_ibc_timeout.py
@@ -9,6 +9,8 @@
 )
 from .utils import ADDRS, eth_to_bech32, wait_for_fn
 
+pytestmark = pytest.mark.ibc_timeout
+
 
 @pytest.fixture(scope="module")
 def ibc(request, tmp_path_factory):

diff --git a/integration_tests/test_ibc_update_client.py b/integration_tests/test_ibc_update_client.py
@@ -5,6 +5,8 @@
 
 from .ibc_utils import prepare_network
 
+pytestmark = pytest.mark.ibc_update_client
+
 
 @pytest.fixture(scope="module")
 def ibc(request, tmp_path_factory):

diff --git a/integration_tests/test_mempool.py b/integration_tests/test_mempool.py
@@ -14,6 +14,8 @@
     wait_for_new_blocks,
 )
 
+pytestmark = pytest.mark.slow
+
 
 @pytest.fixture(scope="module")
 def cronos_mempool(tmp_path_factory):

diff --git a/integration_tests/test_pruned_node.py b/integration_tests/test_pruned_node.py
@@ -16,6 +16,8 @@
     wait_for_new_blocks,
 )
 
+pytestmark = pytest.mark.slow
+
 
 @pytest.fixture(scope="module")
 def cronos(request, tmp_path_factory):

diff --git a/integration_tests/test_replay_block.py b/integration_tests/test_replay_block.py
@@ -8,6 +8,8 @@
 from .network import setup_custom_cronos
 from .utils import ADDRS, CONTRACTS, KEYS, deploy_contract, sign_transaction
 
+pytestmark = pytest.mark.slow
+
 
 @pytest.fixture(scope="module")
 def custom_cronos(tmp_path_factory):
@@ -17,7 +19,7 @@ def custom_cronos(tmp_path_factory):
     )
 
 
-def test_replay_block(custom_cronos):
+def test_block_overflow(custom_cronos):
     w3: web3.Web3 = custom_cronos.w3
     contract = deploy_contract(
         w3,

diff --git a/integration_tests/test_upgrade.py b/integration_tests/test_upgrade.py
@@ -22,6 +22,8 @@
     wait_for_port,
 )
 
+pytestmark = pytest.mark.upgrade
+
 
 def init_cosmovisor(home):
     """

diff --git a/scripts/run-integration-tests b/scripts/run-integration-tests
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 set -e
 cd "$(dirname "$0")"
 
@@ -10,4 +10,12 @@ cd ../integration_tests/contracts
 HUSKY_SKIP_INSTALL=1 npm install
 npm run typechain
 cd ..
-nix-shell --run "pytest -vv -s"
+TESTS_TO_RUN="${TESTS_TO_RUN:-all}"
+if [[ "$TESTS_TO_RUN" == "all" ]]; then
+  echo "run all tests"
+  cmd="pytest -vv -s"
+else
+  echo "run tests matching $TESTS_TO_RUN"
+  cmd="pytest -vv -s -m '$TESTS_TO_RUN'"
+fi
+nix-shell --run "$cmd"
-Original file line number
+Diff line change
@@ Expand Up / @@ -22,6 +22,8 @@ @@
         wait_for_port,
     )
+    pytestmark = pytest.mark.upgrade
     def init_cosmovisor(home):
         """
@@ Expand Down @@