From 7afb5f2ec3174dfa031a183599015b6f37d9dbed Mon Sep 17 00:00:00 2001 From: HuangYi Date: Wed, 20 Dec 2023 16:06:28 +0800 Subject: [PATCH] Problem: integration tests CI spend a lot of time Solution: - run in parallel using markers and job matrix --- .github/workflows/test.yml | 5 +++++ Makefile | 6 ++++++ integration_tests/conftest.py | 9 +++++++++ integration_tests/test_ibc.py | 2 ++ integration_tests/test_ibc_rly.py | 2 ++ integration_tests/test_ibc_rly_gas.py | 2 ++ integration_tests/test_ibc_timeout.py | 2 ++ integration_tests/test_ibc_update_client.py | 2 ++ integration_tests/test_ica.py | 2 ++ integration_tests/test_ica_precompile.py | 2 ++ integration_tests/test_upgrade.py | 2 ++ scripts/run-integration-tests | 12 +++++++++++- 12 files changed, 47 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cfd942cb9c..58fd581095 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,6 +17,11 @@ jobs: integration_tests: runs-on: ubuntu-latest timeout-minutes: 240 + strategy: + matrix: + tests: [unmarked ibc ica upgrade] + env: + TESTS_TO_RUN: ${{ matrix.tests }} steps: - uses: actions/checkout@v3 with: diff --git a/Makefile b/Makefile index 98b786a746..573f820fc0 100644 --- a/Makefile +++ b/Makefile @@ -219,6 +219,12 @@ test-sim-profile: ### Integration Test ### ############################################################################### +# possible values: +# - all: run all integration tests +# - unmarked: run integration tests that are not marked +# - marker1,marker2: markers separated by comma, run integration tests that are marked with any of the markers +TESTS_TO_RUN ?= all + run-integration-tests: @make gen-bindings-contracts @nix-shell ./integration_tests/shell.nix --run ./scripts/run-integration-tests diff --git a/integration_tests/conftest.py b/integration_tests/conftest.py index 980d4a9627..1a74ee7754 100644 --- a/integration_tests/conftest.py +++ b/integration_tests/conftest.py @@ -13,6 +13,15 @@ def pytest_configure(config): config.addinivalue_line("markers", "slow: marks tests as slow") config.addinivalue_line("markers", "gravity: gravity bridge test cases") + config.addinivalue_line("markers", "ibc: marks ibc tests") + config.addinivalue_line("markers", "ica: marks ica tests") + config.addinivalue_line("markers", "upgrade: marks upgrade tests") + + +def pytest_collection_modifyitems(items, config): + for item in items: + if not any(item.iter_markers()): + item.add_marker("unmarked") @pytest.fixture(scope="session") diff --git a/integration_tests/test_ibc.py b/integration_tests/test_ibc.py index f7950c19c7..4d824a213b 100644 --- a/integration_tests/test_ibc.py +++ b/integration_tests/test_ibc.py @@ -12,6 +12,8 @@ ) from .utils import ADDRS, CONTRACTS, deploy_contract, send_transaction, wait_for_fn +pytestmark = pytest.mark.gov + @pytest.fixture(scope="module", params=[True, False]) def ibc(request, tmp_path_factory): diff --git a/integration_tests/test_ibc_rly.py b/integration_tests/test_ibc_rly.py index 3aa8e54b62..b42e2fea51 100644 --- a/integration_tests/test_ibc_rly.py +++ b/integration_tests/test_ibc_rly.py @@ -28,6 +28,8 @@ wait_for_new_blocks, ) +pytestmark = pytest.mark.gov + CONTRACT = "0x0000000000000000000000000000000000000065" contract_info = json.loads(CONTRACT_ABIS["IRelayerModule"].read_text()) method_map = get_method_map(contract_info) diff --git a/integration_tests/test_ibc_rly_gas.py b/integration_tests/test_ibc_rly_gas.py index 4d3e150bc8..4c4cad45a8 100644 --- a/integration_tests/test_ibc_rly_gas.py +++ b/integration_tests/test_ibc_rly_gas.py @@ -4,6 +4,8 @@ from .ibc_utils import log_gas_records, prepare_network, rly_transfer from .utils import wait_for_new_blocks +pytestmark = pytest.mark.gov + @pytest.fixture(scope="module", params=["ibc_rly", "ibc_rly_evm"]) def ibc(request, tmp_path_factory): diff --git a/integration_tests/test_ibc_timeout.py b/integration_tests/test_ibc_timeout.py index 98b9f611b1..1a6ef43161 100644 --- a/integration_tests/test_ibc_timeout.py +++ b/integration_tests/test_ibc_timeout.py @@ -9,6 +9,8 @@ ) from .utils import ADDRS, eth_to_bech32, wait_for_fn +pytestmark = pytest.mark.gov + @pytest.fixture(scope="module") def ibc(request, tmp_path_factory): diff --git a/integration_tests/test_ibc_update_client.py b/integration_tests/test_ibc_update_client.py index 3be46f8aa5..4f80ed4d3a 100644 --- a/integration_tests/test_ibc_update_client.py +++ b/integration_tests/test_ibc_update_client.py @@ -6,6 +6,8 @@ from .ibc_utils import prepare_network from .utils import approve_proposal, wait_for_fn +pytestmark = pytest.mark.gov + @pytest.fixture(scope="module") def ibc(request, tmp_path_factory): diff --git a/integration_tests/test_ica.py b/integration_tests/test_ica.py index 7c10d04a32..ba3ca2382a 100644 --- a/integration_tests/test_ica.py +++ b/integration_tests/test_ica.py @@ -17,6 +17,8 @@ ) from .utils import CONTRACTS, wait_for_fn +pytestmark = pytest.mark.ica + @pytest.fixture(scope="module") def ibc(request, tmp_path_factory): diff --git a/integration_tests/test_ica_precompile.py b/integration_tests/test_ica_precompile.py index 3474cbd3fa..437db524d0 100644 --- a/integration_tests/test_ica_precompile.py +++ b/integration_tests/test_ica_precompile.py @@ -27,6 +27,8 @@ wait_for_fn, ) +pytestmark = pytest.mark.ica + CONTRACT = "0x0000000000000000000000000000000000000066" connid = "connection-0" no_timeout = 300000000000 diff --git a/integration_tests/test_upgrade.py b/integration_tests/test_upgrade.py index 2d164541d9..72ed440c1e 100644 --- a/integration_tests/test_upgrade.py +++ b/integration_tests/test_upgrade.py @@ -24,6 +24,8 @@ wait_for_port, ) +pytestmark = pytest.mark.upgrade + @pytest.fixture(scope="module") def testnet(tmp_path_factory): diff --git a/scripts/run-integration-tests b/scripts/run-integration-tests index ffd159b251..9a9c007d96 100755 --- a/scripts/run-integration-tests +++ b/scripts/run-integration-tests @@ -7,7 +7,17 @@ export TMPDIR=/tmp echo "build test contracts" cd ../integration_tests/contracts +( flock -w 60 -n 9 || exit 1; HUSKY_SKIP_INSTALL=1 npm install npm run typechain +) 9> lockfile cd .. -nix-shell --run "pytest -vv -s" \ No newline at end of file +TESTS_TO_RUN="${TESTS_TO_RUN:-all}" +if [[ "$TESTS_TO_RUN" == "all" ]]; then + echo "run all tests" + cmd="pytest -vv -s" +else + echo "run tests matching $TESTS_TO_RUN" + cmd="pytest -vv -s -m '$TESTS_TO_RUN'" +fi +nix-shell --run "$cmd"