Have a dummy in-RAM coordinator for the functional tests

.cirrus.yml

@@ -2,10 +2,11 @@ task:
   name: 'Functional tests'
   container:
     image: rust:latest
+  timeout_in: 90m  # https://cirrus-ci.org/faq/#instance-timed-out
 
   env:
-    EXECUTOR_WORKERS: 1
-    VERBOSE: 1
+    EXECUTOR_WORKERS: 3
+    VERBOSE: 0
     LOG_LEVEL: debug
     TIMEOUT: 300
     BITCOIND_VERSION: 22.0
@@ -47,7 +48,7 @@ task:
     folder: tests/servers/cosignerd
     fingerprint_script: git rev-parse HEAD:tests/servers/cosignerd
   servers_build_script: |
-    for server in "coordinatord" "miradord" "cosignerd"; do
+    for server in "miradord" "cosignerd"; do
       if [ ! -f tests/servers/$server/Cargo.lock ]; then
         git submodule update --init --recursive --remote ./tests/servers/$server
         cd ./tests/servers/$server
@@ -56,7 +57,7 @@ task:
       fi
     done
 
-  deps_script: apt update && apt install -y postgresql python3 python3-pip
+  deps_script: apt update && apt install -y python3 python3-pip
 
   pip_cache:
     folder: ~/.cache/pip
@@ -71,11 +72,6 @@ task:
     tar -xzf $ARCHIVE_NAME
     export BITCOIND_PATH=$BITCOIND_DIR_NAME/bin/bitcoind
 
-    # Setup the postgres instance for the servers
-    pg_ctlcluster 13 main start
-    su -c "psql -c \"CREATE ROLE test CREATEDB LOGIN PASSWORD 'test'\"" - postgres
-    export POSTGRES_USER=test POSTGRES_PASS=test
-
     # Run the functional tests
     REVAULTD_PATH=$PWD/target/release/revaultd pytest $TEST_GROUP -vvv -n 2
 

tests/README.md

@@ -39,18 +39,7 @@ variables: `POSTGRES_USER`, `POSTGRES_PASS` and optionally `POSTGRES_HOST` (if n
 `localhost`). The framework will take care of creating a database for each process
 launched, and to drop it at teardown time.  
 
-#### Without the servers
-
-You can write and run tests that don't need the servers. By default, if no `POSTGRES_*`
-environment variables are passed, the framework will skip the tests that depends on
-servers:
-
-```
-# Adapt `-n`, `-v`, `timeout` and other environment variables to your needs
-pytest tests/ -vvv -n4 --ignore tests/servers/
-```
-
-#### With the servers
+#### Compiling the servers
 
 For spinning up the servers, the framework will need access to the binaries. Therefore you
 must `init` the submodules (if you did not `clone` with `--recursive`) and compile the
@@ -59,21 +48,41 @@ servers code:
 # From the root of the repository
 git submodule update --init --recursive
 cd tests/servers
-cd coordinatord && cargo build && cd ..
 cd cosignerd && cargo build && cd ..
 cd miradord && cargo build && cd ..
 ```
 
-When you need a new version of the servers, you can update the submodules:
+When you need a new version of the servers, you can update the submodules. You can use a
+script from `contrib/` that updates the servers and recompiles them:
 ```
 # From the root of the repository
-cd tests/servers
-git submodule update --remote --recursive
+./contrib/recompile_tests_servers.sh
+```
+
+#### Using the real Coordinator
+
+By default, the functional tests will use a dummy in-RAM coordinator (see
+[`test_framework/coordinatord.py`](test_framework/coordinatord.py).
+
+The tests can be ran using `coordinatord`, and some require its use.
+
+In order to use it you'll first need to compile it:
+```
+# From the root of the repository
+git submodule update --init --recursive
+cd tests/servers/coordinatord && cargo build
+cd ../../../
 ```
 
-To run the server-requiring tests, pass the postgres credentials to the framework:
+And then you'll need to set a Postgre backend up. The easiest way to do so is by using Docker:
 ```
-POSTGRES_USER="test" POSTGRES_PASS="test" TEST_DEBUG=1 pytest -vvv -n8 --timeout=1800
+docker run --rm -d -p 5432:5432 --name postgres-coordinatord -e POSTGRES_PASSWORD=revaultd_tests -e POSTGRES_USER=revaultd_tests -e POSTGRES_DB=coordinator_db postgres:alpine
+```
+
+To run the tests with, pass the postgres credentials to the framework:
+```
+# From the root of the repository
+POSTGRES_USER=revaultd_tests POSTGRES_PASS=revaultd_tests pytest -vvv --ignore tests/servers -n 10
 ```
 
 
@@ -89,7 +98,7 @@ You can override the config at runtime with the `--log-cli-level` option:
 POSTGRES_USER=test POSTGRES_PASS=test pytest -vvv --log-cli-level=DEBUG -k test_getrevocationtxs
 ```
 
-Note that we log each daemon log, and we start them with `log_level = "trace"`.
+Note that we log each daemon log if `VERBOSE=1`, and we start them with `log_level = "debug"`.
 
 #### Profiling
 

tests/fixtures.py

@@ -224,12 +224,6 @@ def revaultd_manager(bitcoind, directory):
 
 @pytest.fixture
 def revault_network(directory, bitcoind, executor):
-    if not POSTGRES_IS_SETUP:
-        raise ValueError(
-            "Please set the POSTGRES_USER, POSTGRES_PASS and "
-            "POSTGRES_HOST environment variables."
-        )
-
     factory = RevaultNetwork(
         directory, bitcoind, executor, POSTGRES_USER, POSTGRES_PASS, POSTGRES_HOST
     )

tests/requirements.txt

@@ -6,6 +6,7 @@ bip32~=3.0
 bip380==0.0.3
 psycopg2-binary==2.9
 pynacl==1.4
+noiseprotocol==0.3.1
 
 # For the bitcoind proxy
 Flask==2.0.3

tests/test_chain.py

@@ -5,12 +5,10 @@
 """
 
 import logging
-import pytest
 
 from fixtures import *
 from test_framework import serializations
 from test_framework.utils import (
-    POSTGRES_IS_SETUP,
     wait_for,
 )
 
@@ -248,7 +246,6 @@ def reorg_deposit(revault_network, bitcoind, deposit, stop_wallets, target_statu
     # TODO: try with tx malleation
 
 
-@pytest.mark.skipif(not POSTGRES_IS_SETUP, reason="Needs Postgres for servers db")
 def test_reorged_deposit_status_1(revault_network, bitcoind):
     # NOTE: bitcoind would discard updating the mempool if the reorg is >10 blocks long.
     revault_network.deploy(4, 2, csv=12, with_watchtowers=False)
@@ -287,7 +284,6 @@ def test_reorged_deposit_status_1(revault_network, bitcoind):
     # TODO: same with 'emergency'
 
 
-@pytest.mark.skipif(not POSTGRES_IS_SETUP, reason="Needs Postgres for servers db")
 def test_reorged_deposit_status_2(revault_network, bitcoind):
     # NOTE: bitcoind would discard updating the mempool if the reorg is >10 blocks long.
     revault_network.deploy(4, 2, csv=3, with_watchtowers=False)
@@ -319,7 +315,6 @@ def test_reorged_deposit_status_2(revault_network, bitcoind):
     # TODO: same with 'unvault_emergency'
 
 
-@pytest.mark.skipif(not POSTGRES_IS_SETUP, reason="Needs Postgres for servers db")
 def test_reorged_unvault(revault_network, bitcoind):
     """Test various scenarii with reorgs around the Unvault transaction of a vault."""
     CSV = 12
@@ -513,7 +508,6 @@ def test_reorged_unvault(revault_network, bitcoind):
                 assert vault[field] is None, field
 
 
-@pytest.mark.skipif(not POSTGRES_IS_SETUP, reason="Needs Postgres for servers db")
 def test_reorged_spend(revault_network, bitcoind):
     CSV = 12
     revault_network.deploy(4, 2, csv=CSV, with_watchtowers=False)
@@ -561,7 +555,6 @@ def test_reorged_spend(revault_network, bitcoind):
                 assert vault[field] is None, field
 
 
-@pytest.mark.skipif(not POSTGRES_IS_SETUP, reason="Needs Postgres for servers db")
 def test_reorged_cancel(revault_network, bitcoind):
     revault_network.deploy(4, 2, csv=12, with_watchtowers=False)
     stks = revault_network.stks()
@@ -655,7 +648,6 @@ def test_reorged_cancel(revault_network, bitcoind):
             assert vault[field] is None, field
 
 
-@pytest.mark.skipif(not POSTGRES_IS_SETUP, reason="Needs Postgres for servers db")
 def test_retrieve_vault_status(revault_network, bitcoind):
     """Test we keep track of coins that moved without us actively noticing it."""
     CSV = 3

tests/test_framework/bitcoind.py

@@ -4,7 +4,7 @@
 import os
 import threading
 
-from cheroot.wsgi import PathInfoDispatcher, Server
+from cheroot.wsgi import Server
 from decimal import Decimal
 from ephemeral_port_reserve import reserve
 from flask import Flask, request, Response