Make test suites quiet(er)

everyvoice/cli.py

@@ -1,3 +1,4 @@
+import enum
 import json
 import platform
 import subprocess
@@ -47,6 +48,7 @@
     synthesize as synthesize_hfg,
 )
 from everyvoice.model.vocoder.HiFiGAN_iSTFT_lightning.hfgl.cli import train as train_hfg
+from everyvoice.run_tests import SUITE_NAMES, run_tests
 from everyvoice.utils import generic_psv_filelist_reader, spinner
 from everyvoice.wizard import (
     ALIGNER_CONFIG_FILENAME_PREFIX,
@@ -556,23 +558,13 @@
 )(inspect_checkpoint)
 
 
-class TestSuites(str, Enum):
-    all = "all"
-    cli = "cli"
-    config = "config"
-    dev = "dev"
-    model = "model"
-    preprocessing = "preprocessing"
-    text = "text"
-    fs2 = "fs2"
+TestSuites = enum.Enum("TestSuites", {name: name for name in SUITE_NAMES})  # type: ignore
 
 
 @app.command(hidden=True)
-def test(suite: TestSuites = typer.Argument(TestSuites.dev)):
+def test(suite: TestSuites = typer.Argument("dev")):
     """Run a test suite"""
-    from everyvoice.run_tests import run_tests
-
-    run_tests(suite)
+    run_tests(suite.value)
 
 
 # Deferred full initialization to optimize the CLI, but still exposed for unit testing.

everyvoice/run_tests.py

@@ -3,6 +3,7 @@
 """ Organize tests into Test Suites
 """
 
+import argparse
 import importlib
 import os
 import re
@@ -21,7 +22,7 @@
 SUITES: dict[str, tuple[str, ...]] = {
     "config": ("test_configs",),
     "loader": ("test_dataloader",),
-    "text": ("test_text", "test_utils"),
+    "text": ("test_text", "test_utils", "test_doctests"),
     "preprocessing": ("test_preprocessing",),
     "model": ("test_model",),
     "cli": ("test_wizard", "test_cli", "test_wizard_helpers"),
@@ -39,6 +40,7 @@
     "fs2",
     "wav2vec2aligner",
 )
+SUITE_NAMES = ["all", "dev"] + sorted(SUITES.keys())
 SUITES["dev"] = sum((SUITES[suite] for suite in dev_suites), start=())
 
 
@@ -86,7 +88,7 @@ def describe_suite(suite: TestSuite):
     )
 
 
-def run_tests(suite: str, describe: bool = False):
+def run_tests(suite: str, describe: bool = False, verbosity=3):
     """Decide which Test Suite to run"""
     logger.info(f"Loading test suite '{suite}'. This may take a while...")
     if suite == "all":
@@ -98,7 +100,7 @@ def run_tests(suite: str, describe: bool = False):
             tests = SUITES[suite]
         else:
             logger.error(
-                f"Please specify a test suite to run: one of '{['all'] + sorted(SUITES.keys())}'."
+                f"Please specify a test suite to run: one of '{['all'] + SUITE_NAMES}'."
             )
             return False
         tests = [
@@ -123,19 +125,27 @@ def run_tests(suite: str, describe: bool = False):
         return True
     else:
         logger.info("Running test suite")
-        return TextTestRunner(verbosity=3).run(test_suite).wasSuccessful()
+        return TextTestRunner(verbosity=verbosity).run(test_suite).wasSuccessful()
 
 
-if __name__ == "__main__":
-    describe = "--describe" in sys.argv
-    if describe:
-        sys.argv.remove("--describe")
-
-    try:
-        suite = sys.argv[1]
-    except IndexError:
-        logger.info('No test suite specified, defaulting to "dev"')
-        suite = "dev"
-    result = run_tests(suite, describe)
+def main():
+    parser = argparse.ArgumentParser(description="Run EveryVoice test suites.")
+    parser.add_argument("--quiet", "-q", action="store_true", help="reduce output")
+    parser.add_argument(
+        "--describe", action="store_true", help="describe the selected test suite"
+    )
+    parser.add_argument(
+        "suite",
+        nargs="?",
+        default="dev",
+        help="the test suite to run [dev]",
+        choices=SUITE_NAMES,
+    )
+    args = parser.parse_args()
+    result = run_tests(args.suite, args.describe, 1 if args.quiet else 3)
     if not result:
         sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

everyvoice/tests/preprocessed_audio_fixture.py

@@ -8,6 +8,7 @@
 from everyvoice.model.e2e.config import FeaturePredictionConfig
 from everyvoice.preprocessor import Preprocessor
 from everyvoice.tests.basic_test_case import BasicTestCase
+from everyvoice.tests.stubs import capture_stderr, capture_stdout, mute_logger
 from everyvoice.utils import collapse_whitespace, lower, nfc_normalize
 
 
@@ -66,12 +67,17 @@ def setUpClass(cls):
         """Generate a preprocessed test set that can be used in various test cases."""
         # We only need to actually run this once
         if not PreprocessedAudioFixture._preprocess_ran:
-            PreprocessedAudioFixture.preprocessor.preprocess(
-                output_path=str(PreprocessedAudioFixture.lj_filelist),
-                cpus=1,
-                overwrite=True,
-                to_process=("audio", "energy", "pitch", "text", "spec"),
-            )
+            with (
+                mute_logger("everyvoice.preprocessor"),
+                capture_stderr(),
+                capture_stdout(),
+            ):
+                PreprocessedAudioFixture.preprocessor.preprocess(
+                    output_path=str(PreprocessedAudioFixture.lj_filelist),
+                    cpus=1,
+                    overwrite=True,
+                    to_process=("audio", "energy", "pitch", "text", "spec"),
+                )
             PreprocessedAudioFixture.lj_preprocessed.mkdir(parents=True, exist_ok=True)
             try:
                 (PreprocessedAudioFixture.lj_preprocessed / "duration").symlink_to(

everyvoice/tests/stubs.py

@@ -2,6 +2,7 @@
 import io
 import logging
 import os
+import sys
 from contextlib import contextmanager, redirect_stderr, redirect_stdout
 from pathlib import Path
 from typing import Any, Generator, Sequence, Union
@@ -321,3 +322,49 @@ def __enter__(self):
     def __exit__(self, *_exc_info):
         for monkey in self.monkeys:
             monkey.__exit__(*_exc_info)
+
+
+@contextmanager
+def silence_c_stdout():
+    """Capture stdout from C output, e.g., from SoundSwallower.
+
+    Note: to capture stdout for both C and Python code, combine this with
+    redirect_stdout(), but you must use capture_c_stdout() first:
+        with capture_c_stdout(), redirect_stdout(io.StringIO()):
+            # code
+
+    Loosely inspired by https://stackoverflow.com/a/24277852, but much simplified to
+    address our narrow needs, namely to silence stdout in a context manager.
+    """
+
+    stdout_fileno = sys.stdout.fileno()
+    stdout_save = os.dup(stdout_fileno)
+    stdout_fd = os.open(os.devnull, os.O_RDWR)
+    os.dup2(stdout_fd, stdout_fileno)
+    yield
+    os.dup2(stdout_save, stdout_fileno)
+    os.close(stdout_save)
+    os.close(stdout_fd)
+
+
+@contextmanager
+def silence_c_stderr():
+    """Capture stderr from C output, e.g., from SoundSwallower.
+
+    Note: to capture stderr for both C and Python code, combine this with
+    redirect_stderr(), but you must use capture_c_stderr() first:
+        with capture_c_stderr(), redirect_stderr(io.StringIO()):
+            # code
+
+    Loosely inspired by https://stackoverflow.com/a/24277852, but much simplified to
+    address our narrow needs, namely to silence stderr in a context manager.
+    """
+
+    stderr_fileno = sys.stderr.fileno()
+    stderr_save = os.dup(stderr_fileno)
+    stderr_fd = os.open(os.devnull, os.O_RDWR)
+    os.dup2(stderr_fd, stderr_fileno)
+    yield
+    os.dup2(stderr_save, stderr_fileno)
+    os.close(stderr_save)
+    os.close(stderr_fd)

everyvoice/tests/test_cli.py

@@ -33,7 +33,7 @@
 )
 from everyvoice.model.vocoder.HiFiGAN_iSTFT_lightning.hfgl.config import HiFiGANConfig
 from everyvoice.model.vocoder.HiFiGAN_iSTFT_lightning.hfgl.model import HiFiGAN
-from everyvoice.tests.stubs import capture_logs, capture_stdout, mute_logger
+from everyvoice.tests.stubs import capture_logs, capture_stdout, silence_c_stderr
 from everyvoice.wizard import (
     SPEC_TO_WAV_CONFIG_FILENAME_PREFIX,
     TEXT_TO_SPEC_CONFIG_FILENAME_PREFIX,
@@ -175,11 +175,12 @@ def test_commands_present(self):
         self.assertIn("https://docs.everyvoice.ca", result.stdout)
 
     def test_command_help_messages(self):
-        for command in self.commands:
-            result = self.runner.invoke(app, [command, "--help"])
-            self.assertEqual(result.exit_code, 0)
-            result = self.runner.invoke(app, [command, "-h"])
-            self.assertEqual(result.exit_code, 0)
+        with silence_c_stderr():
+            for command in self.commands:
+                result = self.runner.invoke(app, [command, "--help"])
+                self.assertEqual(result.exit_code, 0)
+                result = self.runner.invoke(app, [command, "-h"])
+                self.assertEqual(result.exit_code, 0)
 
     def test_update_schema(self):
         dummy_contact = ContactInformation(
@@ -233,16 +234,17 @@ def test_update_schema(self):
         self.assertIn("FileExistsError", str(result))
 
     def test_evaluate(self):
-        result = self.runner.invoke(
-            app,
-            [
-                "evaluate",
-                "-f",
-                self.data_dir / "LJ010-0008.wav",
-                "-r",
-                self.data_dir / "lj" / "wavs" / "LJ050-0269.wav",
-            ],
-        )
+        with silence_c_stderr():
+            result = self.runner.invoke(
+                app,
+                [
+                    "evaluate",
+                    "-f",
+                    self.data_dir / "LJ010-0008.wav",
+                    "-r",
+                    self.data_dir / "lj" / "wavs" / "LJ050-0269.wav",
+                ],
+            )
         self.assertEqual(result.exit_code, 0)
         self.assertIn("LJ010-0008", result.stdout)
         self.assertIn("STOI", result.stdout)
@@ -272,13 +274,15 @@ def test_evaluate(self):
         )
 
     def test_inspect_checkpoint_help(self):
-        result = self.runner.invoke(app, ["inspect-checkpoint", "--help"])
+        with silence_c_stderr():
+            result = self.runner.invoke(app, ["inspect-checkpoint", "--help"])
         self.assertIn("inspect-checkpoint [OPTIONS] MODEL_PATH", result.stdout)
 
     def test_inspect_checkpoint(self):
-        result = self.runner.invoke(
-            app, ["inspect-checkpoint", str(self.data_dir / "test.ckpt")]
-        )
+        with silence_c_stderr():
+            result = self.runner.invoke(
+                app, ["inspect-checkpoint", str(self.data_dir / "test.ckpt")]
+            )
         self.assertIn('global_step": 52256', result.stdout)
         self.assertIn(
             "We couldn't read your file, possibly because the version of EveryVoice that created it is incompatible with your installed version.",
@@ -291,7 +295,7 @@ def test_preprocessing_with_wrong_config(self):
         """
         The user should have a friendly message that informs them that they used the wrong config file type.
         """
-        with capture_logs() as output:
+        with silence_c_stderr(), capture_logs() as output:
             result = self.runner.invoke(
                 app,
                 [
@@ -324,7 +328,7 @@ class TestBaseCLIHelper(TestCase):
     def test_save_configuration_to_log_dir(self):
         with (
             TemporaryDirectory(ignore_cleanup_errors=True) as tempdir,
-            mute_logger("everyvoice.base_cli.helpers"),
+            silence_c_stderr(),
         ):
             tempdir = Path(tempdir)
             config = FastSpeech2Config(