From e82fbf83c887e7c2d6a5a12340dfe62d570de7cc Mon Sep 17 00:00:00 2001 From: Eric Joanis Date: Thu, 7 Nov 2024 17:30:20 -0500 Subject: [PATCH 1/6] feat: add --quiet option to run_tests.py and refactor the runner --- everyvoice/run_tests.py | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/everyvoice/run_tests.py b/everyvoice/run_tests.py index 73ff37a3..f6470cd7 100755 --- a/everyvoice/run_tests.py +++ b/everyvoice/run_tests.py @@ -3,6 +3,7 @@ """ Organize tests into Test Suites """ +import argparse import importlib import os import re @@ -39,6 +40,7 @@ "fs2", "wav2vec2aligner", ) +SUITE_NAMES = ["all", "dev"] + sorted(SUITES.keys()) SUITES["dev"] = sum((SUITES[suite] for suite in dev_suites), start=()) @@ -86,7 +88,7 @@ def describe_suite(suite: TestSuite): ) -def run_tests(suite: str, describe: bool = False): +def run_tests(suite: str, describe: bool = False, verbosity=3): """Decide which Test Suite to run""" logger.info(f"Loading test suite '{suite}'. This may take a while...") if suite == "all": @@ -98,7 +100,7 @@ def run_tests(suite: str, describe: bool = False): tests = SUITES[suite] else: logger.error( - f"Please specify a test suite to run: one of '{['all'] + sorted(SUITES.keys())}'." + f"Please specify a test suite to run: one of '{['all'] + SUITE_NAMES}'." ) return False tests = [ @@ -123,19 +125,27 @@ def run_tests(suite: str, describe: bool = False): return True else: logger.info("Running test suite") - return TextTestRunner(verbosity=3).run(test_suite).wasSuccessful() + return TextTestRunner(verbosity=verbosity).run(test_suite).wasSuccessful() -if __name__ == "__main__": - describe = "--describe" in sys.argv - if describe: - sys.argv.remove("--describe") - - try: - suite = sys.argv[1] - except IndexError: - logger.info('No test suite specified, defaulting to "dev"') - suite = "dev" - result = run_tests(suite, describe) +def main(): + parser = argparse.ArgumentParser(description="Run EveryVoice test suites.") + parser.add_argument("--quiet", "-q", action="store_true", help="reduce output") + parser.add_argument( + "--describe", action="store_true", help="describe the selected test suite" + ) + parser.add_argument( + "suite", + nargs="?", + default="dev", + help="the test suite to run [dev]", + choices=SUITE_NAMES, + ) + args = parser.parse_args() + result = run_tests(args.suite, args.describe, 1 if args.quiet else 3) if not result: sys.exit(1) + + +if __name__ == "__main__": + main() From ff7fdc349949749fda4a0fec540934659d87aad2 Mon Sep 17 00:00:00 2001 From: Eric Joanis Date: Tue, 3 Dec 2024 11:27:31 -0500 Subject: [PATCH 2/6] fix(tests): make "everyvoice test" use the same suite list as run_tests --- everyvoice/cli.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/everyvoice/cli.py b/everyvoice/cli.py index 896a7174..90e71bb9 100644 --- a/everyvoice/cli.py +++ b/everyvoice/cli.py @@ -1,3 +1,4 @@ +import enum import json import platform import subprocess @@ -47,6 +48,7 @@ synthesize as synthesize_hfg, ) from everyvoice.model.vocoder.HiFiGAN_iSTFT_lightning.hfgl.cli import train as train_hfg +from everyvoice.run_tests import SUITE_NAMES, run_tests from everyvoice.utils import generic_psv_filelist_reader, spinner from everyvoice.wizard import ( ALIGNER_CONFIG_FILENAME_PREFIX, @@ -556,23 +558,13 @@ def check_data( )(inspect_checkpoint) -class TestSuites(str, Enum): - all = "all" - cli = "cli" - config = "config" - dev = "dev" - model = "model" - preprocessing = "preprocessing" - text = "text" - fs2 = "fs2" +TestSuites = enum.Enum("TestSuites", {name: name for name in SUITE_NAMES}) # type: ignore @app.command(hidden=True) -def test(suite: TestSuites = typer.Argument(TestSuites.dev)): +def test(suite: TestSuites = typer.Argument("dev")): """Run a test suite""" - from everyvoice.run_tests import run_tests - - run_tests(suite) + run_tests(suite.value) # Deferred full initialization to optimize the CLI, but still exposed for unit testing. From 155b7cc4cfc04dbfc18663999ffa7076573ac873 Mon Sep 17 00:00:00 2001 From: Eric Joanis Date: Thu, 7 Nov 2024 18:02:08 -0500 Subject: [PATCH 3/6] refactor(tests): silence test_preprocessing and processed_audio_fixture Unit test should be quiet. --- .../tests/preprocessed_audio_fixture.py | 18 ++++-- everyvoice/tests/test_preprocessing.py | 64 ++++++++++++++----- 2 files changed, 60 insertions(+), 22 deletions(-) diff --git a/everyvoice/tests/preprocessed_audio_fixture.py b/everyvoice/tests/preprocessed_audio_fixture.py index 79bd3523..c2a1cef6 100644 --- a/everyvoice/tests/preprocessed_audio_fixture.py +++ b/everyvoice/tests/preprocessed_audio_fixture.py @@ -8,6 +8,7 @@ from everyvoice.model.e2e.config import FeaturePredictionConfig from everyvoice.preprocessor import Preprocessor from everyvoice.tests.basic_test_case import BasicTestCase +from everyvoice.tests.stubs import capture_stderr, capture_stdout, mute_logger from everyvoice.utils import collapse_whitespace, lower, nfc_normalize @@ -66,12 +67,17 @@ def setUpClass(cls): """Generate a preprocessed test set that can be used in various test cases.""" # We only need to actually run this once if not PreprocessedAudioFixture._preprocess_ran: - PreprocessedAudioFixture.preprocessor.preprocess( - output_path=str(PreprocessedAudioFixture.lj_filelist), - cpus=1, - overwrite=True, - to_process=("audio", "energy", "pitch", "text", "spec"), - ) + with ( + mute_logger("everyvoice.preprocessor"), + capture_stderr(), + capture_stdout(), + ): + PreprocessedAudioFixture.preprocessor.preprocess( + output_path=str(PreprocessedAudioFixture.lj_filelist), + cpus=1, + overwrite=True, + to_process=("audio", "energy", "pitch", "text", "spec"), + ) PreprocessedAudioFixture.lj_preprocessed.mkdir(parents=True, exist_ok=True) try: (PreprocessedAudioFixture.lj_preprocessed / "duration").symlink_to( diff --git a/everyvoice/tests/test_preprocessing.py b/everyvoice/tests/test_preprocessing.py index 9c2f0a57..da8a60ca 100644 --- a/everyvoice/tests/test_preprocessing.py +++ b/everyvoice/tests/test_preprocessing.py @@ -24,6 +24,7 @@ from everyvoice.tests.basic_test_case import BasicTestCase from everyvoice.tests.preprocessed_audio_fixture import PreprocessedAudioFixture from everyvoice.tests.stubs import ( + capture_stderr, capture_stdout, monkeypatch, mute_logger, @@ -39,7 +40,7 @@ class PreprocessingTest(PreprocessedAudioFixture, BasicTestCase): def test_run_doctest(self): """Run doctests in everyvoice.text.text_processing""" - results = doctest.testmod(everyvoice.text.text_processor) + results = doctest.testmod(everyvoice.text.text_processor, verbose=False) self.assertFalse(results.failed, results) # def test_compute_stats(self): @@ -71,9 +72,10 @@ def test_no_permissions(self): FeaturePredictionConfig(**no_permissions_args) def test_check_data(self): - checked_data = self.preprocessor.check_data( - self.filelist, heavy_objective_evaluation=True - ) + with capture_stderr(): + checked_data = self.preprocessor.check_data( + self.filelist, heavy_objective_evaluation=True + ) self.assertIn("pesq", checked_data[0]) self.assertIn("stoi", checked_data[0]) self.assertIn("si_sdr", checked_data[0]) @@ -147,7 +149,8 @@ def test_remove_silence(self): def test_process_empty_audio(self): for fn in ["empty.wav", "zeros.wav"]: - audio, sr = self.preprocessor.process_audio(self.data_dir / fn) + with mute_logger("everyvoice.preprocessor.preprocessor"): + audio, sr = self.preprocessor.process_audio(self.data_dir / fn) self.assertEqual(audio, None) self.assertEqual(sr, None) @@ -431,6 +434,7 @@ def test_text_processing(self): preprocessor = Preprocessor(fp_config) with ( capture_stdout() as output, + capture_stderr(), mute_logger("everyvoice.preprocessor"), ): preprocessor.preprocess( @@ -531,7 +535,11 @@ def test_incremental_preprocess(self): ) = self.get_simple_config(tmpdir) fp_config.preprocessing.source_data[0].filelist = partial_filelist - with capture_stdout() as output, mute_logger("everyvoice.preprocessor"): + with ( + capture_stdout() as output, + capture_stderr(), + mute_logger("everyvoice.preprocessor"), + ): Preprocessor(fp_config).preprocess( output_path=lj_filelist, cpus=1, to_process=to_process ) @@ -539,13 +547,21 @@ def test_incremental_preprocess(self): self.assertRegex(output.getvalue(), r"previously processed files *0") fp_config.preprocessing.source_data[0].filelist = full_filelist - with capture_stdout() as output, mute_logger("everyvoice.preprocessor"): + with ( + capture_stdout() as output, + capture_stderr(), + mute_logger("everyvoice.preprocessor"), + ): Preprocessor(fp_config).preprocess( output_path=lj_filelist, cpus=1, to_process=to_process ) self.assertRegex(output.getvalue(), r"processed files *2") self.assertRegex(output.getvalue(), r"previously processed files *3") - with capture_stdout() as output, mute_logger("everyvoice.preprocessor"): + with ( + capture_stdout() as output, + capture_stderr(), + mute_logger("everyvoice.preprocessor"), + ): Preprocessor(fp_config).preprocess( output_path=lj_filelist, cpus=1, @@ -563,7 +579,11 @@ def test_gotta_do_audio_first(self): fp_config, lj_filelist, _, _, _ = self.get_simple_config(tmpdir) to_process_no_audio = ("energy", "pitch", "attn", "text", "spec") - with self.assertRaises(SystemExit), capture_stdout(): + with ( + self.assertRaises(SystemExit), + capture_stdout(), + mute_logger("everyvoice.preprocessor.preprocessor"), + ): Preprocessor(fp_config).preprocess( output_path=lj_filelist, cpus=1, to_process=to_process_no_audio ) @@ -586,7 +606,12 @@ def test_empty_preprocess(self): print("empty|foo bar baz|foo bar baz|noone|und", file=f) fp_config.preprocessing.source_data[0].filelist = input_filelist - with self.assertRaises(SystemExit), capture_stdout(): + with ( + self.assertRaises(SystemExit), + capture_stdout(), + capture_stderr(), + mute_logger("everyvoice.preprocessor.preprocessor"), + ): Preprocessor(fp_config).preprocess( output_path=lj_filelist, cpus=1, to_process=to_process ) @@ -596,13 +621,16 @@ def test_config_lock(self): tmpdir = Path(tmpdir) fp_config, lj_filelist, _, _, to_process = self.get_simple_config(tmpdir) - Preprocessor(fp_config).preprocess( - output_path=lj_filelist, cpus=1, to_process=to_process - ) + with ( + mute_logger("everyvoice.preprocessor"), + capture_stderr(), + capture_stdout(), + ): + Preprocessor(fp_config).preprocess( + output_path=lj_filelist, cpus=1, to_process=to_process + ) def fail_config_lock(config_object, element, value, message): - import everyvoice.preprocessor.preprocessor - with monkeypatch(config_object, element, value): with self.assertRaises(SystemExit): with patch_logger( @@ -726,7 +754,11 @@ def test_hierarchy(self): fp_config.preprocessing.save_dir = preprocessed_dir preprocessor = Preprocessor(fp_config) - with mute_logger("everyvoice.preprocessor"), capture_stdout(): + with ( + mute_logger("everyvoice.preprocessor"), + capture_stdout(), + capture_stderr(), + ): preprocessor.preprocess( output_path=filelist, cpus=2, From 787033d1531a45b0ef875828a78c4eeaeeba4e70 Mon Sep 17 00:00:00 2001 From: Eric Joanis Date: Fri, 8 Nov 2024 17:38:34 -0500 Subject: [PATCH 4/6] refactor(tests): silence test_cli/configs/model/dataloader/utils/text/wizard --- everyvoice/tests/stubs.py | 47 +++ everyvoice/tests/test_cli.py | 48 ++-- everyvoice/tests/test_configs.py | 58 ++-- everyvoice/tests/test_dataloader.py | 31 +- everyvoice/tests/test_model.py | 430 +++++++++++++++------------- everyvoice/tests/test_text.py | 32 ++- everyvoice/tests/test_utils.py | 21 +- everyvoice/tests/test_wizard.py | 4 + 8 files changed, 393 insertions(+), 278 deletions(-) diff --git a/everyvoice/tests/stubs.py b/everyvoice/tests/stubs.py index fee1541b..04d40731 100644 --- a/everyvoice/tests/stubs.py +++ b/everyvoice/tests/stubs.py @@ -2,6 +2,7 @@ import io import logging import os +import sys from contextlib import contextmanager, redirect_stderr, redirect_stdout from pathlib import Path from typing import Any, Generator, Sequence, Union @@ -321,3 +322,49 @@ def __enter__(self): def __exit__(self, *_exc_info): for monkey in self.monkeys: monkey.__exit__(*_exc_info) + + +@contextmanager +def silence_c_stdout(): + """Capture stdout from C output, e.g., from SoundSwallower. + + Note: to capture stdout for both C and Python code, combine this with + redirect_stdout(), but you must use capture_c_stdout() first: + with capture_c_stdout(), redirect_stdout(io.StringIO()): + # code + + Loosely inspired by https://stackoverflow.com/a/24277852, but much simplified to + address our narrow needs, namely to silence stdout in a context manager. + """ + + stdout_fileno = sys.stdout.fileno() + stdout_save = os.dup(stdout_fileno) + stdout_fd = os.open(os.devnull, os.O_RDWR) + os.dup2(stdout_fd, stdout_fileno) + yield + os.dup2(stdout_save, stdout_fileno) + os.close(stdout_save) + os.close(stdout_fd) + + +@contextmanager +def silence_c_stderr(): + """Capture stderr from C output, e.g., from SoundSwallower. + + Note: to capture stderr for both C and Python code, combine this with + redirect_stderr(), but you must use capture_c_stderr() first: + with capture_c_stderr(), redirect_stderr(io.StringIO()): + # code + + Loosely inspired by https://stackoverflow.com/a/24277852, but much simplified to + address our narrow needs, namely to silence stderr in a context manager. + """ + + stderr_fileno = sys.stderr.fileno() + stderr_save = os.dup(stderr_fileno) + stderr_fd = os.open(os.devnull, os.O_RDWR) + os.dup2(stderr_fd, stderr_fileno) + yield + os.dup2(stderr_save, stderr_fileno) + os.close(stderr_save) + os.close(stderr_fd) diff --git a/everyvoice/tests/test_cli.py b/everyvoice/tests/test_cli.py index 76090bf0..e74fa4b1 100644 --- a/everyvoice/tests/test_cli.py +++ b/everyvoice/tests/test_cli.py @@ -33,7 +33,7 @@ ) from everyvoice.model.vocoder.HiFiGAN_iSTFT_lightning.hfgl.config import HiFiGANConfig from everyvoice.model.vocoder.HiFiGAN_iSTFT_lightning.hfgl.model import HiFiGAN -from everyvoice.tests.stubs import capture_logs, capture_stdout, mute_logger +from everyvoice.tests.stubs import capture_logs, capture_stdout, silence_c_stderr from everyvoice.wizard import ( SPEC_TO_WAV_CONFIG_FILENAME_PREFIX, TEXT_TO_SPEC_CONFIG_FILENAME_PREFIX, @@ -175,11 +175,12 @@ def test_commands_present(self): self.assertIn("https://docs.everyvoice.ca", result.stdout) def test_command_help_messages(self): - for command in self.commands: - result = self.runner.invoke(app, [command, "--help"]) - self.assertEqual(result.exit_code, 0) - result = self.runner.invoke(app, [command, "-h"]) - self.assertEqual(result.exit_code, 0) + with silence_c_stderr(): + for command in self.commands: + result = self.runner.invoke(app, [command, "--help"]) + self.assertEqual(result.exit_code, 0) + result = self.runner.invoke(app, [command, "-h"]) + self.assertEqual(result.exit_code, 0) def test_update_schema(self): dummy_contact = ContactInformation( @@ -233,16 +234,17 @@ def test_update_schema(self): self.assertIn("FileExistsError", str(result)) def test_evaluate(self): - result = self.runner.invoke( - app, - [ - "evaluate", - "-f", - self.data_dir / "LJ010-0008.wav", - "-r", - self.data_dir / "lj" / "wavs" / "LJ050-0269.wav", - ], - ) + with silence_c_stderr(): + result = self.runner.invoke( + app, + [ + "evaluate", + "-f", + self.data_dir / "LJ010-0008.wav", + "-r", + self.data_dir / "lj" / "wavs" / "LJ050-0269.wav", + ], + ) self.assertEqual(result.exit_code, 0) self.assertIn("LJ010-0008", result.stdout) self.assertIn("STOI", result.stdout) @@ -272,13 +274,15 @@ def test_evaluate(self): ) def test_inspect_checkpoint_help(self): - result = self.runner.invoke(app, ["inspect-checkpoint", "--help"]) + with silence_c_stderr(): + result = self.runner.invoke(app, ["inspect-checkpoint", "--help"]) self.assertIn("inspect-checkpoint [OPTIONS] MODEL_PATH", result.stdout) def test_inspect_checkpoint(self): - result = self.runner.invoke( - app, ["inspect-checkpoint", str(self.data_dir / "test.ckpt")] - ) + with silence_c_stderr(): + result = self.runner.invoke( + app, ["inspect-checkpoint", str(self.data_dir / "test.ckpt")] + ) self.assertIn('global_step": 52256', result.stdout) self.assertIn( "We couldn't read your file, possibly because the version of EveryVoice that created it is incompatible with your installed version.", @@ -291,7 +295,7 @@ def test_preprocessing_with_wrong_config(self): """ The user should have a friendly message that informs them that they used the wrong config file type. """ - with capture_logs() as output: + with silence_c_stderr(), capture_logs() as output: result = self.runner.invoke( app, [ @@ -324,7 +328,7 @@ class TestBaseCLIHelper(TestCase): def test_save_configuration_to_log_dir(self): with ( TemporaryDirectory(ignore_cleanup_errors=True) as tempdir, - mute_logger("everyvoice.base_cli.helpers"), + silence_c_stderr(), ): tempdir = Path(tempdir) config = FastSpeech2Config( diff --git a/everyvoice/tests/test_configs.py b/everyvoice/tests/test_configs.py index ae0112ef..68b86ffe 100644 --- a/everyvoice/tests/test_configs.py +++ b/everyvoice/tests/test_configs.py @@ -39,6 +39,7 @@ HiFiGANTrainingConfig, ) from everyvoice.tests.basic_test_case import BasicTestCase +from everyvoice.tests.stubs import mute_logger, silence_c_stderr from everyvoice.utils import ( expand_config_string_syntax, load_config_from_json_or_yaml_path, @@ -183,13 +184,14 @@ def test_config_partial(self): _writer_helper(vocoder_config, tempdir / "vocoder.json") self.assertTrue(isinstance(vocoder_config, VocoderConfig)) # E2E Config - e2e_config = EveryVoiceConfig( - contact=self.contact, - path_to_aligner_config_file=(tempdir / "aligner.json"), - path_to_feature_prediction_config_file=(tempdir / "fp.json"), - path_to_training_config_file=(tempdir / "training.json"), - path_to_vocoder_config_file=(tempdir / "vocoder.json"), - ) + with mute_logger("everyvoice.config.utils"): + e2e_config = EveryVoiceConfig( + contact=self.contact, + path_to_aligner_config_file=(tempdir / "aligner.json"), + path_to_feature_prediction_config_file=(tempdir / "fp.json"), + path_to_training_config_file=(tempdir / "training.json"), + path_to_vocoder_config_file=(tempdir / "vocoder.json"), + ) self.assertTrue(isinstance(e2e_config, EveryVoiceConfig)) def test_config_partial_override(self): @@ -200,19 +202,22 @@ def test_config_partial_override(self): tf.write(AudioConfig().model_dump_json()) tf.flush() # override with actual class - config = PreprocessingConfig( - path_to_audio_config_file=tf.name, - audio=AudioConfig(min_audio_length=1.0), - ) + with mute_logger("everyvoice.config.utils"): + config = PreprocessingConfig( + path_to_audio_config_file=tf.name, + audio=AudioConfig(min_audio_length=1.0), + ) self.assertEqual(config.audio.min_audio_length, 1.0) # override with dict - config = PreprocessingConfig( - path_to_audio_config_file=tf.name, audio={"max_audio_length": 1.0} - ) + with mute_logger("everyvoice.config.utils"): + config = PreprocessingConfig( + path_to_audio_config_file=tf.name, audio={"max_audio_length": 1.0} + ) self.assertEqual(config.audio.max_audio_length, 1.0) # pass something invalid - with self.assertRaises(ValidationError): - PreprocessingConfig(path_to_audio_config_file=tf.name, audio=1.0) + with mute_logger("everyvoice.config.utils"): + with self.assertRaises(ValidationError): + PreprocessingConfig(path_to_audio_config_file=tf.name, audio=1.0) def test_update_from_file(self): """Test that updating the config from yaml/json works""" @@ -383,7 +388,8 @@ def test_aligner_config(self): self.assertFalse(Path(training["logger"]["save_dir"]).is_absolute()) self.assertFalse(Path(training["training_filelist"]).is_absolute()) self.assertFalse(Path(training["validation_filelist"]).is_absolute()) - config = AlignerConfig.load_config_from_path(config_path) + with silence_c_stderr(): + config = AlignerConfig.load_config_from_path(config_path) # print(config.model_dump_json(indent=2)) self.assertTrue(isinstance(config, AlignerConfig)) self.assertEqual(config.preprocessing.dataset, self.DATASET_NAME) @@ -426,7 +432,8 @@ def test_feature_prediction_config(self): self.assertFalse(Path(training["logger"]["save_dir"]).is_absolute()) self.assertFalse(Path(training["training_filelist"]).is_absolute()) self.assertFalse(Path(training["validation_filelist"]).is_absolute()) - config = FeaturePredictionConfig.load_config_from_path(config_path) + with silence_c_stderr(): + config = FeaturePredictionConfig.load_config_from_path(config_path) # print(config.model_dump_json(indent=2)) self.assertEqual(config.preprocessing.dataset, self.DATASET_NAME) self.validate_config_path(config.path_to_text_config_file) @@ -474,7 +481,8 @@ def test_everyvoice_config(self): self.assertFalse(Path(training["logger"]["save_dir"]).is_absolute()) self.assertFalse(Path(training["training_filelist"]).is_absolute()) self.assertFalse(Path(training["validation_filelist"]).is_absolute()) - config = EveryVoiceConfig.load_config_from_path(config_path) + with silence_c_stderr(): + config = EveryVoiceConfig.load_config_from_path(config_path) # print(config.model_dump_json(indent=2)) self.assertTrue(isinstance(config, EveryVoiceConfig)) self.assertEqual( @@ -512,7 +520,7 @@ def test_absolute_path(self): validation_filelist=tempdir / "validation_filelist.psv", ) (tempdir / training.logger.save_dir).mkdir(parents=True, exist_ok=True) - print(tempdir, training.training_filelist) + # print(tempdir, training.training_filelist) (training.training_filelist).touch(exist_ok=True) (training.validation_filelist).touch(exist_ok=True) _writer_helper(training, aligner_training_path) @@ -536,7 +544,8 @@ def test_absolute_path(self): _writer_helper(aligner_config, aligner_config_path) # Reload and validate - config = AlignerConfig.load_config_from_path(aligner_config_path) + with mute_logger("everyvoice.config.utils"): + config = AlignerConfig.load_config_from_path(aligner_config_path) self.assertTrue(isinstance(config, AlignerConfig)) self.assertEqual(config.preprocessing.dataset, self.DATASET_NAME) self.validate_config_path(config.path_to_model_config_file) @@ -587,8 +596,11 @@ def test_missing_path(self): # Create the missing partial config file by deleting. # NOTE, we need the file to exists if we want to write its parent config to disk. (tempdir / "preprocessing.json").unlink() - with self.assertRaises(ValidationError): - config = AlignerConfig.load_config_from_path(tempdir / "aligner.json") + with mute_logger("everyvoice.config.utils"): + with self.assertRaises(ValidationError): + config = AlignerConfig.load_config_from_path( + tempdir / "aligner.json" + ) class BaseTrainingConfigTest(TestCase): diff --git a/everyvoice/tests/test_dataloader.py b/everyvoice/tests/test_dataloader.py index 367ee51a..e9e3cc19 100644 --- a/everyvoice/tests/test_dataloader.py +++ b/everyvoice/tests/test_dataloader.py @@ -1,5 +1,3 @@ -from tqdm import tqdm - from everyvoice.config.type_definitions import TargetTrainingTextRepresentationLevel from everyvoice.dataloader import BaseDataModule from everyvoice.dataloader.imbalanced_sampler import ImbalancedDatasetSampler @@ -17,6 +15,7 @@ ) from everyvoice.tests.basic_test_case import BasicTestCase from everyvoice.tests.preprocessed_audio_fixture import PreprocessedAudioFixture +from everyvoice.tests.stubs import mute_logger from everyvoice.utils import filter_dataset_based_on_target_text_representation_level @@ -57,7 +56,7 @@ def test_spec_dataset(self): self.config.vocoder, use_segments=True, ) - for sample in tqdm(dataset): + for sample in dataset: spec, audio, basename, spec_from_audio = sample self.assertTrue(isinstance(basename, str)) self.assertEqual(spec.size(), spec_from_audio.size()) @@ -84,20 +83,22 @@ def test_hifi_data_loader(self): def test_filter_dataset(self): train_dataset = [{"character_tokens": "b", "phone_tokens": ""}] * 4 with self.assertRaises(SystemExit) as cm: - filter_dataset_based_on_target_text_representation_level( - TargetTrainingTextRepresentationLevel.characters, - train_dataset, - "training", - 6, - ) + with mute_logger("everyvoice.utils"): + filter_dataset_based_on_target_text_representation_level( + TargetTrainingTextRepresentationLevel.characters, + train_dataset, + "training", + 6, + ) self.assertEqual(cm.exception.code, 1) with self.assertRaises(SystemExit) as cm: - filter_dataset_based_on_target_text_representation_level( - TargetTrainingTextRepresentationLevel.ipa_phones, - train_dataset, - "training", - 4, - ) + with mute_logger("everyvoice.utils"): + filter_dataset_based_on_target_text_representation_level( + TargetTrainingTextRepresentationLevel.ipa_phones, + train_dataset, + "training", + 4, + ) self.assertEqual(cm.exception.code, 1) train_ds = filter_dataset_based_on_target_text_representation_level( TargetTrainingTextRepresentationLevel.characters, diff --git a/everyvoice/tests/test_model.py b/everyvoice/tests/test_model.py index 77744d70..a8e9e9a4 100644 --- a/everyvoice/tests/test_model.py +++ b/everyvoice/tests/test_model.py @@ -25,7 +25,7 @@ from everyvoice.model.vocoder.HiFiGAN_iSTFT_lightning.hfgl.config import HiFiGANConfig from everyvoice.model.vocoder.HiFiGAN_iSTFT_lightning.hfgl.model import HiFiGAN from everyvoice.tests.basic_test_case import BasicTestCase -from everyvoice.tests.stubs import monkeypatch +from everyvoice.tests.stubs import monkeypatch, mute_logger, silence_c_stderr from everyvoice.wizard import ( ALIGNER_CONFIG_FILENAME_PREFIX, SPEC_TO_WAV_CONFIG_FILENAME_PREFIX, @@ -85,35 +85,37 @@ def test_checkpoints_only_contain_serializable_content(self): we can help allow our models to be loaded by other versions of EveryVoice. This test ensures the hyperparameters only contain JSON serializable content """ - SERIAL_SAFE_MODELS = [ - HiFiGAN( - HiFiGANConfig.load_config_from_path( - self.config_dir / f"{SPEC_TO_WAV_CONFIG_FILENAME_PREFIX}.yaml" - ) - ), - FastSpeech2( - FastSpeech2Config.load_config_from_path( - self.config_dir / f"{TEXT_TO_SPEC_CONFIG_FILENAME_PREFIX}.yaml" + with silence_c_stderr(): + SERIAL_SAFE_MODELS = [ + HiFiGAN( + HiFiGANConfig.load_config_from_path( + self.config_dir / f"{SPEC_TO_WAV_CONFIG_FILENAME_PREFIX}.yaml" + ) ), - stats=Stats( - pitch=StatsInfo( - min=0, max=1, std=2, mean=3, norm_min=4, norm_max=5 + FastSpeech2( + FastSpeech2Config.load_config_from_path( + self.config_dir / f"{TEXT_TO_SPEC_CONFIG_FILENAME_PREFIX}.yaml" ), - energy=StatsInfo( - min=7, max=8, std=9, mean=10, norm_min=11, norm_max=12 + stats=Stats( + pitch=StatsInfo( + min=0, max=1, std=2, mean=3, norm_min=4, norm_max=5 + ), + energy=StatsInfo( + min=7, max=8, std=9, mean=10, norm_min=11, norm_max=12 + ), ), + lang2id={"foo": 0, "bar": 1}, + speaker2id={"baz": 0, "qux": 1}, + ), # we should probably also test that the error about the variance adaptor is raised + Aligner( + DFAlignerConfig.load_config_from_path( + self.config_dir / f"{ALIGNER_CONFIG_FILENAME_PREFIX}.yaml" + ) ), - lang2id={"foo": 0, "bar": 1}, - speaker2id={"baz": 0, "qux": 1}, - ), # we should probably also test that the error about the variance adaptor is raised - Aligner( - DFAlignerConfig.load_config_from_path( - self.config_dir / f"{ALIGNER_CONFIG_FILENAME_PREFIX}.yaml" - ) - ), - ] + ] for model in SERIAL_SAFE_MODELS: - trainer = Trainer() + with silence_c_stderr(): + trainer = Trainer() with tempfile.TemporaryDirectory() as tmpdir_str: # Hacky way to connect the trainer with a model instead of trainer.fit(model) just for testing # https://lightning.ai/forums/t/saving-a-lightningmodule-without-a-trainer/2217/2 @@ -183,20 +185,24 @@ def test_model_is_not_a_feature_prediction(self): from pytorch_lightning.callbacks import ModelCheckpoint with tempfile.TemporaryDirectory() as tmpdir_str: - model = HiFiGAN( - HiFiGANConfig.load_config_from_path( - self.config_dir / f"{SPEC_TO_WAV_CONFIG_FILENAME_PREFIX}.yaml" + with silence_c_stderr(): + model = HiFiGAN( + HiFiGANConfig.load_config_from_path( + self.config_dir / f"{SPEC_TO_WAV_CONFIG_FILENAME_PREFIX}.yaml" + ) + ) + with silence_c_stderr(): + trainer = Trainer( + default_root_dir=tmpdir_str, + enable_progress_bar=False, + logger=False, + max_epochs=1, + limit_train_batches=1, + limit_val_batches=1, + callbacks=[ + ModelCheckpoint(dirpath=tmpdir_str, every_n_train_steps=1) + ], ) - ) - trainer = Trainer( - default_root_dir=tmpdir_str, - enable_progress_bar=False, - logger=False, - max_epochs=1, - limit_train_batches=1, - limit_val_batches=1, - callbacks=[ModelCheckpoint(dirpath=tmpdir_str, every_n_train_steps=1)], - ) trainer.strategy.connect(model) ckpt_fn = tmpdir_str + "/checkpoint.ckpt" trainer.save_checkpoint(ckpt_fn) @@ -208,7 +214,10 @@ def test_model_is_not_a_feature_prediction(self): "Unable to load config. Possible causes: is it really a FastSpeech2Config? or the correct version?" ), ): - FastSpeech2.load_from_checkpoint(ckpt_fn) + with mute_logger( + "everyvoice.model.feature_prediction.FastSpeech2_lightning.fs2.model" + ): + FastSpeech2.load_from_checkpoint(ckpt_fn) def test_model_is_not_a_vocoder(self): """ @@ -218,30 +227,34 @@ def test_model_is_not_a_vocoder(self): from pytorch_lightning.callbacks import ModelCheckpoint with tempfile.TemporaryDirectory() as tmpdir_str: - model = FastSpeech2( - FastSpeech2Config.load_config_from_path( - self.config_dir / f"{TEXT_TO_SPEC_CONFIG_FILENAME_PREFIX}.yaml" - ), - stats=Stats( - pitch=StatsInfo( - min=0, max=1, std=2, mean=3, norm_min=4, norm_max=5 + with mute_logger("everyvoice.config.text_config"): + model = FastSpeech2( + FastSpeech2Config.load_config_from_path( + self.config_dir / f"{TEXT_TO_SPEC_CONFIG_FILENAME_PREFIX}.yaml" ), - energy=StatsInfo( - min=7, max=8, std=9, mean=10, norm_min=11, norm_max=12 + stats=Stats( + pitch=StatsInfo( + min=0, max=1, std=2, mean=3, norm_min=4, norm_max=5 + ), + energy=StatsInfo( + min=7, max=8, std=9, mean=10, norm_min=11, norm_max=12 + ), ), - ), - lang2id={"foo": 0, "bar": 1}, - speaker2id={"baz": 0, "qux": 1}, - ) - trainer = Trainer( - default_root_dir=tmpdir_str, - enable_progress_bar=False, - logger=False, - max_epochs=1, - limit_train_batches=1, - limit_val_batches=1, - callbacks=[ModelCheckpoint(dirpath=tmpdir_str, every_n_train_steps=1)], - ) + lang2id={"foo": 0, "bar": 1}, + speaker2id={"baz": 0, "qux": 1}, + ) + with silence_c_stderr(): + trainer = Trainer( + default_root_dir=tmpdir_str, + enable_progress_bar=False, + logger=False, + max_epochs=1, + limit_train_batches=1, + limit_val_batches=1, + callbacks=[ + ModelCheckpoint(dirpath=tmpdir_str, every_n_train_steps=1) + ], + ) trainer.strategy.connect(model) ckpt_fn = tmpdir_str + "/checkpoint.ckpt" trainer.save_checkpoint(ckpt_fn) @@ -253,7 +266,10 @@ def test_model_is_not_a_vocoder(self): "Unable to load config. Possible causes: is it really a VocoderConfig? or the correct version?" ), ): - HiFiGAN.load_from_checkpoint(ckpt_fn) + with mute_logger( + "everyvoice.model.vocoder.HiFiGAN_iSTFT_lightning.hfgl.model" + ): + HiFiGAN.load_from_checkpoint(ckpt_fn) def test_wrong_model_type(self): """ @@ -263,46 +279,53 @@ def test_wrong_model_type(self): from pytorch_lightning.callbacks import ModelCheckpoint with tempfile.TemporaryDirectory() as tmpdir_str: - model = FastSpeech2( - FastSpeech2Config.load_config_from_path( - self.config_dir / f"{TEXT_TO_SPEC_CONFIG_FILENAME_PREFIX}.yaml" - ), - stats=Stats( - pitch=StatsInfo( - min=0, max=1, std=2, mean=3, norm_min=4, norm_max=5 - ), - energy=StatsInfo( - min=7, max=8, std=9, mean=10, norm_min=11, norm_max=12 - ), - ), - lang2id={"foo": 0, "bar": 1}, - speaker2id={"baz": 0, "qux": 1}, - ) - trainer = Trainer( - default_root_dir=tmpdir_str, - enable_progress_bar=False, - logger=False, - max_epochs=1, - limit_train_batches=1, - limit_val_batches=1, - callbacks=[ModelCheckpoint(dirpath=tmpdir_str, every_n_train_steps=1)], - ) - trainer.strategy.connect(model) - ckpt_fn = tmpdir_str + "/checkpoint.ckpt" - trainer.save_checkpoint(ckpt_fn) - m = torch.load(ckpt_fn) - self.assertIn("model_info", m.keys()) - m["model_info"]["name"] = "BAD_TYPE" - torch.save(m, ckpt_fn) - m = torch.load(ckpt_fn) - self.assertIn("model_info", m.keys()) - self.assertEqual(m["model_info"]["name"], "BAD_TYPE") - # self.assertEqual(m["model_info"]["version"], "1.0") - with self.assertRaisesRegex( - TypeError, - r"Wrong model type \(BAD_TYPE\), we are expecting a 'FastSpeech2' model", - ): - FastSpeech2.load_from_checkpoint(ckpt_fn) + if True: + with mute_logger("everyvoice.config.text_config"): + model = FastSpeech2( + FastSpeech2Config.load_config_from_path( + self.config_dir + / f"{TEXT_TO_SPEC_CONFIG_FILENAME_PREFIX}.yaml" + ), + stats=Stats( + pitch=StatsInfo( + min=0, max=1, std=2, mean=3, norm_min=4, norm_max=5 + ), + energy=StatsInfo( + min=7, max=8, std=9, mean=10, norm_min=11, norm_max=12 + ), + ), + lang2id={"foo": 0, "bar": 1}, + speaker2id={"baz": 0, "qux": 1}, + ) + with silence_c_stderr(): + trainer = Trainer( + default_root_dir=tmpdir_str, + enable_progress_bar=False, + logger=False, + max_epochs=1, + limit_train_batches=1, + limit_val_batches=1, + callbacks=[ + ModelCheckpoint(dirpath=tmpdir_str, every_n_train_steps=1) + ], + ) + trainer.strategy.connect(model) + ckpt_fn = tmpdir_str + "/checkpoint.ckpt" + trainer.save_checkpoint(ckpt_fn) + m = torch.load(ckpt_fn) + self.assertIn("model_info", m.keys()) + m["model_info"]["name"] = "BAD_TYPE" + torch.save(m, ckpt_fn) + m = torch.load(ckpt_fn) + self.assertIn("model_info", m.keys()) + self.assertEqual(m["model_info"]["name"], "BAD_TYPE") + # self.assertEqual(m["model_info"]["version"], "1.0") + with self.assertRaisesRegex( + TypeError, + r"Wrong model type \(BAD_TYPE\), we are expecting a 'FastSpeech2' model", + ): + with mute_logger("everyvoice.config.text_config"): + FastSpeech2.load_from_checkpoint(ckpt_fn) def test_missing_model_version(self): """ @@ -311,59 +334,65 @@ def test_missing_model_version(self): from pytorch_lightning import Trainer from pytorch_lightning.callbacks import ModelCheckpoint - tests = ( - ( - Aligner, - Aligner( - DFAlignerConfig.load_config_from_path( - self.config_dir / f"{ALIGNER_CONFIG_FILENAME_PREFIX}.yaml" - ) - ), - ), - ( - FastSpeech2, - FastSpeech2( - FastSpeech2Config.load_config_from_path( - self.config_dir / f"{TEXT_TO_SPEC_CONFIG_FILENAME_PREFIX}.yaml" + with mute_logger("everyvoice.config.text_config"): + tests = ( + ( + Aligner, + Aligner( + DFAlignerConfig.load_config_from_path( + self.config_dir / f"{ALIGNER_CONFIG_FILENAME_PREFIX}.yaml" + ) ), - stats=Stats( - pitch=StatsInfo( - min=0, max=1, std=2, mean=3, norm_min=4, norm_max=5 + ), + ( + FastSpeech2, + FastSpeech2( + FastSpeech2Config.load_config_from_path( + self.config_dir + / f"{TEXT_TO_SPEC_CONFIG_FILENAME_PREFIX}.yaml" ), - energy=StatsInfo( - min=7, max=8, std=9, mean=10, norm_min=11, norm_max=12 + stats=Stats( + pitch=StatsInfo( + min=0, max=1, std=2, mean=3, norm_min=4, norm_max=5 + ), + energy=StatsInfo( + min=7, max=8, std=9, mean=10, norm_min=11, norm_max=12 + ), ), + lang2id={"foo": 0, "bar": 1}, + speaker2id={"baz": 0, "qux": 1}, + ), + ), # we should probably also test that the error about the variance adaptor is raised + ( + HiFiGAN, + HiFiGAN( + HiFiGANConfig.load_config_from_path( + self.config_dir + / f"{SPEC_TO_WAV_CONFIG_FILENAME_PREFIX}.yaml" + ) ), - lang2id={"foo": 0, "bar": 1}, - speaker2id={"baz": 0, "qux": 1}, - ), - ), # we should probably also test that the error about the variance adaptor is raised - ( - HiFiGAN, - HiFiGAN( - HiFiGANConfig.load_config_from_path( - self.config_dir / f"{SPEC_TO_WAV_CONFIG_FILENAME_PREFIX}.yaml" - ) ), - ), - ) + ) CANARY_VERSION = "CANARY_VERSION" with tempfile.TemporaryDirectory() as tmpdir_str: for ModelType, model in tests: with self.subTest(ModelType=ModelType): model._VERSION = CANARY_VERSION - trainer = Trainer( - default_root_dir=tmpdir_str, - enable_progress_bar=False, - logger=False, - max_epochs=1, - limit_train_batches=1, - limit_val_batches=1, - callbacks=[ - ModelCheckpoint(dirpath=tmpdir_str, every_n_train_steps=1) - ], - ) + with silence_c_stderr(): + trainer = Trainer( + default_root_dir=tmpdir_str, + enable_progress_bar=False, + logger=False, + max_epochs=1, + limit_train_batches=1, + limit_val_batches=1, + callbacks=[ + ModelCheckpoint( + dirpath=tmpdir_str, every_n_train_steps=1 + ) + ], + ) trainer.strategy.connect(model) ckpt_fn = tmpdir_str + "/checkpoint.ckpt" trainer.save_checkpoint(ckpt_fn) @@ -373,7 +402,8 @@ def test_missing_model_version(self): self.assertEqual(m["model_info"]["version"], CANARY_VERSION) del m["model_info"]["version"] torch.save(m, ckpt_fn) - model = ModelType.load_from_checkpoint(ckpt_fn) + with mute_logger("everyvoice.config.text_config"): + model = ModelType.load_from_checkpoint(ckpt_fn) self.assertEqual(model._VERSION, "1.0") def test_newer_model_version(self): @@ -383,59 +413,65 @@ def test_newer_model_version(self): from pytorch_lightning import Trainer from pytorch_lightning.callbacks import ModelCheckpoint - tests = ( - ( - Aligner, - Aligner( - DFAlignerConfig.load_config_from_path( - self.config_dir / f"{ALIGNER_CONFIG_FILENAME_PREFIX}.yaml" - ) - ), - ), - ( - FastSpeech2, - FastSpeech2( - FastSpeech2Config.load_config_from_path( - self.config_dir / f"{TEXT_TO_SPEC_CONFIG_FILENAME_PREFIX}.yaml" + with mute_logger("everyvoice.config.text_config"): + tests = ( + ( + Aligner, + Aligner( + DFAlignerConfig.load_config_from_path( + self.config_dir / f"{ALIGNER_CONFIG_FILENAME_PREFIX}.yaml" + ) ), - stats=Stats( - pitch=StatsInfo( - min=0, max=1, std=2, mean=3, norm_min=4, norm_max=5 + ), + ( + FastSpeech2, + FastSpeech2( + FastSpeech2Config.load_config_from_path( + self.config_dir + / f"{TEXT_TO_SPEC_CONFIG_FILENAME_PREFIX}.yaml" ), - energy=StatsInfo( - min=7, max=8, std=9, mean=10, norm_min=11, norm_max=12 + stats=Stats( + pitch=StatsInfo( + min=0, max=1, std=2, mean=3, norm_min=4, norm_max=5 + ), + energy=StatsInfo( + min=7, max=8, std=9, mean=10, norm_min=11, norm_max=12 + ), ), + lang2id={"foo": 0, "bar": 1}, + speaker2id={"baz": 0, "qux": 1}, + ), + ), # we should probably also test that the error about the variance adaptor is raised + ( + HiFiGAN, + HiFiGAN( + HiFiGANConfig.load_config_from_path( + self.config_dir + / f"{SPEC_TO_WAV_CONFIG_FILENAME_PREFIX}.yaml" + ) ), - lang2id={"foo": 0, "bar": 1}, - speaker2id={"baz": 0, "qux": 1}, - ), - ), # we should probably also test that the error about the variance adaptor is raised - ( - HiFiGAN, - HiFiGAN( - HiFiGANConfig.load_config_from_path( - self.config_dir / f"{SPEC_TO_WAV_CONFIG_FILENAME_PREFIX}.yaml" - ) ), - ), - ) + ) NEWER_VERSION = "100.0" with tempfile.TemporaryDirectory() as tmpdir_str: for ModelType, model in tests: with self.subTest(ModelType=ModelType): model._VERSION = NEWER_VERSION - trainer = Trainer( - default_root_dir=tmpdir_str, - enable_progress_bar=False, - logger=False, - max_epochs=1, - limit_train_batches=1, - limit_val_batches=1, - callbacks=[ - ModelCheckpoint(dirpath=tmpdir_str, every_n_train_steps=1) - ], - ) + with silence_c_stderr(): + trainer = Trainer( + default_root_dir=tmpdir_str, + enable_progress_bar=False, + logger=False, + max_epochs=1, + limit_train_batches=1, + limit_val_batches=1, + callbacks=[ + ModelCheckpoint( + dirpath=tmpdir_str, every_n_train_steps=1 + ) + ], + ) trainer.strategy.connect(model) ckpt_fn = tmpdir_str + "/checkpoint.ckpt" trainer.save_checkpoint(ckpt_fn) @@ -447,7 +483,8 @@ def test_newer_model_version(self): ValueError, r"Your model was created with a newer version of EveryVoice, please update your software.", ): - ModelType.load_from_checkpoint(ckpt_fn) + with mute_logger("everyvoice.config.text_config"): + ModelType.load_from_checkpoint(ckpt_fn) class TestLoadingConfig(BasicTestCase): @@ -469,13 +506,15 @@ def test_config_versionless(self): for ConfigType, filename in self.configs: with self.subTest(ConfigType=ConfigType): - arguments = ConfigType.load_config_from_path( - self.config_dir / f"{filename}.yaml" - ).model_dump() + with silence_c_stderr(): + arguments = ConfigType.load_config_from_path( + self.config_dir / f"{filename}.yaml" + ).model_dump() del arguments["VERSION"] self.assertNotIn("VERSION", arguments) - c = ConfigType(**arguments) + with silence_c_stderr(): + c = ConfigType(**arguments) self.assertEqual(c.VERSION, "1.0") def test_config_newer_version(self): @@ -485,9 +524,10 @@ def test_config_newer_version(self): for ConfigType, filename in self.configs: with self.subTest(ConfigType=ConfigType): - reference = ConfigType.load_config_from_path( - self.config_dir / f"{filename}.yaml" - ) + with silence_c_stderr(): + reference = ConfigType.load_config_from_path( + self.config_dir / f"{filename}.yaml" + ) NEWER_VERSION = "100.0" reference.VERSION = NEWER_VERSION diff --git a/everyvoice/tests/test_text.py b/everyvoice/tests/test_text.py index ec8bf955..631c1dc0 100644 --- a/everyvoice/tests/test_text.py +++ b/everyvoice/tests/test_text.py @@ -13,6 +13,7 @@ from everyvoice.config.text_config import Punctuation, Symbols, TextConfig from everyvoice.model.feature_prediction.config import FeaturePredictionConfig from everyvoice.tests.basic_test_case import BasicTestCase +from everyvoice.tests.stubs import silence_c_stderr from everyvoice.text.features import N_PHONOLOGICAL_FEATURES from everyvoice.text.lookups import build_lookup, lookuptables_from_data from everyvoice.text.phonemizer import AVAILABLE_G2P_ENGINES, get_g2p_engine @@ -63,12 +64,13 @@ def test_hardcoded_symbols(self): def test_cleaners_with_upper(self): text = "hello world" text_upper = "HELLO WORLD" - upper_text_processor = TextProcessor( - TextConfig( - cleaners=[collapse_whitespace, lower], - symbols=Symbols(letters=list(string.ascii_letters)), - ), - ) + with silence_c_stderr(): + upper_text_processor = TextProcessor( + TextConfig( + cleaners=[collapse_whitespace, lower], + symbols=Symbols(letters=list(string.ascii_letters)), + ), + ) sequence = upper_text_processor.encode_text(text_upper) self.assertEqual(upper_text_processor.decode_tokens(sequence, "", ""), text) @@ -78,12 +80,13 @@ def test_no_duplicate_punctuation(self): def test_punctuation(self): text = "hello! How are you? My name's: foo;." - upper_text_processor = TextProcessor( - TextConfig( - cleaners=[collapse_whitespace, lower], - symbols=Symbols(letters=list(string.ascii_letters)), - ), - ) + with silence_c_stderr(): + upper_text_processor = TextProcessor( + TextConfig( + cleaners=[collapse_whitespace, lower], + symbols=Symbols(letters=list(string.ascii_letters)), + ), + ) tokens = upper_text_processor.apply_tokenization( upper_text_processor.normalize_text(text) ) @@ -210,7 +213,7 @@ def test_duplicates_removed(self): symbols=Symbols(letters=list(string.ascii_letters), duplicate=["e"]) ) ) - self.assertEquals( + self.assertEqual( len([x for x in duplicate_symbols_text_processor.symbols if x == "e"]), 1 ) @@ -253,7 +256,8 @@ def test_normalization(self): def test_missing_symbol(self): text = "h3llo world" - sequence = self.base_text_processor.encode_text(text) + with silence_c_stderr(): + sequence = self.base_text_processor.encode_text(text) self.assertNotEqual(self.base_text_processor.decode_tokens(sequence), text) self.assertIn("3", self.base_text_processor.missing_symbols) self.assertEqual(self.base_text_processor.missing_symbols["3"], 1) diff --git a/everyvoice/tests/test_utils.py b/everyvoice/tests/test_utils.py index d57d0e55..8037577b 100644 --- a/everyvoice/tests/test_utils.py +++ b/everyvoice/tests/test_utils.py @@ -19,7 +19,7 @@ path_is_a_directory, relative_to_absolute_path, ) -from everyvoice.tests.stubs import capture_logs +from everyvoice.tests.stubs import capture_logs, patch_logger, silence_c_stderr from everyvoice.utils import write_filelist from everyvoice.utils.heavy import get_device_from_accelerator @@ -32,7 +32,7 @@ def test_version_is_pep440_compliant(self): class UtilsTest(TestCase): def test_run_doctest(self): """Run doctests in everyvoice.utils""" - results = doctest.testmod(everyvoice.utils) + results = doctest.testmod(everyvoice.utils, verbose=False) self.assertFalse(results.failed, results) def test_write_filelist(self): @@ -248,14 +248,16 @@ def test_using_a_directory(self): """ Automatically create a directory. """ - with tempfile.TemporaryDirectory() as tmpdir, capture_logs() as output: + with tempfile.TemporaryDirectory() as tmpdir: path = Path(tmpdir) / "test_using_a_directory" self.assertFalse(path.exists()) - dir = DirectoryPathMustExist(path=path) + with patch_logger(everyvoice.config.validation_helpers) as logger: + with self.assertLogs(logger) as cm: + dir = DirectoryPathMustExist(path=path) self.assertEqual(dir.path, path) self.assertIn( f"Directory at {path} does not exist. Creating...", - output[0], + "".join(cm.output), ) self.assertTrue(path.exists()) self.assertTrue(dir.path.exists()) @@ -263,10 +265,11 @@ def test_using_a_directory(self): class GetDeviceFromAcceleratorTest(TestCase): def test_auto(self): - self.assertEqual( - get_device_from_accelerator("auto"), - torch.device("cuda:0" if torch.cuda.is_available() else "cpu"), - ) + with silence_c_stderr(): + self.assertEqual( + get_device_from_accelerator("auto"), + torch.device("cuda:0" if torch.cuda.is_available() else "cpu"), + ) def test_cpu(self): self.assertEqual(get_device_from_accelerator("cpu"), torch.device("cpu")) diff --git a/everyvoice/tests/test_wizard.py b/everyvoice/tests/test_wizard.py index 7826320f..ad870f95 100644 --- a/everyvoice/tests/test_wizard.py +++ b/everyvoice/tests/test_wizard.py @@ -1453,6 +1453,8 @@ def test_festival(self): def test_multilingual_multispeaker_true_config(self): """ + Test mismatched multi-monolingual and multi-monospeaker datasets. + Makes sure that multilingual and multispeaker parameters of config are set to true when two monolingual and monospeaker datasets are provided with different specified languages and speakers. """ with tempfile.TemporaryDirectory() as tmpdir: @@ -1621,6 +1623,8 @@ def test_multilingual_multispeaker_true_config(self): def test_multilingual_multispeaker_false_config(self): """ + Test matched multi-monospeaker and multi-monolingual datasets. + Makes sure that multilingual and multispeaker parameters of config are set to false when two monolingual and monospeaker datasets are provided with the specified languages and speakers which are the same. """ with tempfile.TemporaryDirectory() as tmpdir: From 3580e1393177d3a290897f6efa1da6114a67bd02 Mon Sep 17 00:00:00 2001 From: Eric Joanis Date: Wed, 27 Nov 2024 17:15:19 -0500 Subject: [PATCH 5/6] refactor(tests): move all doctest execution to one place also include doctests for one module we had forgotten to activate --- everyvoice/run_tests.py | 2 +- everyvoice/tests/test_doctests.py | 23 +++++++++++++++++++++++ everyvoice/tests/test_preprocessing.py | 6 ------ everyvoice/tests/test_text.py | 13 ------------- everyvoice/tests/test_utils.py | 6 ------ 5 files changed, 24 insertions(+), 26 deletions(-) create mode 100644 everyvoice/tests/test_doctests.py diff --git a/everyvoice/run_tests.py b/everyvoice/run_tests.py index f6470cd7..c781c5ab 100755 --- a/everyvoice/run_tests.py +++ b/everyvoice/run_tests.py @@ -22,7 +22,7 @@ SUITES: dict[str, tuple[str, ...]] = { "config": ("test_configs",), "loader": ("test_dataloader",), - "text": ("test_text", "test_utils"), + "text": ("test_text", "test_utils", "test_doctests"), "preprocessing": ("test_preprocessing",), "model": ("test_model",), "cli": ("test_wizard", "test_cli", "test_wizard_helpers"), diff --git a/everyvoice/tests/test_doctests.py b/everyvoice/tests/test_doctests.py new file mode 100644 index 00000000..7101f9a2 --- /dev/null +++ b/everyvoice/tests/test_doctests.py @@ -0,0 +1,23 @@ +import doctest +from unittest import TestCase + +import everyvoice.demo.app +import everyvoice.text +import everyvoice.utils + + +class RunDocTests(TestCase): + + def test_run_all_doctests(self): + for module_with_doctests in ( + everyvoice.demo.app, + everyvoice.text.features, + everyvoice.text.text_processor, + everyvoice.text.utils, + everyvoice.utils, + ): + with self.subTest( + "Running doctests in", module=module_with_doctests.__name__ + ): + results = doctest.testmod(module_with_doctests) + self.assertFalse(results.failed, results) diff --git a/everyvoice/tests/test_preprocessing.py b/everyvoice/tests/test_preprocessing.py index da8a60ca..c3eb7dbd 100644 --- a/everyvoice/tests/test_preprocessing.py +++ b/everyvoice/tests/test_preprocessing.py @@ -1,4 +1,3 @@ -import doctest import os import shutil import tempfile @@ -38,11 +37,6 @@ class PreprocessingTest(PreprocessedAudioFixture, BasicTestCase): filelist = generic_psv_filelist_reader(BasicTestCase.data_dir / "metadata.psv") - def test_run_doctest(self): - """Run doctests in everyvoice.text.text_processing""" - results = doctest.testmod(everyvoice.text.text_processor, verbose=False) - self.assertFalse(results.failed, results) - # def test_compute_stats(self): # feat_prediction_config = EveryVoiceConfig.load_config_from_path().feature_prediction # preprocessor = Preprocessor(feat_prediction_config) diff --git a/everyvoice/tests/test_text.py b/everyvoice/tests/test_text.py index 631c1dc0..2cf915b2 100644 --- a/everyvoice/tests/test_text.py +++ b/everyvoice/tests/test_text.py @@ -1,4 +1,3 @@ -import doctest import string from pathlib import Path from typing import Dict, List @@ -7,8 +6,6 @@ from pydantic import ValidationError -import everyvoice.demo.app -import everyvoice.text.utils from everyvoice import exceptions from everyvoice.config.text_config import Punctuation, Symbols, TextConfig from everyvoice.model.feature_prediction.config import FeaturePredictionConfig @@ -35,16 +32,6 @@ def setUp(self) -> None: TextConfig(symbols=Symbols(letters=list(string.ascii_letters))), ) - def test_run_demo_doctest(self): - """Run doctests in everyvoice.demo""" - results = doctest.testmod(everyvoice.demo.app) - self.assertFalse(results.failed, results) - - def test_run_doctest(self): - """Run doctests in everyvoice.utils""" - results = doctest.testmod(everyvoice.text) - self.assertFalse(results.failed, results) - def test_text_to_sequence(self): text = "hello world" sequence = self.base_text_processor.encode_text(text) diff --git a/everyvoice/tests/test_utils.py b/everyvoice/tests/test_utils.py index 8037577b..55aac878 100644 --- a/everyvoice/tests/test_utils.py +++ b/everyvoice/tests/test_utils.py @@ -1,4 +1,3 @@ -import doctest import re import tempfile from pathlib import Path @@ -30,11 +29,6 @@ def test_version_is_pep440_compliant(self): class UtilsTest(TestCase): - def test_run_doctest(self): - """Run doctests in everyvoice.utils""" - results = doctest.testmod(everyvoice.utils, verbose=False) - self.assertFalse(results.failed, results) - def test_write_filelist(self): """Filelist should write files with headers in order""" basic_files = [ From 22c67a61a4662f0cff61321dfe6a10345932c9e2 Mon Sep 17 00:00:00 2001 From: Eric Joanis Date: Thu, 28 Nov 2024 11:36:14 -0500 Subject: [PATCH 6/6] chore: update fs2 submodule to quiet its test suites too --- everyvoice/model/feature_prediction/FastSpeech2_lightning | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/everyvoice/model/feature_prediction/FastSpeech2_lightning b/everyvoice/model/feature_prediction/FastSpeech2_lightning index 7b50b383..2fd8eb45 160000 --- a/everyvoice/model/feature_prediction/FastSpeech2_lightning +++ b/everyvoice/model/feature_prediction/FastSpeech2_lightning @@ -1 +1 @@ -Subproject commit 7b50b383254ee70ce7bd6082a6ac825e0d553063 +Subproject commit 2fd8eb45beaf7e2cecec69204a0e2a3975c1b844