From b15593c45227c542268a416468232d64a4fdc18d Mon Sep 17 00:00:00 2001 From: Vidalnt Date: Sat, 14 Sep 2024 18:24:46 -0500 Subject: [PATCH 01/11] Update Pre_Compiled.yml - Linux added --- .github/workflows/pre_compiled.yml | 101 +++++++++++++++++++---------- 1 file changed, 68 insertions(+), 33 deletions(-) diff --git a/.github/workflows/pre_compiled.yml b/.github/workflows/pre_compiled.yml index 433442a0..e6275568 100644 --- a/.github/workflows/pre_compiled.yml +++ b/.github/workflows/pre_compiled.yml @@ -16,33 +16,26 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies run: | .\run-install.bat .\env\python.exe core.py "prerequisites" --models "True" --exe "True" --pretraineds_v1_f0 "False" --pretraineds_v2_f0 "True" --pretraineds_v1_nof0 "False" --pretraineds_v2_nof0 "False" - - name: Clean up unnecessary files run: | - Remove-Item -Path 'LICENSE', 'run-tensorboard.sh', 'run-install.sh', 'run-applio.sh', 'run-install.bat', 'requirements.txt', '.gitignore' -Force -ErrorAction SilentlyContinue - Remove-Item -Path '.github' -Recurse -Force -ErrorAction SilentlyContinue - + Remove-Item -Path 'LICENSE', 'run-tensorboard.sh', 'run-install.sh', 'run-applio.sh', 'run-install.bat', 'requirements.txt', '.gitignore', '.github', '.git' -Recurse -Force -ErrorAction SilentlyContinue - name: Read version from config.json id: get-version run: | $version = (Get-Content -Path './assets/config.json' | ConvertFrom-Json).version echo "version=$version" >> $env:GITHUB_OUTPUT - - name: Create ZIP file run: | $version = '${{ steps.get-version.outputs.version }}' Compress-Archive -Path 'D:\a\Applio\Applio\*' -DestinationPath "D:\a\Applio\ApplioV$version.zip" - - name: Upload Windows Compiled env: HF_TOKEN: ${{ secrets.HF_TOKEN }} @@ -65,7 +58,71 @@ jobs: repo_id=repo_id, token=token) " + Linux: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10.12"] + + steps: + - uses: actions/checkout@v3 + - name: Free up space on runner + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Read version from config.json + id: get-version + run: | + version=$(jq -r '.version' assets/config.json) + echo "version=$version" >> $GITHUB_OUTPUT + - name: Create virtual environment + run: | + pip install huggingface_hub + python -m venv .venv + . .venv/bin/activate + python -m ensurepip + python -m pip install "pip<24.1" + python -m pip install -r requirements.txt + python -m pip install torch==2.3.1 torchvision torchaudio --upgrade --index-url https://download.pytorch.org/whl/cu121 + python core.py "prerequisites" --models "True" --exe "False" --pretraineds_v1_f0 "False" --pretraineds_v2_f0 "True" --pretraineds_v1_nof0 "False" --pretraineds_v2_nof0 "False" + find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete + deactivate + rm -f .venv/bin/Activate.ps1 .venv/bin/activate .venv/bin/activate.csh .venv/bin/activate.fish .venv/bin/pip .venv/bin/pip3 .venv/bin/pip3.10 .venv/bin/python .venv/bin/python3 .venv/bin/python3.10 + - name: Clean up unnecessary files + run: | + rm -f LICENSE run-tensorboard.bat run-install.bat run-applio.bat run-install.sh requirements.txt .gitignore + rm -rf .github .git + - name: Create ZIP file + run: | + zip -9 -r ApplioV${{ steps.get-version.outputs.version }}.zip . + - name: Upload Linux Compiled + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + version: ${{ steps.get-version.outputs.version }} + run: | + python -c " + import os + from huggingface_hub import login, upload_file + + token = os.getenv('HF_TOKEN') + version = os.getenv('version') + login(token) + repo_id = 'IAHispano/Applio' + repo_file_path = f'ApplioV{version}.zip' + upload_file( + commit_message=f'{version}', + path_or_fileobj=repo_file_path, + path_in_repo=f'Compiled/Linux/ApplioV{version}.zip', + repo_id=repo_id, + token=token) + " Kaggle: runs-on: ubuntu-latest strategy: @@ -74,25 +131,21 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Free up space on runner run: | sudo rm -rf /usr/share/dotnet sudo rm -rf /opt/ghc sudo rm -rf "/usr/local/share/boost" sudo rm -rf "$AGENT_TOOLSDIRECTORY" - - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Read version from config.json id: get-version run: | version=$(jq -r '.version' assets/config.json) echo "version=$version" >> $GITHUB_OUTPUT - - name: Install uv and create virtual environment run: | sudo apt-get install -y pigz @@ -105,31 +158,17 @@ jobs: . .venv/bin/activate python -m ensurepip python -m pip install "pip<24.1" - python -m pip install torch==2.3.1 torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121 + python -m pip install torch==2.3.1 torchvision torchaudio --upgrade --index-url https://download.pytorch.org/whl/cu121 deactivate wget https://huggingface.co/IAHispano/Applio/resolve/main/Enviroment/Kaggle/bin_kaggle.tar.gz?download=true -O bin_kaggle.tar.gz tar -xzf bin_kaggle.tar.gz cp -r bin/* .venv/bin/ rm -rf bin rm bin_kaggle.tar.gz - rm -f .venv/pyvenv.cfg - rm -f .venv/bin/activate - rm -f .venv/bin/activate.bat - rm -f .venv/bin/activate.csh - rm -f .venv/bin/activate.fish - rm -f .venv/bin/activate.nu - rm -f .venv/bin/activate.ps1 - rm -f .venv/bin/activate_this.py - rm -f .venv/bin/deactivate.bat - rm -f .venv/bin/pydoc.bat - rm -f .venv/bin/python - rm -f .venv/bin/python3 - rm -f .venv/bin/python3.10 - + rm -f .venv/pyvenv.cfg .venv/bin/activate .venv/bin/activate.bat .venv/bin/activate.csh .venv/bin/activate.fish .venv/bin/activate.nu .venv/bin/activate.ps1 .venv/bin/activate_this.py .venv/bin/deactivate.bat .venv/bin/pydoc.bat .venv/bin/python .venv/bin/python3 .venv/bin/python3.10 - name: Create TAR Env run: | tar --use-compress-program="pigz --best --recursive | pv" -cvf KaggleV2.tar.gz .venv - - name: Upload Kaggle Env env: HF_TOKEN: ${{ secrets.HF_TOKEN }} @@ -161,12 +200,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Download and extract pre-packaged dependencies run: | wget https://huggingface.co/IAHispano/Applio/resolve/main/Enviroment/Colab/tmp_packages.tar.gz?download=true -O prepackaged.tar.gz @@ -195,11 +232,10 @@ jobs: print(f'Finished recording filesystem timestamps to {output_file}.') scan_and_write(\"/opt/hostedtoolcache/Python/3.10.12/x64\", \"./usr_files.csv\")" - - name: Install dependencies run: | pip install -r requirements.txt - + pip install torch==2.3.1 torchvision torchaudio --upgrade --index-url https://download.pytorch.org/whl/cu121 - name: Record final state of site-packages run: | python -c " @@ -219,7 +255,6 @@ jobs: print(f'Finished recording filesystem timestamps to {output_file}.') scan_and_write(\"/opt/hostedtoolcache/Python/3.10.12/x64\", \"./usr_files_new.csv\")" - - name: Create TAR file run: | python -c " From 1ceb575934c70ea086753d0aceab81c1a85f1a68 Mon Sep 17 00:00:00 2001 From: Vidalnt Date: Sun, 15 Sep 2024 09:30:53 -0500 Subject: [PATCH 02/11] Delete __pycache__ "pre_compiled.yml" --- .github/workflows/pre_compiled.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pre_compiled.yml b/.github/workflows/pre_compiled.yml index e6275568..95e37ff7 100644 --- a/.github/workflows/pre_compiled.yml +++ b/.github/workflows/pre_compiled.yml @@ -27,6 +27,7 @@ jobs: - name: Clean up unnecessary files run: | Remove-Item -Path 'LICENSE', 'run-tensorboard.sh', 'run-install.sh', 'run-applio.sh', 'run-install.bat', 'requirements.txt', '.gitignore', '.github', '.git' -Recurse -Force -ErrorAction SilentlyContinue + Get-ChildItem -Path . -Include __pycache__ -Recurse -Directory | Remove-Item -Force -Recurse -ErrorAction SilentlyContinue - name: Read version from config.json id: get-version run: | From ece72ee2ddaadab8a5e463ca4c056742106a6aff Mon Sep 17 00:00:00 2001 From: Alexey Shmelev Date: Sun, 15 Sep 2024 21:45:48 -0400 Subject: [PATCH 03/11] 1) removed two non-pitch guidance functions from data_utils 2) using the same audio loader and collate functions regardless of pitch guidance 3) added proper device handling intead of parameter 4) removed most of pitch guidance code branching, now net_g gets None values for pitch and pitchf 5) removed 2 copy-pasted calls for extact_model 6) streamlined the pipeline logic --- rvc/infer/pipeline.py | 119 ++---- rvc/train/data_utils.py | 215 ----------- rvc/train/train.py | 786 ++++++++++++---------------------------- 3 files changed, 281 insertions(+), 839 deletions(-) diff --git a/rvc/infer/pipeline.py b/rvc/infer/pipeline.py index 45b9c18b..5f0ffd68 100644 --- a/rvc/infer/pipeline.py +++ b/rvc/infer/pipeline.py @@ -259,7 +259,9 @@ def get_f0_hybrid( for method in methods: f0 = None if method == "crepe": - f0 = self.get_f0_crepe(x, f0_min, f0_max, p_len, int(hop_length)) + f0 = self.get_f0_crepe_computation( + x, f0_min, f0_max, p_len, int(hop_length) + ) elif method == "rmvpe": self.model_rmvpe = RMVPE0Predictor( os.path.join("rvc", "models", "predictors", "rmvpe.pt"), @@ -412,82 +414,44 @@ def voice_conversion( version: Model version ("v1" or "v2"). protect: Protection level for preserving the original pitch. """ - feats = torch.from_numpy(audio0) - if self.is_half: - feats = feats.half() - else: - feats = feats.float() - if feats.dim() == 2: - feats = feats.mean(-1) - assert feats.dim() == 1, feats.dim() - feats = feats.view(1, -1) - padding_mask = torch.BoolTensor(feats.shape).to(self.device).fill_(False) - - with torch.no_grad(): - feats = model(feats.to(self.device))["last_hidden_state"] - feats = ( - model.final_proj(feats[0]).unsqueeze(0) if version == "v1" else feats - ) - if protect < 0.5 and pitch != None and pitchf != None: - feats0 = feats.clone() - if ( - isinstance(index, type(None)) == False - and isinstance(big_npy, type(None)) == False - and index_rate != 0 - ): - npy = feats[0].cpu().numpy() - if self.is_half: - npy = npy.astype("float32") - - score, ix = index.search(npy, k=8) - weight = np.square(1 / score) - weight /= weight.sum(axis=1, keepdims=True) - npy = np.sum(big_npy[ix] * np.expand_dims(weight, axis=2), axis=1) - - if self.is_half: - npy = npy.astype("float16") - feats = ( - torch.from_numpy(npy).unsqueeze(0).to(self.device) * index_rate - + (1 - index_rate) * feats - ) - - feats = F.interpolate(feats.permute(0, 2, 1), scale_factor=2).permute(0, 2, 1) - if protect < 0.5 and pitch != None and pitchf != None: - feats0 = F.interpolate(feats0.permute(0, 2, 1), scale_factor=2).permute( - 0, 2, 1 - ) - p_len = audio0.shape[0] // self.window - if feats.shape[1] < p_len: - p_len = feats.shape[1] - if pitch != None and pitchf != None: - pitch = pitch[:, :p_len] - pitchf = pitchf[:, :p_len] - - if protect < 0.5 and pitch != None and pitchf != None: - pitchff = pitchf.clone() - pitchff[pitchf > 0] = 1 - pitchff[pitchf < 1] = protect - pitchff = pitchff.unsqueeze(-1) - feats = feats * pitchff + feats0 * (1 - pitchff) - feats = feats.to(feats0.dtype) - p_len = torch.tensor([p_len], device=self.device).long() with torch.no_grad(): - if pitch != None and pitchf != None: - audio1 = ( - (net_g.infer(feats, p_len, pitch, pitchf, sid)[0][0, 0]) - .data.cpu() - .float() - .numpy() - ) + pitch_guidance = pitch != None and pitchf != None + # prepare source audio + feats = torch.from_numpy(audio0).half() if self.is_half else torch.from_numpy(audio0).float() + feats = feats.mean(-1) if feats.dim() == 2 else feats + assert feats.dim() == 1, feats.dim() + feats = feats.view(1, -1).to(self.device) + # extract features + feats = model(feats)["last_hidden_state"] + feats = model.final_proj(feats[0]).unsqueeze(0) if version == "v1" else feats + # make a copy for pitch guidance and protection + feats0 = feats.clone() if pitch_guidance else None + if index: # set by parent function, only true if index is available, loaded, and index rate > 0 + feats = self._retrieve_speaker_embeddings(feats, index, big_npy, index_rate) + # feature upsampling + feats = F.interpolate(feats.permute(0, 2, 1), scale_factor=2).permute(0, 2, 1) + # adjust the length if the audio is short + p_len = min(audio0.shape[0] // self.window, feats.shape[1]) + if pitch_guidance: + feats0 = F.interpolate(feats0.permute(0, 2, 1), scale_factor=2).permute(0, 2, 1) + pitch, pitchf = pitch[:, :p_len], pitchf[:, :p_len] + # Pitch protection blending + if protect < 0.5: + pitchff = pitchf.clone() + pitchff[pitchf > 0] = 1 + pitchff[pitchf < 1] = protect + feats = feats * pitchff.unsqueeze(-1) + feats0 * (1 - pitchff.unsqueeze(-1)) + feats = feats.to(feats0.dtype) else: - audio1 = ( - (net_g.infer(feats, p_len, sid)[0][0, 0]).data.cpu().float().numpy() - ) - del feats, p_len, padding_mask - if torch.cuda.is_available(): - torch.cuda.empty_cache() + pitch, pitchf = None, None + p_len = torch.tensor([p_len], device=self.device).long() + audio1 = ((net_g.infer(feats, p_len, pitch, pitchf, sid)[0][0, 0]).data.cpu().float().numpy()) + # clean up + del feats, feats0, p_len + if torch.cuda.is_available(): + torch.cuda.empty_cache() return audio1 - + def _retrieve_speaker_embeddings(self, feats, index, big_npy, index_rate): npy = feats[0].cpu().numpy() npy = npy.astype("float32") if self.is_half else npy @@ -496,12 +460,9 @@ def _retrieve_speaker_embeddings(self, feats, index, big_npy, index_rate): weight /= weight.sum(axis=1, keepdims=True) npy = np.sum(big_npy[ix] * np.expand_dims(weight, axis=2), axis=1) npy = npy.astype("float16") if self.is_half else npy - feats = ( - torch.from_numpy(npy).unsqueeze(0).to(self.device) * index_rate - + (1 - index_rate) * feats - ) + feats = torch.from_numpy(npy).unsqueeze(0).to(self.device) * index_rate + (1 - index_rate) * feats return feats - + def pipeline( self, model, diff --git a/rvc/train/data_utils.py b/rvc/train/data_utils.py index 2013558f..a4e7ef0f 100644 --- a/rvc/train/data_utils.py +++ b/rvc/train/data_utils.py @@ -6,7 +6,6 @@ from mel_processing import spectrogram_torch from utils import load_filepaths_and_text, load_wav_to_torch - class TextAudioLoaderMultiNSFsid(torch.utils.data.Dataset): """ Dataset that loads text and audio pairs. @@ -164,7 +163,6 @@ def __len__(self): """ return len(self.audiopaths_and_text) - class TextAudioCollateMultiNSFsid: """ Collates text and audio data for training. @@ -243,219 +241,6 @@ def __call__(self, batch): ) -class TextAudioLoader(torch.utils.data.Dataset): - """ - Dataset that loads text and audio pairs. - - Args: - hparams: Hyperparameters. - """ - - def __init__(self, hparams): - self.audiopaths_and_text = load_filepaths_and_text(hparams.training_files) - self.max_wav_value = hparams.max_wav_value - self.sample_rate = hparams.sample_rate - self.filter_length = hparams.filter_length - self.hop_length = hparams.hop_length - self.win_length = hparams.win_length - self.sample_rate = hparams.sample_rate - self.min_text_len = getattr(hparams, "min_text_len", 1) - self.max_text_len = getattr(hparams, "max_text_len", 5000) - self._filter() - - def _filter(self): - """ - Filters audio paths and text pairs based on text length. - """ - audiopaths_and_text_new = [] - lengths = [] - for entry in self.audiopaths_and_text: - if len(entry) >= 3: - audiopath, text, dv = entry[:3] - if self.min_text_len <= len(text) and len(text) <= self.max_text_len: - audiopaths_and_text_new.append([audiopath, text, dv]) - lengths.append(os.path.getsize(audiopath) // (3 * self.hop_length)) - - self.audiopaths_and_text = audiopaths_and_text_new - self.lengths = lengths - - def get_sid(self, sid): - """ - Converts speaker ID to a LongTensor. - - Args: - sid (str): Speaker ID. - """ - try: - sid = torch.LongTensor([int(sid)]) - except ValueError as error: - print(f"Error converting speaker ID '{sid}' to integer. Exception: {error}") - sid = torch.LongTensor([0]) - return sid - - def get_audio_text_pair(self, audiopath_and_text): - """ - Loads and processes audio and text data for a single pair. - - Args: - audiopath_and_text (list): List containing audio path, text, and speaker ID. - """ - file = audiopath_and_text[0] - phone = audiopath_and_text[1] - dv = audiopath_and_text[2] - - phone = self.get_labels(phone) - spec, wav = self.get_audio(file) - dv = self.get_sid(dv) - - len_phone = phone.size()[0] - len_spec = spec.size()[-1] - if len_phone != len_spec: - len_min = min(len_phone, len_spec) - len_wav = len_min * self.hop_length - spec = spec[:, :len_min] - wav = wav[:, :len_wav] - phone = phone[:len_min, :] - return (spec, wav, phone, dv) - - def get_labels(self, phone): - """ - Loads and processes phoneme labels. - - Args: - phone (str): Path to phoneme label file. - """ - phone = np.load(phone) - phone = np.repeat(phone, 2, axis=0) - n_num = min(phone.shape[0], 900) - phone = phone[:n_num, :] - phone = torch.FloatTensor(phone) - return phone - - def get_audio(self, filename): - """ - Loads and processes audio data. - - Args: - filename (str): Path to audio file. - """ - audio, sample_rate = load_wav_to_torch(filename) - if sample_rate != self.sample_rate: - raise ValueError( - f"{sample_rate} SR doesn't match target {self.sample_rate} SR" - ) - audio_norm = audio - audio_norm = audio_norm.unsqueeze(0) - spec_filename = filename.replace(".wav", ".spec.pt") - if os.path.exists(spec_filename): - try: - spec = torch.load(spec_filename) - except Exception as error: - print(f"An error occurred getting spec from {spec_filename}: {error}") - spec = spectrogram_torch( - audio_norm, - self.filter_length, - self.hop_length, - self.win_length, - center=False, - ) - spec = torch.squeeze(spec, 0) - torch.save(spec, spec_filename, _use_new_zipfile_serialization=False) - else: - spec = spectrogram_torch( - audio_norm, - self.filter_length, - self.hop_length, - self.win_length, - center=False, - ) - spec = torch.squeeze(spec, 0) - torch.save(spec, spec_filename, _use_new_zipfile_serialization=False) - return spec, audio_norm - - def __getitem__(self, index): - """ - Returns a single audio-text pair. - - Args: - index (int): Index of the data sample. - """ - return self.get_audio_text_pair(self.audiopaths_and_text[index]) - - def __len__(self): - """ - Returns the length of the dataset. - """ - return len(self.audiopaths_and_text) - - -class TextAudioCollate: - """ - Collates text and audio data for training. - - Args: - return_ids (bool, optional): Whether to return sample IDs. Defaults to False. - """ - - def __init__(self, return_ids=False): - self.return_ids = return_ids - - def __call__(self, batch): - """ - Collates a batch of data samples. - - Args: - batch (list): List of data samples. - """ - _, ids_sorted_decreasing = torch.sort( - torch.LongTensor([x[0].size(1) for x in batch]), dim=0, descending=True - ) - - max_spec_len = max([x[0].size(1) for x in batch]) - max_wave_len = max([x[1].size(1) for x in batch]) - spec_lengths = torch.LongTensor(len(batch)) - wave_lengths = torch.LongTensor(len(batch)) - spec_padded = torch.FloatTensor(len(batch), batch[0][0].size(0), max_spec_len) - wave_padded = torch.FloatTensor(len(batch), 1, max_wave_len) - spec_padded.zero_() - wave_padded.zero_() - - max_phone_len = max([x[2].size(0) for x in batch]) - phone_lengths = torch.LongTensor(len(batch)) - phone_padded = torch.FloatTensor( - len(batch), max_phone_len, batch[0][2].shape[1] - ) - phone_padded.zero_() - sid = torch.LongTensor(len(batch)) - - for i in range(len(ids_sorted_decreasing)): - row = batch[ids_sorted_decreasing[i]] - - spec = row[0] - spec_padded[i, :, : spec.size(1)] = spec - spec_lengths[i] = spec.size(1) - - wave = row[1] - wave_padded[i, :, : wave.size(1)] = wave - wave_lengths[i] = wave.size(1) - - phone = row[2] - phone_padded[i, : phone.size(0), :] = phone - phone_lengths[i] = phone.size(0) - - sid[i] = row[3] - - return ( - phone_padded, - phone_lengths, - spec_padded, - spec_lengths, - wave_padded, - wave_lengths, - sid, - ) - - class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler): """ Distributed sampler that groups data into buckets based on length. diff --git a/rvc/train/train.py b/rvc/train/train.py index 014abbb5..52cd4f8a 100644 --- a/rvc/train/train.py +++ b/rvc/train/train.py @@ -36,9 +36,7 @@ from data_utils import ( DistributedBucketSampler, - TextAudioCollate, TextAudioCollateMultiNSFsid, - TextAudioLoader, TextAudioLoaderMultiNSFsid, ) @@ -74,7 +72,6 @@ overtraining_detector = strtobool(sys.argv[14]) overtraining_threshold = int(sys.argv[15]) sync_graph = strtobool(sys.argv[16]) -use_cpu = strtobool(sys.argv[17]) current_dir = os.getcwd() experiment_dir = os.path.join(current_dir, "logs", model_name) @@ -86,10 +83,6 @@ config = HParams(**config) config.data.training_files = os.path.join(experiment_dir, "filelist.txt") -if not use_cpu: - os.environ["CUDA_VISIBLE_DEVICES"] = gpus.replace("-", ",") -n_gpus = len(gpus.split("-")) if not use_cpu else 1 - torch.backends.cudnn.deterministic = False torch.backends.cudnn.benchmark = False @@ -102,7 +95,6 @@ smoothed_loss_disc_history = [] lowest_value = {"step": 0, "value": float("inf"), "epoch": 0} training_file_path = os.path.join(experiment_dir, "training_data.json") -overtrain_info = None import logging @@ -156,37 +148,25 @@ def main(): os.environ["MASTER_ADDR"] = "localhost" os.environ["MASTER_PORT"] = str(randint(20000, 55555)) # Check sample rate - first_wav_file = next( - ( - filename - for filename in os.listdir(os.path.join(experiment_dir, "sliced_audios")) - if filename.endswith(".wav") - ), - None, - ) - if first_wav_file: - audio = os.path.join(experiment_dir, "sliced_audios", first_wav_file) - _, sr = load_wav_to_torch(audio) + wavs = glob.glob(os.path.join(os.path.join(experiment_dir, "sliced_audios"), "*.wav")) + if wavs: + _, sr = load_wav_to_torch(wavs[0]) if sr != sample_rate: - try: - raise ValueError( - f"Error: Pretrained model sample rate ({sample_rate} Hz) does not match dataset audio sample rate ({sr} Hz)." - ) - except ValueError as e: - print( - f"Error: Pretrained model sample rate ({sample_rate} Hz) does not match dataset audio sample rate ({sr} Hz)." - ) - sys.exit(1) + print(f"Error: Pretrained model sample rate ({sample_rate} Hz) does not match dataset audio sample rate ({sr} Hz).") + os._exit(1) else: print("No wav file found.") - use_gpu = torch.cuda.is_available() and not use_cpu - device = torch.device("cuda" if use_gpu else "cpu") - - if use_gpu: + if torch.cuda.is_available(): + device = torch.device("cuda") n_gpus = torch.cuda.device_count() + elif torch.backends.mps.is_available(): + device = torch.device("mps") + n_gpus = 1 else: + device = torch.device("cpu") n_gpus = 1 + print("Training with CPU, this will take a long time.") def start(): """ @@ -259,24 +239,11 @@ def continue_overtrain_detector(training_file_path): smoothed_loss_gen_history, ) = load_from_json(training_file_path) - if use_cpu: - n_gpus = 1 - print("Training with CPU, this will take a long time.") - else: - n_gpus = torch.cuda.device_count() - - if torch.cuda.is_available() == False and torch.backends.mps.is_available() == True: - n_gpus = 1 - if n_gpus < 1 and not use_cpu: - print("GPU not detected, reverting to CPU (not recommended)") - n_gpus = 1 - - if sync_graph == True: - print( - "Sync graph is now activated! With sync graph enabled, the model undergoes a single epoch of training. Once the graphs are synchronized, training proceeds for the previously specified number of epochs." - ) + if sync_graph: + print("Sync graph is now activated! With sync graph enabled, the model undergoes a single epoch of training. Once the graphs are synchronized, training proceeds for the previously specified number of epochs.") custom_total_epoch = 1 custom_save_every_weights = True + start() # Synchronize graphs by modifying config files @@ -321,19 +288,14 @@ def edit_config(config_file): edit_config(rvc_config_file) # Clean up unnecessary files - for root, dirs, files in os.walk( - os.path.join(now_dir, "logs", model_name), topdown=False - ): + for root, dirs, files in os.walk(os.path.join(now_dir, "logs", model_name), topdown=False): for name in files: file_path = os.path.join(root, name) file_name, file_extension = os.path.splitext(name) - if file_extension == ".0": - os.remove(file_path) - elif ("D" in name or "G" in name) and file_extension == ".pth": - os.remove(file_path) - elif ( - "added" in name or "trained" in name - ) and file_extension == ".index": + if (file_extension == ".0" or + (file_name.startswith("D_") and file_extension == ".pth") or + (file_name.startswith("G_") and file_extension == ".pth") or + (file_name.startswith("added") and file_extension == ".index")): os.remove(file_path) for name in dirs: if name == "eval": @@ -355,7 +317,6 @@ def edit_config(config_file): continue_overtrain_detector(training_file_path) start() - def run( rank, n_gpus, @@ -403,23 +364,16 @@ def run( if torch.cuda.is_available(): torch.cuda.set_device(rank) - - # Zluda - if torch.cuda.is_available() and torch.cuda.get_device_name().endswith("[ZLUDA]"): - print("Disabling CUDNN for traning with Zluda") - torch.backends.cudnn.enabled = False - torch.backends.cuda.enable_flash_sdp(False) - torch.backends.cuda.enable_math_sdp(True) - torch.backends.cuda.enable_mem_efficient_sdp(False) + if torch.cuda.get_device_name().endswith("[ZLUDA]"): + print("Disabling CUDNN for traning with Zluda") + torch.backends.cudnn.enabled = False + torch.backends.cuda.enable_flash_sdp(False) + torch.backends.cuda.enable_math_sdp(True) + torch.backends.cuda.enable_mem_efficient_sdp(False) # Create datasets and dataloaders - if pitch_guidance == True: - train_dataset = TextAudioLoaderMultiNSFsid(config.data) - elif pitch_guidance == False: - train_dataset = TextAudioLoader(config.data) - else: - raise ValueError(f"Unexpected value for pitch_guidance: {pitch_guidance}") - + train_dataset = TextAudioLoaderMultiNSFsid(config.data) + collate_fn = TextAudioCollateMultiNSFsid() train_sampler = DistributedBucketSampler( train_dataset, batch_size * n_gpus, @@ -429,11 +383,6 @@ def run( shuffle=True, ) - if pitch_guidance == True: - collate_fn = TextAudioCollateMultiNSFsid() - elif pitch_guidance == False: - collate_fn = TextAudioCollate() - train_loader = DataLoader( train_dataset, num_workers=4, @@ -450,49 +399,29 @@ def run( config.data.filter_length // 2 + 1, config.train.segment_size // config.data.hop_length, **config.model, - use_f0=pitch_guidance == True, + use_f0=pitch_guidance, is_half=config.train.fp16_run and device.type == "cuda", sr=sample_rate, - ) - - net_g = net_g.to(device) + ).to(device) if version == "v1": - net_d = MultiPeriodDiscriminator(config.model.use_spectral_norm) + net_d = MultiPeriodDiscriminator(config.model.use_spectral_norm).to(device) else: - net_d = MultiPeriodDiscriminatorV2(config.model.use_spectral_norm) + net_d = MultiPeriodDiscriminatorV2(config.model.use_spectral_norm).to(device) - net_d = net_d.to(device) - - optim_g = torch.optim.AdamW( - net_g.parameters(), - config.train.learning_rate, - betas=config.train.betas, - eps=config.train.eps, - ) - optim_d = torch.optim.AdamW( - net_d.parameters(), - config.train.learning_rate, - betas=config.train.betas, - eps=config.train.eps, - ) + optim_g = torch.optim.AdamW(net_g.parameters(), config.train.learning_rate, betas=config.train.betas, eps=config.train.eps) + optim_d = torch.optim.AdamW(net_d.parameters(), config.train.learning_rate, betas=config.train.betas, eps=config.train.eps) # Wrap models with DDP for multi-gpu processing if n_gpus > 1 and device.type == "cuda": net_g = DDP(net_g, device_ids=[rank]) net_d = DDP(net_d, device_ids=[rank]) - # else: - # net_g = DDP(net_g) - # net_d = DDP(net_d) + # Load checkpoint if available try: print("Starting training...") - _, _, _, epoch_str = load_checkpoint( - latest_checkpoint_path(experiment_dir, "D_*.pth"), net_d, optim_d - ) - _, _, _, epoch_str = load_checkpoint( - latest_checkpoint_path(experiment_dir, "G_*.pth"), net_g, optim_g - ) + _, _, _, epoch_str = load_checkpoint(latest_checkpoint_path(experiment_dir, "D_*.pth"), net_d, optim_d) + _, _, _, epoch_str = load_checkpoint(latest_checkpoint_path(experiment_dir, "G_*.pth"), net_g, optim_g) epoch_str += 1 global_step = (epoch_str - 1) * len(train_loader) @@ -504,40 +433,26 @@ def run( verify_checkpoint_shapes(pretrainG, net_g) print(f"Loaded pretrained (G) '{pretrainG}'") if hasattr(net_g, "module"): - net_g.module.load_state_dict( - torch.load(pretrainG, map_location="cpu")["model"] - ) - + net_g.module.load_state_dict(torch.load(pretrainG, map_location="cpu")["model"]) else: - net_g.load_state_dict( - torch.load(pretrainG, map_location="cpu")["model"] - ) + net_g.load_state_dict(torch.load(pretrainG, map_location="cpu")["model"]) if pretrainD != "": if rank == 0: print(f"Loaded pretrained (D) '{pretrainD}'") if hasattr(net_d, "module"): - net_d.module.load_state_dict( - torch.load(pretrainD, map_location="cpu")["model"] - ) - + net_d.module.load_state_dict(torch.load(pretrainD, map_location="cpu")["model"]) else: - net_d.load_state_dict( - torch.load(pretrainD, map_location="cpu")["model"] - ) + net_d.load_state_dict(torch.load(pretrainD, map_location="cpu")["model"]) # Initialize schedulers and scaler - scheduler_g = torch.optim.lr_scheduler.ExponentialLR( - optim_g, gamma=config.train.lr_decay, last_epoch=epoch_str - 2 - ) - scheduler_d = torch.optim.lr_scheduler.ExponentialLR( - optim_d, gamma=config.train.lr_decay, last_epoch=epoch_str - 2 - ) + scheduler_g = torch.optim.lr_scheduler.ExponentialLR(optim_g, gamma=config.train.lr_decay, last_epoch=epoch_str - 2) + scheduler_d = torch.optim.lr_scheduler.ExponentialLR(optim_d, gamma=config.train.lr_decay, last_epoch=epoch_str - 2) - optim_d.step() optim_g.step() + optim_d.step() - scaler = GradScaler(enabled=config.train.fp16_run and not use_cpu) + scaler = GradScaler(enabled=config.train.fp16_run and device.type == "cuda") cache = [] for epoch in range(epoch_str, total_epoch + 1): @@ -554,7 +469,7 @@ def run( cache, custom_save_every_weights, custom_total_epoch, - use_cpu, + device, ) else: train_and_evaluate( @@ -569,12 +484,11 @@ def run( cache, custom_save_every_weights, custom_total_epoch, - use_cpu, + device, ) scheduler_g.step() scheduler_d.step() - def train_and_evaluate( rank, epoch, @@ -587,7 +501,7 @@ def train_and_evaluate( cache, custom_save_every_weights, custom_total_epoch, - use_cpu, + device, ): """ Trains and evaluates the model for one epoch. @@ -624,75 +538,27 @@ def train_and_evaluate( net_d.train() # Data caching - if cache_data_in_gpu and not use_cpu: + if device.type == "cuda" and cache_data_in_gpu: data_iterator = cache if cache == []: for batch_idx, info in enumerate(train_loader): - if pitch_guidance == True: + phone, phone_lengths, pitch, pitchf, spec, spec_lengths, wave, wave_lengths, sid = info + cache.append( ( - phone, - phone_lengths, - pitch, - pitchf, - spec, - spec_lengths, - wave, - wave_lengths, - sid, - ) = info - elif pitch_guidance == False: - ( - phone, - phone_lengths, - spec, - spec_lengths, - wave, - wave_lengths, - sid, - ) = info - if torch.cuda.is_available(): - phone = phone.cuda(rank, non_blocking=True) - phone_lengths = phone_lengths.cuda(rank, non_blocking=True) - if pitch_guidance == True: - pitch = pitch.cuda(rank, non_blocking=True) - pitchf = pitchf.cuda(rank, non_blocking=True) - sid = sid.cuda(rank, non_blocking=True) - spec = spec.cuda(rank, non_blocking=True) - spec_lengths = spec_lengths.cuda(rank, non_blocking=True) - wave = wave.cuda(rank, non_blocking=True) - wave_lengths = wave_lengths.cuda(rank, non_blocking=True) - if pitch_guidance == True: - cache.append( + batch_idx, ( - batch_idx, - ( - phone, - phone_lengths, - pitch, - pitchf, - spec, - spec_lengths, - wave, - wave_lengths, - sid, - ), - ) - ) - elif pitch_guidance == False: - cache.append( - ( - batch_idx, - ( - phone, - phone_lengths, - spec, - spec_lengths, - wave, - wave_lengths, - sid, - ), + phone.cuda(rank, non_blocking=True), + phone_lengths.cuda(rank, non_blocking=True), + pitch.cuda(rank, non_blocking=True) if pitch_guidance else None, + pitchf.cuda(rank, non_blocking=True) if pitch_guidance else None, + spec.cuda(rank, non_blocking=True), + spec_lengths.cuda(rank, non_blocking=True), + wave.cuda(rank, non_blocking=True), + wave_lengths.cuda(rank, non_blocking=True), + sid.cuda(rank, non_blocking=True) ) ) + ) else: shuffle(cache) else: @@ -701,81 +567,41 @@ def train_and_evaluate( epoch_recorder = EpochRecorder() with tqdm(total=len(train_loader), leave=False) as pbar: for batch_idx, info in data_iterator: - if pitch_guidance == True: - ( - phone, - phone_lengths, - pitch, - pitchf, - spec, - spec_lengths, - wave, - wave_lengths, - sid, - ) = info - elif pitch_guidance == False: - phone, phone_lengths, spec, spec_lengths, wave, wave_lengths, sid = info - if ( - (cache_data_in_gpu == False) - and not use_cpu - and torch.cuda.is_available() - ): + phone, phone_lengths, pitch, pitchf, spec, spec_lengths, wave, wave_lengths, sid = info + if device.type == "cuda" and not cache_data_in_gpu: phone = phone.cuda(rank, non_blocking=True) phone_lengths = phone_lengths.cuda(rank, non_blocking=True) - if pitch_guidance == True: - pitch = pitch.cuda(rank, non_blocking=True) - pitchf = pitchf.cuda(rank, non_blocking=True) + pitch = pitch.cuda(rank, non_blocking=True) if pitch_guidance else None + pitchf = pitchf.cuda(rank, non_blocking=True) if pitch_guidance else None sid = sid.cuda(rank, non_blocking=True) spec = spec.cuda(rank, non_blocking=True) spec_lengths = spec_lengths.cuda(rank, non_blocking=True) wave = wave.cuda(rank, non_blocking=True) wave_lengths = wave_lengths.cuda(rank, non_blocking=True) - elif use_cpu: - phone = phone.cpu() - phone_lengths = phone_lengths.cpu() - if pitch_guidance == True: - pitch = pitch.cpu() - pitchf = pitchf.cpu() - sid = sid.cpu() - spec = spec.cpu() - spec_lengths = spec_lengths.cpu() - wave = wave.cpu() - wave_lengths = wave_lengths.cpu() + else: + phone = phone.to(device) + phone_lengths = phone_lengths.to(device) + pitch = pitch.to(device) if pitch_guidance else None + pitchf = pitchf.to(device) if pitch_guidance else None + sid = sid.to(device) + spec = spec.to(device) + spec_lengths = spec_lengths.to(device) + wave = wave.to(device) + wave_lengths = wave_lengths.to(device) # Forward pass - with autocast(enabled=config.train.fp16_run and not use_cpu): - if pitch_guidance == True: - ( - y_hat, - ids_slice, - x_mask, - z_mask, - (z, z_p, m_p, logs_p, m_q, logs_q), - ) = net_g( - phone, phone_lengths, pitch, pitchf, spec, spec_lengths, sid - ) - elif pitch_guidance == False: - ( - y_hat, - ids_slice, - x_mask, - z_mask, - (z, z_p, m_p, logs_p, m_q, logs_q), - ) = net_g(phone, phone_lengths, spec, spec_lengths, sid) + use_amp = config.train.fp16_run and device.type == "cuda" + with autocast(enabled=use_amp): + (y_hat, ids_slice, x_mask, z_mask, (z, z_p, m_p, logs_p, m_q, logs_q)) = net_g(phone, phone_lengths, pitch, pitchf, spec, spec_lengths, sid) mel = spec_to_mel_torch( spec, config.data.filter_length, config.data.n_mel_channels, config.data.sample_rate, config.data.mel_fmin, - config.data.mel_fmax, - ) - y_mel = commons.slice_segments( - mel, - ids_slice, - config.train.segment_size // config.data.hop_length, - dim=3, + config.data.mel_fmax ) + y_mel = commons.slice_segments(mel, ids_slice, config.train.segment_size // config.data.hop_length, dim=3) with autocast(enabled=False): y_hat_mel = mel_spectrogram_torch( y_hat.float().squeeze(1), @@ -785,23 +611,14 @@ def train_and_evaluate( config.data.hop_length, config.data.win_length, config.data.mel_fmin, - config.data.mel_fmax, + config.data.mel_fmax ) - if config.train.fp16_run == True and not use_cpu: + if use_amp: y_hat_mel = y_hat_mel.half() - wave = commons.slice_segments( - wave, - ids_slice * config.data.hop_length, - config.train.segment_size, - dim=3, - ) - + wave = commons.slice_segments(wave, ids_slice * config.data.hop_length, config.train.segment_size, dim=3) y_d_hat_r, y_d_hat_g, _, _ = net_d(wave, y_hat.detach()) with autocast(enabled=False): - loss_disc, losses_disc_r, losses_disc_g = discriminator_loss( - y_d_hat_r, y_d_hat_g - ) - + loss_disc, losses_disc_r, losses_disc_g = discriminator_loss(y_d_hat_r, y_d_hat_g) # Discriminator backward and update optim_d.zero_grad() scaler.scale(loss_disc).backward() @@ -810,13 +627,11 @@ def train_and_evaluate( scaler.step(optim_d) # Generator backward and update - with autocast(enabled=config.train.fp16_run and not use_cpu): + with autocast(enabled=use_amp): y_d_hat_r, y_d_hat_g, fmap_r, fmap_g = net_d(wave, y_hat) with autocast(enabled=False): loss_mel = F.l1_loss(y_mel, y_hat_mel) * config.train.c_mel - loss_kl = ( - kl_loss(z_p, logs_q, m_p, logs_p, z_mask) * config.train.c_kl - ) + loss_kl = (kl_loss(z_p, logs_q, m_p, logs_p, z_mask) * config.train.c_kl) loss_fm = feature_loss(fmap_r, fmap_g) loss_gen, losses_gen = generator_loss(y_d_hat_g) loss_gen_all = loss_gen + loss_fm + loss_mel + loss_kl @@ -827,9 +642,7 @@ def train_and_evaluate( lowest_value["epoch"] = epoch # print(f'Lowest generator loss updated: {lowest_value["value"]} at epoch {epoch}, step {global_step}') if epoch > lowest_value["epoch"]: - print( - "Alert: The lower generating loss has been exceeded by a lower loss in a subsequent epoch." - ) + print("Alert: The lower generating loss has been exceeded by a lower loss in a subsequent epoch.") optim_g.zero_grad() scaler.scale(loss_gen_all).backward() @@ -842,12 +655,10 @@ def train_and_evaluate( if rank == 0: if global_step % config.train.log_interval == 0: lr = optim_g.param_groups[0]["lr"] - if loss_mel > 75: loss_mel = 75 if loss_kl > 9: loss_kl = 9 - scalar_dict = { "loss/g/total": loss_gen_all, "loss/d/total": loss_disc, @@ -855,29 +666,13 @@ def train_and_evaluate( "grad_norm_d": grad_norm_d, "grad_norm_g": grad_norm_g, } - scalar_dict.update( - { - "loss/g/fm": loss_fm, - "loss/g/mel": loss_mel, - "loss/g/kl": loss_kl, - } - ) - scalar_dict.update( - {f"loss/g/{i}": v for i, v in enumerate(losses_gen)} - ) - scalar_dict.update( - {f"loss/d_r/{i}": v for i, v in enumerate(losses_disc_r)} - ) - scalar_dict.update( - {f"loss/d_g/{i}": v for i, v in enumerate(losses_disc_g)} - ) + scalar_dict.update({"loss/g/fm": loss_fm, "loss/g/mel": loss_mel, "loss/g/kl": loss_kl}) + scalar_dict.update({f"loss/g/{i}": v for i, v in enumerate(losses_gen)}) + scalar_dict.update({f"loss/d_r/{i}": v for i, v in enumerate(losses_disc_r)}) + scalar_dict.update({f"loss/d_g/{i}": v for i, v in enumerate(losses_disc_g)}) image_dict = { - "slice/mel_org": plot_spectrogram_to_numpy( - y_mel[0].data.cpu().numpy() - ), - "slice/mel_gen": plot_spectrogram_to_numpy( - y_hat_mel[0].data.cpu().numpy() - ), + "slice/mel_org": plot_spectrogram_to_numpy(y_mel[0].data.cpu().numpy()), + "slice/mel_gen": plot_spectrogram_to_numpy(y_hat_mel[0].data.cpu().numpy()), "all/mel": plot_spectrogram_to_numpy(mel[0].data.cpu().numpy()), } summarize( @@ -891,265 +686,166 @@ def train_and_evaluate( pbar.update(1) # Save checkpoint - if epoch % save_every_epoch == False and rank == 0: - checkpoint_suffix = ( - f"{global_step if save_only_latest == False else 2333333}.pth" - ) - save_checkpoint( - net_g, - optim_g, - config.train.learning_rate, - epoch, - os.path.join(experiment_dir, "G_" + checkpoint_suffix), - ) - save_checkpoint( - net_d, - optim_d, - config.train.learning_rate, - epoch, - os.path.join(experiment_dir, "D_" + checkpoint_suffix), - ) - if rank == 0 and custom_save_every_weights == True: - if hasattr(net_g, "module"): - ckpt = net_g.module.state_dict() + model_add = [] + model_del = [] + done = False + + if rank == 0: + # Save weights every N epochs + if epoch % save_every_epoch == 0: + checkpoint_suffix = (f"{2333333 if save_only_latest else global_step}.pth") + save_checkpoint(net_g, optim_g, config.train.learning_rate, epoch, os.path.join(experiment_dir, "G_" + checkpoint_suffix)) + save_checkpoint(net_d, optim_d, config.train.learning_rate, epoch, os.path.join(experiment_dir, "D_" + checkpoint_suffix)) + if custom_save_every_weights: + model_add.append(os.path.join(experiment_dir, f"{model_name}_{epoch}e_{global_step}s.pth")) + overtrain_info = "" + # Check overtraining + if overtraining_detector and rank == 0 and epoch > 1: + # Add the current loss to the history + current_loss_disc = float(loss_disc) + loss_disc_history.append(current_loss_disc) + # Update smoothed loss history with loss_disc + smoothed_value_disc = update_exponential_moving_average(smoothed_loss_disc_history, current_loss_disc) + # Check overtraining with smoothed loss_disc + is_overtraining_disc = check_overtraining(smoothed_loss_disc_history, overtraining_threshold * 2) + if is_overtraining_disc: + consecutive_increases_disc += 1 else: - ckpt = net_g.state_dict() - if overtraining_detector and epoch > 1: - overtrain_info = f"Smoothed loss_g {smoothed_value_gen:.3f} and loss_d {smoothed_value_disc:.3f}" + consecutive_increases_disc = 0 + # Add the current loss_gen to the history + current_loss_gen = float(lowest_value["value"]) + loss_gen_history.append(current_loss_gen) + # Update the smoothed loss_gen history + smoothed_value_gen = update_exponential_moving_average(smoothed_loss_gen_history, current_loss_gen) + # Check for overtraining with the smoothed loss_gen + is_overtraining_gen = check_overtraining(smoothed_loss_gen_history, overtraining_threshold, 0.01) + if is_overtraining_gen: + consecutive_increases_gen += 1 else: - overtrain_info = "" - extract_model( - ckpt=ckpt, - sr=sample_rate, - pitch_guidance=pitch_guidance == True, - name=model_name, - model_dir=os.path.join( - experiment_dir, - f"{model_name}_{epoch}e_{global_step}s.pth", - ), - epoch=epoch, - step=global_step, - version=version, - hps=hps, - overtrain_info=overtrain_info, - ) - - def check_overtraining(smoothed_loss_history, threshold, epsilon=0.004): - """ - Checks for overtraining based on the smoothed loss history. - - Args: - smoothed_loss_history (list): List of smoothed losses for each epoch. - threshold (int): Number of consecutive epochs with insignificant changes or increases to consider overtraining. - epsilon (float): The maximum change considered insignificant. - """ - if len(smoothed_loss_history) < threshold + 1: - return False - - for i in range(-threshold, -1): - if smoothed_loss_history[i + 1] > smoothed_loss_history[i]: - return True - if abs(smoothed_loss_history[i + 1] - smoothed_loss_history[i]) >= epsilon: - return False - - return True - - def update_exponential_moving_average( - smoothed_loss_history, new_value, smoothing=0.987 - ): - """ - Updates the exponential moving average with a new value. - - Args: - smoothed_loss_history (list): List of smoothed values. - new_value (float): New value to be added. - smoothing (float): Smoothing factor. - """ - if not smoothed_loss_history: - smoothed_value = new_value - else: - smoothed_value = ( - smoothing * smoothed_loss_history[-1] + (1 - smoothing) * new_value - ) - smoothed_loss_history.append(smoothed_value) - return smoothed_value - - def save_to_json( - file_path, - loss_disc_history, - smoothed_loss_disc_history, - loss_gen_history, - smoothed_loss_gen_history, - ): - """ - Save the training history to a JSON file. - """ - data = { - "loss_disc_history": loss_disc_history, - "smoothed_loss_disc_history": smoothed_loss_disc_history, - "loss_gen_history": loss_gen_history, - "smoothed_loss_gen_history": smoothed_loss_gen_history, - } - with open(file_path, "w") as f: - json.dump(data, f) - - if overtraining_detector and rank == 0 and epoch > 1: - # Add the current loss to the history - current_loss_disc = float(loss_disc) - loss_disc_history.append(current_loss_disc) - - # Update smoothed loss history with loss_disc - smoothed_value_disc = update_exponential_moving_average( - smoothed_loss_disc_history, current_loss_disc - ) - - # Check overtraining with smoothed loss_disc - is_overtraining_disc = check_overtraining( - smoothed_loss_disc_history, overtraining_threshold * 2 - ) - if is_overtraining_disc: - consecutive_increases_disc += 1 - else: - consecutive_increases_disc = 0 - # Add the current loss_gen to the history - current_loss_gen = float(lowest_value["value"]) - loss_gen_history.append(current_loss_gen) - - # Update the smoothed loss_gen history - smoothed_value_gen = update_exponential_moving_average( - smoothed_loss_gen_history, current_loss_gen - ) - - # Check for overtraining with the smoothed loss_gen - is_overtraining_gen = check_overtraining( - smoothed_loss_gen_history, overtraining_threshold, 0.01 - ) - if is_overtraining_gen: - consecutive_increases_gen += 1 - else: - consecutive_increases_gen = 0 - - overtrain_info = f"Smoothed loss_g {smoothed_value_gen:.3f} and loss_d {smoothed_value_disc:.3f}" - # Save the data in the JSON file if the epoch is divisible by save_every_epoch - if epoch % save_every_epoch == 0: - save_to_json( - training_file_path, - loss_disc_history, - smoothed_loss_disc_history, - loss_gen_history, - smoothed_loss_gen_history, - ) - - if ( - is_overtraining_gen - and consecutive_increases_gen == overtraining_threshold - or is_overtraining_disc - and consecutive_increases_disc == (overtraining_threshold * 2) - ): - print( - f"Overtraining detected at epoch {epoch} with smoothed loss_g {smoothed_value_gen:.3f} and loss_d {smoothed_value_disc:.3f}" - ) - os._exit(2333333) - else: - print( - f"New best epoch {epoch} with smoothed loss_g {smoothed_value_gen:.3f} and loss_d {smoothed_value_disc:.3f}" - ) - old_model_files = glob.glob( - os.path.join(experiment_dir, f"{model_name}_*e_*s_best_epoch.pth") - ) - for file in old_model_files: - os.remove(file) - - if hasattr(net_g, "module"): - ckpt = net_g.module.state_dict() + consecutive_increases_gen = 0 + overtrain_info = f"Smoothed loss_g {smoothed_value_gen:.3f} and loss_d {smoothed_value_disc:.3f}" + # Save the data in the JSON file if the epoch is divisible by save_every_epoch + if epoch % save_every_epoch == 0: + save_to_json(training_file_path, loss_disc_history, smoothed_loss_disc_history, loss_gen_history, smoothed_loss_gen_history) + + if is_overtraining_gen and consecutive_increases_gen == overtraining_threshold or is_overtraining_disc and consecutive_increases_disc == overtraining_threshold * 2: + print(f"Overtraining detected at epoch {epoch} with smoothed loss_g {smoothed_value_gen:.3f} and loss_d {smoothed_value_disc:.3f}") + done = True else: - ckpt = net_g.state_dict() - if overtraining_detector != True: - overtrain_info = None - extract_model( - ckpt=ckpt, - sr=sample_rate, - pitch_guidance=pitch_guidance == True, - name=model_name, - model_dir=os.path.join( - experiment_dir, - f"{model_name}_{epoch}e_{global_step}s_best_epoch.pth", - ), - epoch=epoch, - step=global_step, - version=version, - hps=hps, - overtrain_info=overtrain_info, - ) + print(f"New best epoch {epoch} with smoothed loss_g {smoothed_value_gen:.3f} and loss_d {smoothed_value_disc:.3f}") + old_model_files = glob.glob(os.path.join(experiment_dir, f"{model_name}_*e_*s_best_epoch.pth")) + for file in old_model_files: + model_del.append(file) + model_add.append(os.path.join(experiment_dir, f"{model_name}_{epoch}e_{global_step}s_best_epoch.pth")) + + # Check completion + if epoch >= custom_total_epoch: + lowest_value_rounded = float(lowest_value["value"]) + lowest_value_rounded = round(lowest_value_rounded, 3) + print(f"Training has been successfully completed with {epoch} epoch, {global_step} steps and {round(loss_gen_all.item(), 3)} loss gen.") + print(f"Lowest generator loss: {lowest_value_rounded} at epoch {lowest_value['epoch']}, step {lowest_value['step']}") + + pid_file_path = os.path.join(experiment_dir, "config.json") + with open(pid_file_path, "r") as pid_file: + pid_data = json.load(pid_file) + with open(pid_file_path, "w") as pid_file: + pid_data.pop("process_pids", None) + json.dump(pid_data, pid_file, indent=4) + # Final model + model_add.append(os.path.join(experiment_dir, f"{model_name}_{epoch}e_{global_step}s.pth")) + done = True + + if model_add: + ckpt = net_g.module.state_dict() if hasattr(net_g, "module") else net_g.state_dict() + for m in model_add: + if not os.path.exists(m): + extract_model( + ckpt=ckpt, + sr=sample_rate, + pitch_guidance=pitch_guidance, + name=model_name, + model_dir=m, + epoch=epoch, + step=global_step, + version=version, + hps=hps, + overtrain_info=overtrain_info + ) + # Clean-up old best epochs + for m in model_del: + os.remove(m) - # Print training progress - if rank == 0: + # Print training progress lowest_value_rounded = float(lowest_value["value"]) lowest_value_rounded = round(lowest_value_rounded, 3) - if epoch > 1 and overtraining_detector == True: + record = f"{model_name} | epoch={epoch} | step={global_step} | {epoch_recorder.record()}" + if epoch > 1: + record = record + f" | lowest_value={lowest_value_rounded} (epoch {lowest_value['epoch']} and step {lowest_value['step']}" + + if overtraining_detector: remaining_epochs_gen = overtraining_threshold - consecutive_increases_gen - remaining_epochs_disc = ( - overtraining_threshold * 2 - ) - consecutive_increases_disc - print( - f"{model_name} | epoch={epoch} | step={global_step} | {epoch_recorder.record()} | lowest_value={lowest_value_rounded} (epoch {lowest_value['epoch']} and step {lowest_value['step']}) | Number of epochs remaining for overtraining: g/total: {remaining_epochs_gen} d/total: {remaining_epochs_disc} | smoothed_loss_gen={smoothed_value_gen:.3f} | smoothed_loss_disc={smoothed_value_disc:.3f}" - ) - elif epoch > 1 and overtraining_detector == False: - print( - f"{model_name} | epoch={epoch} | step={global_step} | {epoch_recorder.record()} | lowest_value={lowest_value_rounded} (epoch {lowest_value['epoch']} and step {lowest_value['step']})" - ) - else: - print( - f"{model_name} | epoch={epoch} | step={global_step} | {epoch_recorder.record()}" - ) + remaining_epochs_disc = overtraining_threshold * 2 - consecutive_increases_disc + record = record + f" | Number of epochs remaining for overtraining: g/total: {remaining_epochs_gen} d/total: {remaining_epochs_disc} | smoothed_loss_gen={smoothed_value_gen:.3f} | smoothed_loss_disc={smoothed_value_disc:.3f}" + print(record) last_loss_gen_all = loss_gen_all - # Save the final model - if epoch >= custom_total_epoch and rank == 0: - lowest_value_rounded = float(lowest_value["value"]) - lowest_value_rounded = round(lowest_value_rounded, 3) - print( - f"Training has been successfully completed with {epoch} epoch, {global_step} steps and {round(loss_gen_all.item(), 3)} loss gen." - ) - print( - f"Lowest generator loss: {lowest_value_rounded} at epoch {lowest_value['epoch']}, step {lowest_value['step']}" - ) + if done: + os._exit(2333333) - pid_file_path = os.path.join(experiment_dir, "config.json") - with open(pid_file_path, "r") as pid_file: - pid_data = json.load(pid_file) - with open(pid_file_path, "w") as pid_file: - pid_data.pop("process_pids", None) - json.dump(pid_data, pid_file, indent=4) +def check_overtraining(smoothed_loss_history, threshold, epsilon=0.004): + """ + Checks for overtraining based on the smoothed loss history. - if not os.path.exists( - os.path.join(experiment_dir, f"{model_name}_{epoch}e_{global_step}s.pth") - ): - if hasattr(net_g, "module"): - ckpt = net_g.module.state_dict() - else: - ckpt = net_g.state_dict() - if overtraining_detector != True: - overtrain_info = None - extract_model( - ckpt=ckpt, - sr=sample_rate, - pitch_guidance=pitch_guidance == True, - name=model_name, - model_dir=os.path.join( - experiment_dir, - f"{model_name}_{epoch}e_{global_step}s.pth", - ), - epoch=epoch, - step=global_step, - version=version, - hps=hps, - overtrain_info=overtrain_info, - ) - sleep(1) - os._exit(2333333) + Args: + smoothed_loss_history (list): List of smoothed losses for each epoch. + threshold (int): Number of consecutive epochs with insignificant changes or increases to consider overtraining. + epsilon (float): The maximum change considered insignificant. + """ + if len(smoothed_loss_history) < threshold + 1: + return False + + for i in range(-threshold, -1): + if smoothed_loss_history[i + 1] > smoothed_loss_history[i]: + return True + if abs(smoothed_loss_history[i + 1] - smoothed_loss_history[i]) >= epsilon: + return False + return True +def update_exponential_moving_average(smoothed_loss_history, new_value, smoothing=0.987): + """ + Updates the exponential moving average with a new value. + + Args: + smoothed_loss_history (list): List of smoothed values. + new_value (float): New value to be added. + smoothing (float): Smoothing factor. + """ + if smoothed_loss_history: + smoothed_value = smoothing * smoothed_loss_history[-1] + (1 - smoothing) * new_value + else: + smoothed_value = new_value + smoothed_loss_history.append(smoothed_value) + return smoothed_value + +def save_to_json( + file_path, + loss_disc_history, + smoothed_loss_disc_history, + loss_gen_history, + smoothed_loss_gen_history, + ): + """ + Save the training history to a JSON file. + """ + data = { + "loss_disc_history": loss_disc_history, + "smoothed_loss_disc_history": smoothed_loss_disc_history, + "loss_gen_history": loss_gen_history, + "smoothed_loss_gen_history": smoothed_loss_gen_history, + } + with open(file_path, "w") as f: + json.dump(data, f) if __name__ == "__main__": torch.multiprocessing.set_start_method("spawn") From 5db5e53ad30cc69007e576d96f85fdd932c9fea8 Mon Sep 17 00:00:00 2001 From: Alexey Shmelev Date: Tue, 17 Sep 2024 12:34:30 -0400 Subject: [PATCH 04/11] added missing ) for the log output --- rvc/train/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rvc/train/train.py b/rvc/train/train.py index 52cd4f8a..4054cad2 100644 --- a/rvc/train/train.py +++ b/rvc/train/train.py @@ -781,7 +781,7 @@ def train_and_evaluate( record = f"{model_name} | epoch={epoch} | step={global_step} | {epoch_recorder.record()}" if epoch > 1: - record = record + f" | lowest_value={lowest_value_rounded} (epoch {lowest_value['epoch']} and step {lowest_value['step']}" + record = record + f" | lowest_value={lowest_value_rounded} (epoch {lowest_value['epoch']} and step {lowest_value['step']})" if overtraining_detector: remaining_epochs_gen = overtraining_threshold - consecutive_increases_gen From 87da6ba9c57c051efd1339d91a316ba9779fcf71 Mon Sep 17 00:00:00 2001 From: Alexey Shmelev Date: Wed, 18 Sep 2024 19:03:43 -0400 Subject: [PATCH 05/11] corrected mismatch between generator and pretrained nonF0 weights --- rvc/lib/algorithm/generators.py | 56 +++++++++++++++++---------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/rvc/lib/algorithm/generators.py b/rvc/lib/algorithm/generators.py index 75fc2ad1..3b4fc75c 100644 --- a/rvc/lib/algorithm/generators.py +++ b/rvc/lib/algorithm/generators.py @@ -35,14 +35,14 @@ def __init__( super(Generator, self).__init__() self.num_kernels = len(resblock_kernel_sizes) self.num_upsamples = len(upsample_rates) - self.conv_pre = torch.nn.Conv1d( - initial_channel, upsample_initial_channel, 7, 1, padding=3 - ) + self.conv_pre = torch.nn.Conv1d(initial_channel, upsample_initial_channel, 7, 1, padding=3) resblock = ResBlock1 if resblock == "1" else ResBlock2 - - self.ups_and_resblocks = torch.nn.ModuleList() + + self.ups = torch.nn.ModuleList() + self.resblocks = torch.nn.ModuleList() + for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)): - self.ups_and_resblocks.append( + self.ups.append( weight_norm( torch.nn.ConvTranspose1d( upsample_initial_channel // (2**i), @@ -57,35 +57,35 @@ def __init__( for j, (k, d) in enumerate( zip(resblock_kernel_sizes, resblock_dilation_sizes) ): - self.ups_and_resblocks.append(resblock(ch, k, d)) + self.resblocks.append(resblock(ch, k, d)) self.conv_post = torch.nn.Conv1d(ch, 1, 7, 1, padding=3, bias=False) - self.ups_and_resblocks.apply(init_weights) + self.ups.apply(init_weights) if gin_channels != 0: self.cond = torch.nn.Conv1d(gin_channels, upsample_initial_channel, 1) - def forward(self, x: torch.Tensor, g: Optional[torch.Tensor] = None): - x = self.conv_pre(x) - if g is not None: - x = x + self.cond(g) + def forward(self, x: torch.Tensor, g: Optional[torch.Tensor] = None): + x = self.conv_pre(x) + if g is not None: + x = x + self.cond(g) - resblock_idx = 0 - for _ in range(self.num_upsamples): - x = torch.nn.functional.leaky_relu(x, LRELU_SLOPE) - x = self.ups_and_resblocks[resblock_idx](x) - resblock_idx += 1 - xs = 0 - for _ in range(self.num_kernels): - xs += self.ups_and_resblocks[resblock_idx](x) - resblock_idx += 1 - x = xs / self.num_kernels + for i in range(self.num_upsamples): + x = torch.nn.functional.leaky_relu(x, LRELU_SLOPE) + x = self.ups[i](x) + xs = None + for j in range(self.num_kernels): + if xs == None: + xs = self.resblocks[i * self.num_kernels + j](x) + else: + xs += self.resblocks[i * self.num_kernels + j](x) + x = xs / self.num_kernels - x = torch.nn.functional.leaky_relu(x) - x = self.conv_post(x) - x = torch.tanh(x) + x = torch.nn.functional.leaky_relu(x) + x = self.conv_post(x) + x = torch.tanh(x) - return x + return x def __prepare_scriptable__(self): """Prepares the module for scripting.""" @@ -100,8 +100,10 @@ def __prepare_scriptable__(self): def remove_weight_norm(self): """Removes weight normalization from the upsampling and residual blocks.""" - for l in self.ups_and_resblocks: + for l in self.ups: remove_weight_norm(l) + for l in self.resblocks: + l.remove_weight_norm() class SineGen(torch.nn.Module): From 825e24402ebb1d2c453b3a8de3fe2382ecd73b0b Mon Sep 17 00:00:00 2001 From: Alexey Shmelev Date: Wed, 18 Sep 2024 19:17:36 -0400 Subject: [PATCH 06/11] fixed exception due to indefined pitchff with pitch_guidance = False branch --- rvc/infer/pipeline.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rvc/infer/pipeline.py b/rvc/infer/pipeline.py index 5f0ffd68..8a988340 100644 --- a/rvc/infer/pipeline.py +++ b/rvc/infer/pipeline.py @@ -650,7 +650,9 @@ def pipeline( audio_max = np.abs(audio_opt).max() / 0.99 if audio_max > 1: audio_opt /= audio_max - del pitch, pitchf, sid + if pitch_guidance: + del pitch, pitchf + del sid if torch.cuda.is_available(): torch.cuda.empty_cache() return audio_opt From 105f6263ebabc743ea28d2a968935236d35fa362 Mon Sep 17 00:00:00 2001 From: Alexey Shmelev Date: Thu, 19 Sep 2024 02:54:29 -0400 Subject: [PATCH 07/11] removed use cpu checkbox --- core.py | 10 ---------- tabs/train/train.py | 7 ------- 2 files changed, 17 deletions(-) diff --git a/core.py b/core.py index afcbaedc..e604f4d2 100644 --- a/core.py +++ b/core.py @@ -517,7 +517,6 @@ def run_train_script( index_algorithm: str = "Auto", cache_data_in_gpu: bool = False, custom_pretrained: bool = False, - use_cpu: bool = False, g_pretrained_path: str = None, d_pretrained_path: str = None, ): @@ -561,7 +560,6 @@ def run_train_script( overtraining_detector, overtraining_threshold, sync_graph, - use_cpu, ], ), ] @@ -1473,13 +1471,6 @@ def parse_arguments(): default="Auto", required=False, ) - train_parser.add_argument( - "--use_cpu", - type=lambda x: bool(strtobool(x)), - choices=[True, False], - help="Force the use of CPU for training.", - default=False, - ) # Parser for 'index' mode index_parser = subparsers.add_parser( @@ -1784,7 +1775,6 @@ def main(): sync_graph=args.sync_graph, index_algorithm=args.index_algorithm, cache_data_in_gpu=args.cache_data_in_gpu, - use_cpu=args.use_cpu, g_pretrained_path=args.g_pretrained_path, d_pretrained_path=args.d_pretrained_path, ) diff --git a/tabs/train/train.py b/tabs/train/train.py index be7e58cd..8a82c153 100644 --- a/tabs/train/train.py +++ b/tabs/train/train.py @@ -621,12 +621,6 @@ def train_tab(): value=True, interactive=True, ) - use_cpu = gr.Checkbox( - label=i18n("Use CPU"), - info=i18n("Force the use of CPU for training."), - value=False, - interactive=True, - ) with gr.Column(): sync_graph = gr.Checkbox( label=i18n("Sync Graph"), @@ -778,7 +772,6 @@ def train_tab(): index_algorithm, cache_dataset_in_gpu, custom_pretrained, - use_cpu, g_pretrained_path, d_pretrained_path, ], From 3d6b84046529288f4f331fbab760093b950fa391 Mon Sep 17 00:00:00 2001 From: blaisewf Date: Thu, 19 Sep 2024 15:56:40 +0200 Subject: [PATCH 08/11] new favicon --- assets/ICON.ico | Bin 108256 -> 15406 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/assets/ICON.ico b/assets/ICON.ico index 340358a598d8a110c798431c8ca99bd580099b02..d7cd78be3beaf7e0b8946ac33ad41ea815c80c3d 100644 GIT binary patch literal 15406 zcmeHOYfMx}6uz}4{k7~a6kqsQ3Mwcd)vAb(NNs{h3~HmTjab?ricM_-P3%9dRjQ&D zY_vYAs3fi;N<#f%4Aei6CdNWh@B!8bsQCD;|BR>e-QC$L)MkIKuxrRmc@FhtAyFg<+=!QO`tAEBK)cbTH)Jb0O${GKlTU_Qv{^-BLZD%1NO zs$M$=&EVj2bteQ9A282&*t3v5n4gUNIUMdePN&PuaP%`BUx(7u#P`=eo-QwwksAcd zF!sGlvP?bS0G29-VHn5Zm#x=6BK3X@Y@T-K+N8Ir$7|l-Ci+q2VeviSYZmz!TOUO| zSs%#ubF+Gw9%Gn0SFiqzva+%RvZhSQpyJ}sDLZ?5pib1)>buM3ragOp8nAw}wY5=e zYpc2rV?D(BiZ=1_?@GskwMM(<4))u~dThoX*r#Bf989M68}=o0=gyO6gMIscHY}Ut zIC0{H6#3YzVXfzUjh3+2=+Q}*F)UtOU|KfEn4e!@S^kC%Uxh7y{rXbN^2^G~!odjEu~{{KEUyNQp6sSRVOAWCJVq%w1hw zhB36YwNq;9Bx&-2Ve;g3+Q0t*UAlCM>gvwYwr$&~rKLs1)ZE-m=g&9L#fukd*RD!R zPEL_3J6M0^<$Xw%mAh5>Po6xXRjWRYxJ-_tpkN7|JzGzSi6g18u~Efv=g!|!<~uH9 zuyX$V1(cDIrRqC=yob)6^Tn2Vd5dhbffgV32Ko7m30SoC27jC00~zFN1dBf|32*;s z6^C#C>GE3T6F$8fkFl6fYOH6(?r4MfLyZNa8Grd>z^KNbJP!56f`OPw=D5(t-*dRU zH>K}9mL1V5CG0VJPU5%Q@c8)nB&XB4lAW2adGbUD2;u+IfD7A`!8Sv#$Xzxrl!8*aJWvfHZP-KYdzcp>KV2d zzzD2b9iS8HV->fE3s^P%Fx=-ev1@k`Kl%ZfSs$ol6}~05iS>7k(SPTesI1cid^-$# zv31NhM}|RbRkkF0b$&OThl@2m*%*k{+9b*TQCU}IVVBAW=V3c`>|}Y*3RWYe%Q&m+pZP~Jwod>@k_!e!S1Ww=*&u(zGh*((F0 zLb(^GsgE8SEW-kYGN~F_|sBUaxv@BABd5)XkhZn>so=EZd&zdGqE? zTD0gRTl~o7il>bmzX@y1a=c=R`DoLo?+CmjVQ(!@%sEGn{3^}%T$f|VYG}-uu@T2V zapGjUeEG6;{7p^G($BWFcIUrwzG~*oo!d85+0k*$^t~sJ$xMm0Kb*W>DExQt-lb{N zBEPSQ6#o17??*IG&ie>?tLxXVOZWfI&aS}OKbIAWG9@Kd&F6&j2eaRiKT1iN81eZB zxf$F=KpxO6M!ojnwRdlon)iu>%|)I?Ma!tCr$-w8?(S|{y0j?bw&!|`8a0Lv9QZ{V ze%$N8y;08JjuJU7o1~<1wqob@LF`psUCs7#*QmR@hi>1#O`V;cbmhtw zYHDgy@6tk#zP>)S9_t=_1?{e0{gcI#-`kekYO4q>$w#3)HpN?M{e_-9;V5_w@9sbn@g$TDkI*fPdi*74931 z9XmcIF+&H~Gcz-baPJuVVZ=EXE?fxAweTbKiyix9(WXY;vA1q$XrS8KTEadRG&wnQ zsJZ#~fDSb^$8Fgs8^J5KfnC$m(n))!fIgWqW2UMX#zJG`Z?@dS5V2W(hi>-}Nh6;-2{y@z31MD8`{(!0g`v?7iet~br*qFxs mjdozu+#BGq#ZtUKgZyeV);Ry#Wc>b&sK}f5Hx>wGfqwuqqa~F9 literal 108256 zcmeHQ2Rv5a8^1~#zQxrvbFaKPhjR|EQ{-KII>+!W1G5OYKFXf5JDF$ zl%t;+cz72LKjVto^2EVW#tQ_LhzP=Gl#ueQN#qIdGB?vu2x+Rj&lW|*DzP^}i>ukX zhJ7c)sdYVTSn_)oNBGSw!)&iH-RUOB8+(8K$XA*ksgZepwLry42Lm^d%kfMrwqfzH1rV%dq&D zVuYIcF3!tkuW%E*=8g=#QTzGM9j9ph)bQo9D#$IXj|rH|m=A;(E?g32yT3=G_14@D zr*-Qr;+l4`&YqX>uHp{yk-%Bq`)l@GT&MdW8M!8N-X+f0<#z0y2YVCfZa?+BZFS5` z&0MNgsZ8sWXVD^AscQs_Fds1|H*0o_nr%9CU3Hz@b7bZlGK;daUzu_{iRx7f*$(S5 za3U^Vb#bw6XIwt4&HYe)>wU1^Hbo1PR{jz`kQLm9e~E9W*o)9+=U}eIa9gEF?@|T8VVO$k01o6-q9^H)jP&q?WefAxR^n zEAwS7cT>zoY*lqATX?^85?(r8;YV)4F{^-wRX{|<^holhdn}jcuH!*0S7(wKwhLt> zVHT;EQ?RR+x?SS5vEIJ@bwt#o)FOEkj(4WW!VQuPEqP@qOQod1u^Vl3D8%D-URU>u z=ia>g)_S_59X0KCZ3H2^4AJbKc`j_SbevmFHhFnS+h@`f>V?p5xS(y>I)}oA!62DM ztBm@2=UgqM!Qtq9@tPaDq!b1W2F5MuhO2u&hnwHsyOn{Ua;3cj!`D@<)D3~U?(z(A z*X1IkqV7_7-6nFLhhch=OI#zlbDm(~P4d%G(#)RfPp{rsligjmA$$W6Y<9BV3>T#0 zP{rE?5$T*Y9O27E7Jpq7pUl>9?r1j^a+RuhM#hCF#33^6ViNosT#}|nMkPi5VzZ7* zh?z09YL=}Rn#I4P0WgHCNt6!{jFk_)7+IEwCoJ82xoEe z6_-~ILUr^9?^QVDDz0nY!$rp>APbp5+tTPj=E;pVt$LE%G=( zO2Hi*p4Kcc|MWxCkz-_t_flQMI5T1|{=4gC+;!;V&ZfR>c`nf);PPZCk)JD>`K6}S z6f*6BgjXLu`n>jTOgYkO*G8@Qf}lpI;nL#?70|5e*Jlzu)qGEJ=R@&{0Vl}cCr?KlX*b$P=DTDOaIH3 zHuT8$W+lYCDfI}so`7Yr{1caDj#=%i(rXwf?57L&#}>%!<0aKt^_RgnCYQoj*}I zV!;0K>nw^7R57KxUv?3hzae`u!%IN>9C?H;)2ruCpB-@~2=}#NZ)rX`i>*$Vy=3YA zG(@fOKly4`wwbe+o>)+uEsA;iq$8c2$UP9zotd#B{}m})cj15Z^z@1;m5T(gb#b3w zh>;=VkYsLUew~{oCHM3o3;Ri?c_hb=A78y!K0c@OlFSgJW-&E_7X0?DEEFp+ln{p`U*0j{g=jo}ieg))lToyJ**`$Mv<3 z7+<#*&RL$(^e$~yQ_O`&X9CV1E^ZdlPH2>VwNg6!v+GOlr%#{muz&a}C*>1^y1Kf9 z)7P3SJe0wz4XxhdYnh+COOK#k5@OA6TWv#fj75?m@}RM}qO#(B1OLl8oM&644G_vT z@(K#IS=@YYB#L*eGct;8vXr&kvjWi!tCo9^q__5%WoMN@_A%+_Y$#6J0g1eL*4B< zB96xFG-?qW@=WEIh6f1f-M84`TA9Rr^`6g}Gb?)Dm44~goPQ?BXMtZ!Ny~vWt7jTt zTKztBcGZw@a&t!~X*dwcu>}b~dvwa`xui!!!=Yl6jfw(>c?uOaZ(45caF#fqSl$Aq zw*O9TDqk}736-E^a#G@xP6lfDM zAbfD`=ux?E$u%n9K1V+`+`s>Q>kdmxL945GkG^}iwzK_tR#x57=J>fi_sq7@xPEQ6 zvTR+yau#MqYgw~dYa4gGo(+SUltB)KQ%{>2a{@`VzQ0c1Zu!Hzp59jKGA}gBXYp)w zBbH_YYPIS=|CB3Q$!VGOqCuV>=?5qtGg(>tyS}j&Px)NuA@jbjJR~($d|^+0*sCg6 zvQvor0ud3BFQ06^)wdO1moVS5g{Jsf*SvX;?;Wy(7&0YGl&BCtOyCLqoV`P`=bcqP z%W=|!AS9`JN{=gxl%$i#8WB;^wt6kI=$)0~`M3ET03oWpTW`5{`o}l6mT+y_ zuw?0D&u0d(*y)w3LOfII`L@Jw+wI`Lj+6tbJ6qiR>gD5a5324uc1pdrObJ+JNO|CP z2*q6{?p)vQZin}`nxjdQR=St45s*K=X|}xK0~(x;+JZ89mdodQDb=<^iQVm0g@ul{ zT9(=-?0Kr5f19gYnvm7YFs|!^((*)`=9^!=RnDzCo|D6qKys|AC;rBblUioCLws4B z+=~Rya`-*n$$Dr9AMJiy(QVtd0lRtX)G37Fi;I-0sj0At$jjVZc~fm>8yg$8&MQHy zd&pcmpJbMpi1K7d#KZ(pEIN9uU8lJy?#-LdZxQ7J-&CW0n398T2-5<)%&}x~ZHbtV zxv>^w94~J{VezI-o0L>kJc-@)&Z{slk~tF)@Z|G<^(*YN_NcDXW6wM~caZ=WMRfl0 z%>C8Xi#3fs^|uid5)ir}l3`&*MKmWoluEQWKX_mHkcygmqq@3`g~cqJj*ep$@3VYa z+7#}Is|MJ8Nt>Bb#qz}s^YGx2IbE3-TrWmMP)1T6BANfX_2q>pFJA1exV4*JM3BXM zssG`_#P1G}%sa*$^z5c*eYETHEq;meSLlPzKW4+104)MdzOmtPK^`a?RHAJlXMIJTPFgDZthUmtEZVM+piqpYNYd{fR)9D4bJKHSN^ z;QD33ZCl!Nd`sWUBZoH{7%Z{<(y-Q0kwYnSJo@bIv&xp!>q^){juD{?tOu6+Sy z%;INtSAU+PD&@STB)XXLRA}h)lliLpFGBB9(Y#-=@6#d*hFKVXIuRod*Uy!UlmpCz zo7C&dXx9etX79?3W_;zRM<-Kb3&bG((0I}?U>zuhFE%euw~Zf=Yh=MQrg#5uVk8+ zX8X~T$>Kq6l+tl7N@Pt|PEJh}DXia*=I8uJym-kvbamd{~TK!|ID6ZZ-kHOS}d zyR$OeH*8I_tXdhB?LLbVS^ebs^Vf{kx3+Hg-|b$x{|%#VdO%JNS$?GK>RFhD!6H3m zaV6XwTL?BkLBN>E#_;55?6A>qj1)YJsi_B$KE`BglXl^|DP|9-yX z#|0xd-%*g4=a7&n2NGOQ818c=N11YuYRWC2Q#BmiLiv4f*BqKzYrX=eoMVu9+7~c) zj#zqWY3ZTJq!p?P)n11HMeL|Toxd{00sdJSRdw|}Pvh+tn@W)QAdRv!Js-qhl%=mM zg^^SZ5h@~gVhlM^05OJ+-wCvIC?mYU2i`c|C?#@=yb2!lW(Y_xuV_g%<$tl4EL8Ou^P><_NQ3c+05l&&s~XOvSS=fh3!xeQ`j`?&o~y+@z&t3{Y4E7}eHZ z?IxXJP0ya=B*~H_k^}@RPKkuPtGunrKv=H^Y7vb>M}VUZ2O@r{D$yUde1No#jZH74 zrk3>yAU1Xx=oG-jP0Up0dU|^Hr2=ZQr()}vmT_?jXv=IqgEZd%bYCWau{(dyyTlhN z{^Y(uIj-qvi^5lV+m#UOwS?hwjz}EH&13)isQyTCbL3V&f>aKI+#}kz4=5el$Yf|@ zvZwfJWgq&Syu6VYFBSmem9w(AOA^<2Bi8E(!x>qezZ`F(lnpScxb+f9N2&K!yMSa^ zJLyRqKmF^qn`U8vS+w4r?H6ZRMo03Nk%5z^Lhrj?dY;HqylGNtluGt;y@rOyyNZ-- zz$Hv$yK$AHSqj-)FAGrhMyI5d_P(znFS_zoo{wnPR#x(z&jP)HtO_}hd3u0Mlk%ID zfH+x4W22HA_KCgaCxZ>tu2l{_%sknG+9ycw-83zaBvP9XOi$~*W0zxMVuZDiHZIOi zuTAf$9R)aVrWGqviEeklB?Jr4rQT^ad0J zXaZPGwx+v*_O+1qQBrk4Rc(+5s7QdBasXy3=2G0N4wUzlB!K{IOK4m!4&Ym~L>?hF zY|7I=>#rHWx0YD1>xA!FfB&HGA1;AlkxKsOfI$fd`D^MrKeMJ6SWCz%4+@#{hasBKnF~E$Ru`&1Z^tM3ZZylh0x%{(}q)q_J?&+D3 zV=cTr$!IQ%D^KS)M}WM)sA(0uDZxg0<7dY4>1&ic+KU_K&cf)lbvb>hKhl*@*xU5q(F(V%_b(i!;h{zKudj&PhVOq z$-z@N>l0{vvuxhHSR)!@0es zNlLGKK5&DY^%{={+EbjqEv1bpKaz~}!*bnwN~rU;*Lz8cz}NpA?yhRhJ{EJ;&xV-& z(v2%8b)~+L>csPWqL@dC=m83I4y8Mhrt2C+cpmoiqjBlT%@i!OrBgr-pDqft^}CZ@ z{q5WQm8<6_O0#C%*&C`~n$Y(4{l?VN=fW9NJQ@=r3r_RCpXCNx>@I-zgqw_NwRw$$BYWo6s*Up?S; zjOG5iQ2HDnhmP;rvufMbvl_U3xEVqTiVqzb^RDZi?wy-NCCzZ;an{3!i&wR^u06?guG)+w zxQHRWdV^l=yedc6mCZ$FJDZD4g+)boC6_-14TX~3-nBP2MO(F<-CJLD@Yb} zC+t3@yg7%~rSI9Ji(cOHzT%Fi+Z<(0!Q6|{(N^VgQg!#|Jbzq!ZM}&EH8u4Wox6UZ zJMjdVhJ)Ly`9YU!foi~$OZm|P#;?iR+UCu1)COJSFJ|`$LxUx}^YZfCThdotM3>p_ zud2_mRnZhwQ#mJiPRpfr^&@WyQJxQPIvlt-w?2+7G~U{E@0dwxLehr2Z%jlPFRb_8 z_4;J~im&b4m{uMjn|EwcL#Oe6!&*-xCPMyjq z@QF5mskyx2?53zQ!Awqz^mgxF6eMc;+UFo)(UmUu4n|R+S(K;w5<$01FY~A%pjkCc z#AmnH%9+uLZ(wNvCFMFWT|8S`mr(oqT86MNzlRdZc1vfwOXvwkE65fGKRjITwfoCw zi<1l@XYL&$`H*hBa|U5Saq*@d>i74CIY002U@XzuD!*yd0vFeC-TBL+^A&We^l}1M z`W#>2y|*C7X&$2>F?gw^#R#Mif<9T#;(GtYEAG!uZ}`f}${I4f7y2FL);RE7Zz*?V z6$!g7+cF0gt|OVX8T*rWmlvzW7^0#bT}~TCI3G1x*IA2OtYG4d%-y~KG~E1jfk8|TQzs4GO)>}zub3sb{Z-2Y`ZjH-C;U9LWZKuZ$e1VV@!J{ z)vlfUHWC+sfxej73T*-tQEgu^8fbDQ_DT%KE5wy1-{tZ)Qi^op#>Ki{EO#P2Yxsx` zyW5)X`xGCYCWY|aHWj&6Y;6OEEMw>HGe#K>^bqabV7PJ7wF>z1$;;MHRk3YfmA2U~ zMA`){CDh#QUJ=nSeU?6kpJch z>NESrzc-qP>B8hZ=8<(WocUG#4axl3NZ3Qgok2}TU_>^lF#1Yl2+Jx?#OShO;F{H* z*0U+ub%-%9b>Xzch1=7Wde|fKSUF{_^D^?w)XhkGNY=!YG@2TbUc8kjwk%+Vo$`lE zx?+l zIc|?JjoZSuySqJ;X_nIRbGV%BTGG7uUQp26xP}|;HnW$oIs+Y<>A`a)?A@hh#_w8K zK8YhpAR}YoChewXHz7Bke8=Svkefv9!Edt?w9kLBkS9QDOpRicEh1g7*6d(?+8)jN z0NEV_W%3sK zMmp;@ zEh)u|_Fd_Dw8dvTza7xE?Jgj-Q?qZBOHbe%=V1!pwK?^kkP(Ll7YFi`7$VNJ2G`@*wTk7l|m z8kO}ZUUDewnI-SNhRtEioQm^FE*wnB-SZaLRc8TMrHvldXq3%1QIGLCb)86tN%6c~ zbvToei>^c$@daIp7BGeRh#;N7U{MR94AT=rnnP)KV7Kr5QAsV zHU0LY;N(l^`BpA=rdM>0P|L)emoLY-n@R3GQq)bN*pvbkea(g%`dJ&T-iCxoa1fbj za^0d|ao>8i%;9rU&Ywb;-gw>hRwt)|QDNf(1fgH({p`HyBI>ttHR`M{tW{(VuUXt0 zT-2@L8f3c{44f5MpZ1PLKFb_^e1XZ&^rX7cJoA>wP>$C11jXiTFCP+bI z8Bxug9?;*8EaOhe1s4KHF+}=yt7fU11`)j7RpZ1O^od|S zm>y(lDeg2Q{OJ2BF6PP=69MhbW)F5COq|?~Pj&`X9-uM5dQb>M#>!UIsU*ug_XeS( z+baegxn|o~Zl%l0v^24|R;ndbgj(ND2sx=PggNI_n(}1fh90i&5bsZiT!gM|H8wsz zk9f<6(oOV5=j@e_mlI`%`1oJ;72l}mNg_f|M?!Yt?uFAb;!MZAc*@didBf>G6I;6# z8O0s+zPwCnnKCWiEDJ_N+v4^|!5ewI)!vfM$aHgJ)jG63DS7cN%nfHLl7hWL$X4mq zxl!jCVl>n|qv#@)gpFv+y2(6RTwFhf?n~8VQ9$P}56U@sbZLYqSWAH@tW%Y}Dzoz- zC-@%&Ux0-ca3#j~9|5vp4$|xTUi{Su6D>6zBQ3Qea1Ovl0QUg63gA8f*8?0nO$ z&z0br3ETzX9D!40q@kkwtDuZyX7p5)GZ<;9qyS%@z~uusoiHA)E<+hXS$O~_%|Jsn zV;mtG^Q7tM$Pu8!?11*ez*PhH4Y<+DV!FqJvV$@_%s@lUMh=(NP0%02oq?8`7tk6B zTqkhT9lNpSAIdZuI3Y$x>ItO<4760NfG${{e+F)>`NpUBvrM~yjL(Cz#`dq30MzZI zfEG*OVEYY^)8CcBdmwWgQ1(dwg&u&smjZVmkNZE@`%KVAT=KVcfR&M&0ARiaxE4J2 z{~UX)&20hofd*(7z-<-d_G6@@CIpzf0Eh3#{ass+#d)YF#Kalc0>bf70D3wKB9MMC za6{36|Mu_Y69lmuOC9LLJ`9ijq|yDbvC@G)n;(S7e$rr%wZXxES{L-S^zPW_Yr{mc|7)0 zR`(;|JpEulfV&WIaDEYA_NS%{KEpK{1L4og3fAp>UtfU7erjTm#Wivu8vw9}^*(;x z&eX;p>Od!4pNDG$z)Whd#*6L@mwtqwFlqL#8!UJ`>~{T;l=vpieqJAWk8{K`6vyAlwIg_wwaS zG%&y))z#KQ`FXif_JvHSh#)_@-^3V=yK)7sudg3=dIkCU=sx{Dli40%&t8~R`15it zMbpym4jZSzbm7gLH|XovuLrdqgT?Fr9_nA*<;y4^H`k<+VPBbt`!?ZO6qAN_9)NvD zM*1IP-yg%V`2YE{XQ+(i>PaKVKFlXozaO^8Qf?(rIP9_d1CFdi<=3s9Y&tM0=!I?W z3lZlhEcXA`fl_p}_=-s()5&D_9qsKVJoZ=}c$Slca&s=h>%bVbD-_qup|4)Onl$)B zJ-8GTJ(>Fj6GnHJu(P6BSx+Yw_EP;n+M@epBIT zYh#VK0mHNbf#(9I6!uVmbF#BhHfBb=4*W(3;5=GYRn?TkAI=kQ-l&Gh|2O!X?>C)t z*kkLj;J`pU{zKpo+kJQL+@7-d7lZL2IEOjhcF=SSw?af{(v9bK8SfoR58$|#xR?mu2H@BLxL;!^xlN^S+cme~@yEgc@Ikj}gTJk{6&`;a z{6m5Qrw#t6y}j}HHnje~@{9+LZr$ z85wx|aqxfm@WHgfAM!d?`|+dY3zi97J2_ST(AJGsPE$2r^39}aga7T+6g>Vo_CNB% zg=vF7Jda|kmgCX#<>}!uZSeQ?IfKU^r~I258BQDg4}kNDMq5s%YCO5MYesyQ%VcdI zswyeq@y99ua9vb+dHIyL0oB!2C?^Ly9)BGCphGTN z<=@oU2w(OGDEm++;F%c}6%|uf2jF>v@cgR5^dJA88ijqM!EEPbe}CHB3y=LEWgn~i zVnPCFZEfw8(*fA-6A@UB$Nx9@Lp?Zi%6rP;f8@|XJodxD9^wz%0R;sGQ$`0KJ$#6A zfb(jxHV*&%{rL~@4Cbk@2fU`H28H|C@tFS}^Zqh|djUO9PVPAYsK5IR_TaG}rrqz4 zKl~q_-5(tpIVtM_xYrNf8vykfmoL13__kmX7#EJZcyUtc04(?LywWLjPsBgfVHQUE z-upM;S?v>j44|aAxcB}Dd>J2p86U2!HmWM4_wU~u)jVNsZ4H_U)?J+gcRviZ)(DQ# zJ%0RX#BI;|`nuk<`FXhiX;~SrEGKI?XbWK9ASNmbEdhHe8yXsVv4S=io)KJESJ!*) zW@<`G?>!&ZmKLb8!g_H2*7x(67c$VHX?O1oYDXGDIwwyYN7qVA_TD7~&m7Or&O&QJ zJ07MF^MMZL3w7mIax&`VV2_Fk3*vDfWjk-Qx(Kmc&a(`a5EDfuR*0g)V0;bEtq#@Z z(%P!oi~ZHOxZb-z2CGltUiAL+w2Q!9Kk$yJm7>BZJg0&m$g(c3xf@r0X6NuX~rW$u3cQ4S^pmLs7Nt|5VKyTZbPAK1e+ zC9={|zwivlXoq{&I2_YoC$r|4G)6)jKJ570eOTuE+i{57fg#RZf%l1KJssX1LvU;e zkKy=Z_$M-!T_g3Q+(SKp_XNYbV5l@j!5zSGY@om14D>&&Lxvh(8Y+(o`fXC(0Xke6 zTAJ+#A^!?(ptkcn9#1Wn(&hhxdWQaRE5S zKT!Fv0r&Y#p);Hz_V5mHJpMQ`&`{rq!h1Y%b)X8&`M|k=DI&8_Bq z{Nan|&wuE_(#?j@a%3SdAWZ=!*J*vMVJd9 zj>9P8O(hvjXI>lu_KJA?r)T?z0btKWOHBvxpOSq*yj~Bf*F6k0)N}~=0XgO4Wi_O% zreYnD53z?ofW0Rk|7nN6$9MF5@s|Slf5Yp*w9x^GwKNugCVHwFz+a8Wf7;+*O-DmD z1B*X=0_+dt@t-#MAMTHRAO7qB|4zIPOnDvXWTc^H!^NMKjt<&@NId>i9{*@^a&lbw zd;I`=Uf@3CbzsWsKo`JPXh876%RU=`pAW#93!Fmb;XETX^gm9H zfa@>y{y@qA!dXrsou7(hccbCSW+1EwdI#nNNCE!$@j5VN+5vEl2kAgq5B3g(UkY3c zUI!+<4nPc-42JVi??JpRz~ScxCvRH-Vy7__#zTE$Wuztm@m+u$O?`up^Y7*64BGny zLt*^8Z(yv55a1t-*MUh_27hhIlD_BsG%3fJfG!g>0M z`eOxaOF+EkXdra{C#^37# zlo6DbCvZ|sv_G%Y9Z1fTQ53Dg=0^fNDEbZSMG#l$Z~ec2KJ;h&-%kYnnE(3;fVW@t-UPn< z|1*L}{gfPnfCOv$e+Lv~^!I}dF#Y`?gZ^Fw`8hj~;ZNBE|4-Qi0zUs7p9R<7k6RdW ze$CI}NpSrcKYu6sogedac!H7oe~ym#;U$2V0A2zUM*^6iNg1(y%lJ9|Z|!N#&*4dM z{kZlV*WTm$aqA1*`UKaHTOa*WUuE)e5O8Ob|FS}M3kWG8U1ft%=jIRMLa z-xKi630&0_;2kRq*cXK+@wdM|-rr0i-mx-(JPj!W@SPgCsf0V0ckHtYlm(iBl%UVn zO@;D~<#n(pum``|M1Z>qR9sBtm;CsaarNG5ayj-AnwXH#dnTz6IJbP9v4Q<}*oPkT zd|1F8-$CaBew?q1d){*Hix+5szaM)1*ijUoz2Df__)A!L7w54fhkv=-Wen*Zm$3_Y z_sGQy5x?+*D}&+s;XP$b*jdLMBOpgOhBD^qLQIFt+i=6dG6B!)9S?fPGEI2CIJ}o+ zxO|Ny+|k2_CN%%>{QTV97vsV^loPzC7v2dwmUxY2S}u+bAKqaz7P>nYX%vIIcgLGw-1Li!iY5&I`FVMNoqu@8{a9c(7HQnO zm+{y6fB4|RgyH{uc-UX(KkDMe3By0Ei*d{6c>8UvEGG>A@P0wqFBxy1vFXD*?1cH3 zPZ<884T1WIrEk2S7bDJ3DBdCetHoEKufTZac=HbLJ6<6yG~xJ%GH`OR8(02eAIoT; z!Gz=;@(=Hi_47SDE;4|;V_cjkEbowitPJ3t%r!MNgUX?*stSeo?e#A!@ZDg2=z@ZL zbf@-?3C}xL2Ji{*p5AX_jC!8*=)HH;)!7N%wNt0pPQY<&_oGM9va+%tGJs>@%M(3(&<)kvx&?)Im-bJ4vj4wQR2Z$SsOZ(_{yI<%bU;~Q{gmPr7b|#Q zHN3w#Bq$KvIWPSKf26Dx?l7_MsoF!F#XdRhyTZA4iJ~l#cZr-dfHtu-+Szr zd)7CpZ2&q^`I8Lb{j9iYjDsKY4(EXXMqWeaAucL{-o11ChYZ|YU4{%lMl@sB&;O0Q zewzmq7}M9?p@qg?ib0`Hj1iARr32SLz%?L4g_+9V;M$+*d{1oeo{?#czYw@1Y}&4W z1b1ci?iB*~Puumca1AWrAMS6OR_mVuzd!Ad0)Dv9X)4?L{=Dfy-24B)*)DLe*R3_)EkbA+vcewX!TJC?smWMtWOeH;lybQVb3(Gt93Btp@ zU{k>!KpqC%^M&PguxFo5g!`f3797wyTJBgVLwg?oGUe={qNt;EdVD5V4y?@ zu*VW$&)318tHaWF`oR1e7z{y3GI}2f_8bQy%y&QbT!<0T@I5O2?@$uJeE;}Ez8T;B zKjdHY-A{tWsqg#?mcC;rr20;aaK++`#T`}({J`x5?#DS193wj?f`Nt-_92*oGXoBm z3s~Qb=UI_3ZI}nl3+Bm8PfZ0l82JyxT>>0DfAKSLf4dB0^MiRu0XOXXeBd5l5Z(f~ zcHpq-{Oe~2q-mi2#r=^1-)Dkw@NBeyP48g&pNBN|ry2Wy-#+p46PR`^PZ0y*0P&dw zv}1X&z{(vy0o|~yLLI|?pXg6mHhar7&>KiUEaMZMW?cFr`|SHdK>NS6k+^gX*AHpN z)=k5WGtt6f>(z-y*Kp~hB(OZP9S(hha{hI6mt9|3vqAIG6sS z7sI8we>kX1(yJx@IuB58{pt9V|KXnMKau4~<8m$LKr1UN#*BWrpItP-J9 zJm-OL8RxjuFC`&9=DLIB1FrQRsXRv-UQ%M^xX~}QN_^buU(U@n;F%s+8IJY|*9Z&n za*sRxu)Q`O`??AX3;)_V7J);YzcO0+2ZMF1Bh}^6hKK88VY_9twBp8tdvw-HOa3+b zA>VMHE-oFT^?QRg;iHx1XyYwrV?l4Hrv5Pxur6n1`s04gsQ8C{#M9nhKlUU*8v^@X zSy@ke&*^#n`c*G~@EzQj0M8kLVW2B6E<)kj{l8J?{tlz19PB8p7vT9(oZx>LUVoR~ zPwPgn#K!jS4TE(QTxb6;`(3}|2et#AK79h#daC}UUzcK{C)D{41MvYdxOVj_ifs?$ zmJ!(J94OvIdH49yBNVoUv3e~oDuU+c|9>AdJacM{Y5k)y2wYm={u(yspZCQG3oJ)( zCMEUi(z9ndzpOLIO=ARp?0nN$%jB2z;8@GHEgGnmh50YvMnEraTsVFO$KY_^PZmF< z9gcxc7J6~%?46?o^iS6D^Myb!;rs=hC&lG$qWhz;bHjl4$u@rk=emBH69)cCJAVh~ z@R(?6;Jo!jp8Lh>^La)ZngQ2N0UqG|^+cWnj({{mc@OsM!vX5jM4C^9^Q9J$w!!Ge zeGkIJG7aZd|D|n(c|+UyQ(48Obud3zX9#WULg29Te6YSA)AkZb53ASkiEYnd>s4lG z--r73t{-wmklyt~QV7z!&S*Y@)WAq22$BJxME`$cpdY^q3`*gr9|rn>Tr%LZpS)_I zAIK3aSFD_|a);}*Sb>uU?#DVU^53u10_VIF0ld|Ly9`_%a9@DyuL~pjAI61g!~zG? z1RM>AuOFR57quOK$`wi-u-Fll>^8Z;ys=+ z$L5LqeA!ReaKHU6Kh#O6n|~`m+K7 z3ZM;OtnYXEV!uN(vGi1Uw)iTPvRvA78F^PzAaaJam`c<~&C^QqYQWATJ%*1@?c zc=tlwl`FmXFg$$tpqHlfv^4bkwRki&C8hW5A|5c-Iu>~jm4@PaITX%sL7BigKX?wr zP~pe&8(cF8=cnLY$&JLsUK%REIXt7Zcf+m&u{_AFl|jXXzOPk;YmVT&0+fT2yxd4- zfNh_TG~O@a;k_kXix>U!9!v8ePuTu|ZJ?3H8>{fxe#uy+H_~)^`$T~Mf71R2`S$k5 z!1JH=Kd}8EK*K-lfAyo`pY=cc(SYrLL){#9`y7NFNcXVi8OY<+`R)96<@Vq`Y;D%W1IUv9Aygz97;W`9(&L{l; z#K3nw{35G0t5*%W-b#Mm+F$6zm7BNc_qrG6iR*^b|AO~h;l{)I;Wym;aQ(r7frG{m z2@3jc{B<&Gde?1W^Ta-(EM=voej6Xs1J_6I)8Es(e*m6EXm5J}UA0ndu(-cq0Mif# zZ96D$cqafn`w^GVK0j8U2m9Xpo`-YEVBfK2VX*JNd*54*04<{~KcI-iF>7o&8c?76 T@s8#9NBPBoG0@TeVBG%!JyQt+ From 57b61a8fe3972dddda1ec377f3c468c8a14689c8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 19 Sep 2024 13:57:01 +0000 Subject: [PATCH 09/11] chore(format): run black on main --- rvc/lib/algorithm/generators.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/rvc/lib/algorithm/generators.py b/rvc/lib/algorithm/generators.py index 3b4fc75c..e08023de 100644 --- a/rvc/lib/algorithm/generators.py +++ b/rvc/lib/algorithm/generators.py @@ -35,12 +35,14 @@ def __init__( super(Generator, self).__init__() self.num_kernels = len(resblock_kernel_sizes) self.num_upsamples = len(upsample_rates) - self.conv_pre = torch.nn.Conv1d(initial_channel, upsample_initial_channel, 7, 1, padding=3) + self.conv_pre = torch.nn.Conv1d( + initial_channel, upsample_initial_channel, 7, 1, padding=3 + ) resblock = ResBlock1 if resblock == "1" else ResBlock2 - + self.ups = torch.nn.ModuleList() self.resblocks = torch.nn.ModuleList() - + for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)): self.ups.append( weight_norm( From 168d0bab0e8b70bccf14b9932bd4b08423634451 Mon Sep 17 00:00:00 2001 From: blaisewf Date: Thu, 19 Sep 2024 21:19:08 +0200 Subject: [PATCH 10/11] change downloading message --- tabs/train/train.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tabs/train/train.py b/tabs/train/train.py index be7e58cd..bbd266ff 100644 --- a/tabs/train/train.py +++ b/tabs/train/train.py @@ -884,7 +884,7 @@ def download_prerequisites(version, pitch_guidance): if version == "v1": if pitch_guidance: gr.Info( - "Downloading v1 prerequisites with pitch guidance... Please wait till it finishes to start preprocessing." + "Checking for v1 prerequisites with pitch guidance... Missing files will be downloaded. If you already have them, this step will be skipped." ) run_prerequisites_script( pretraineds_v1_f0=True, @@ -896,7 +896,7 @@ def download_prerequisites(version, pitch_guidance): ) else: gr.Info( - "Downloading v1 prerequisites without pitch guidance... Please wait till it finishes to start preprocessing." + "Checking for v1 prerequisites without pitch guidance... Missing files will be downloaded. If you already have them, this step will be skipped." ) run_prerequisites_script( pretraineds_v1_f0=False, @@ -909,7 +909,7 @@ def download_prerequisites(version, pitch_guidance): elif version == "v2": if pitch_guidance: gr.Info( - "Downloading v2 prerequisites with pitch guidance... Please wait till it finishes to start preprocessing." + "Checking for v2 prerequisites with pitch guidance... Missing files will be downloaded. If you already have them, this step will be skipped." ) run_prerequisites_script( pretraineds_v1_f0=False, @@ -921,7 +921,7 @@ def download_prerequisites(version, pitch_guidance): ) else: gr.Info( - "Downloading v2 prerequisites without pitch guidance... Please wait till it finishes to start preprocessing." + "Checking for v2 prerequisites without pitch guidance... Missing files will be downloaded. If you already have them, this step will be skipped." ) run_prerequisites_script( pretraineds_v1_f0=False, @@ -932,7 +932,7 @@ def download_prerequisites(version, pitch_guidance): exe=False, ) gr.Info( - "Prerequisites downloaded successfully, you may now start preprocessing." + "Prerequisites check complete. Missing files were downloaded, and you may now start preprocessing." ) def toggle_visible_embedder_custom(embedder_model): From d58fbf2e3dbd3a897a6746a968676ce84e6eb74b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 19 Sep 2024 19:20:38 +0000 Subject: [PATCH 11/11] chore(format): run black on main --- rvc/infer/pipeline.py | 48 ++++-- rvc/train/data_utils.py | 2 + rvc/train/train.py | 318 +++++++++++++++++++++++++++++++--------- 3 files changed, 287 insertions(+), 81 deletions(-) diff --git a/rvc/infer/pipeline.py b/rvc/infer/pipeline.py index 8a988340..7c8f58c1 100644 --- a/rvc/infer/pipeline.py +++ b/rvc/infer/pipeline.py @@ -417,41 +417,62 @@ def voice_conversion( with torch.no_grad(): pitch_guidance = pitch != None and pitchf != None # prepare source audio - feats = torch.from_numpy(audio0).half() if self.is_half else torch.from_numpy(audio0).float() + feats = ( + torch.from_numpy(audio0).half() + if self.is_half + else torch.from_numpy(audio0).float() + ) feats = feats.mean(-1) if feats.dim() == 2 else feats assert feats.dim() == 1, feats.dim() feats = feats.view(1, -1).to(self.device) # extract features feats = model(feats)["last_hidden_state"] - feats = model.final_proj(feats[0]).unsqueeze(0) if version == "v1" else feats + feats = ( + model.final_proj(feats[0]).unsqueeze(0) if version == "v1" else feats + ) # make a copy for pitch guidance and protection feats0 = feats.clone() if pitch_guidance else None - if index: # set by parent function, only true if index is available, loaded, and index rate > 0 - feats = self._retrieve_speaker_embeddings(feats, index, big_npy, index_rate) - # feature upsampling - feats = F.interpolate(feats.permute(0, 2, 1), scale_factor=2).permute(0, 2, 1) + if ( + index + ): # set by parent function, only true if index is available, loaded, and index rate > 0 + feats = self._retrieve_speaker_embeddings( + feats, index, big_npy, index_rate + ) + # feature upsampling + feats = F.interpolate(feats.permute(0, 2, 1), scale_factor=2).permute( + 0, 2, 1 + ) # adjust the length if the audio is short p_len = min(audio0.shape[0] // self.window, feats.shape[1]) if pitch_guidance: - feats0 = F.interpolate(feats0.permute(0, 2, 1), scale_factor=2).permute(0, 2, 1) + feats0 = F.interpolate(feats0.permute(0, 2, 1), scale_factor=2).permute( + 0, 2, 1 + ) pitch, pitchf = pitch[:, :p_len], pitchf[:, :p_len] # Pitch protection blending if protect < 0.5: pitchff = pitchf.clone() pitchff[pitchf > 0] = 1 pitchff[pitchf < 1] = protect - feats = feats * pitchff.unsqueeze(-1) + feats0 * (1 - pitchff.unsqueeze(-1)) + feats = feats * pitchff.unsqueeze(-1) + feats0 * ( + 1 - pitchff.unsqueeze(-1) + ) feats = feats.to(feats0.dtype) else: pitch, pitchf = None, None p_len = torch.tensor([p_len], device=self.device).long() - audio1 = ((net_g.infer(feats, p_len, pitch, pitchf, sid)[0][0, 0]).data.cpu().float().numpy()) + audio1 = ( + (net_g.infer(feats, p_len, pitch, pitchf, sid)[0][0, 0]) + .data.cpu() + .float() + .numpy() + ) # clean up del feats, feats0, p_len if torch.cuda.is_available(): torch.cuda.empty_cache() return audio1 - + def _retrieve_speaker_embeddings(self, feats, index, big_npy, index_rate): npy = feats[0].cpu().numpy() npy = npy.astype("float32") if self.is_half else npy @@ -460,9 +481,12 @@ def _retrieve_speaker_embeddings(self, feats, index, big_npy, index_rate): weight /= weight.sum(axis=1, keepdims=True) npy = np.sum(big_npy[ix] * np.expand_dims(weight, axis=2), axis=1) npy = npy.astype("float16") if self.is_half else npy - feats = torch.from_numpy(npy).unsqueeze(0).to(self.device) * index_rate + (1 - index_rate) * feats + feats = ( + torch.from_numpy(npy).unsqueeze(0).to(self.device) * index_rate + + (1 - index_rate) * feats + ) return feats - + def pipeline( self, model, diff --git a/rvc/train/data_utils.py b/rvc/train/data_utils.py index a4e7ef0f..b6c3d644 100644 --- a/rvc/train/data_utils.py +++ b/rvc/train/data_utils.py @@ -6,6 +6,7 @@ from mel_processing import spectrogram_torch from utils import load_filepaths_and_text, load_wav_to_torch + class TextAudioLoaderMultiNSFsid(torch.utils.data.Dataset): """ Dataset that loads text and audio pairs. @@ -163,6 +164,7 @@ def __len__(self): """ return len(self.audiopaths_and_text) + class TextAudioCollateMultiNSFsid: """ Collates text and audio data for training. diff --git a/rvc/train/train.py b/rvc/train/train.py index 4054cad2..4f4ce221 100644 --- a/rvc/train/train.py +++ b/rvc/train/train.py @@ -148,11 +148,15 @@ def main(): os.environ["MASTER_ADDR"] = "localhost" os.environ["MASTER_PORT"] = str(randint(20000, 55555)) # Check sample rate - wavs = glob.glob(os.path.join(os.path.join(experiment_dir, "sliced_audios"), "*.wav")) + wavs = glob.glob( + os.path.join(os.path.join(experiment_dir, "sliced_audios"), "*.wav") + ) if wavs: _, sr = load_wav_to_torch(wavs[0]) if sr != sample_rate: - print(f"Error: Pretrained model sample rate ({sample_rate} Hz) does not match dataset audio sample rate ({sr} Hz).") + print( + f"Error: Pretrained model sample rate ({sample_rate} Hz) does not match dataset audio sample rate ({sr} Hz)." + ) os._exit(1) else: print("No wav file found.") @@ -240,10 +244,12 @@ def continue_overtrain_detector(training_file_path): ) = load_from_json(training_file_path) if sync_graph: - print("Sync graph is now activated! With sync graph enabled, the model undergoes a single epoch of training. Once the graphs are synchronized, training proceeds for the previously specified number of epochs.") + print( + "Sync graph is now activated! With sync graph enabled, the model undergoes a single epoch of training. Once the graphs are synchronized, training proceeds for the previously specified number of epochs." + ) custom_total_epoch = 1 custom_save_every_weights = True - + start() # Synchronize graphs by modifying config files @@ -288,14 +294,18 @@ def edit_config(config_file): edit_config(rvc_config_file) # Clean up unnecessary files - for root, dirs, files in os.walk(os.path.join(now_dir, "logs", model_name), topdown=False): + for root, dirs, files in os.walk( + os.path.join(now_dir, "logs", model_name), topdown=False + ): for name in files: file_path = os.path.join(root, name) file_name, file_extension = os.path.splitext(name) - if (file_extension == ".0" or - (file_name.startswith("D_") and file_extension == ".pth") or - (file_name.startswith("G_") and file_extension == ".pth") or - (file_name.startswith("added") and file_extension == ".index")): + if ( + file_extension == ".0" + or (file_name.startswith("D_") and file_extension == ".pth") + or (file_name.startswith("G_") and file_extension == ".pth") + or (file_name.startswith("added") and file_extension == ".index") + ): os.remove(file_path) for name in dirs: if name == "eval": @@ -317,6 +327,7 @@ def edit_config(config_file): continue_overtrain_detector(training_file_path) start() + def run( rank, n_gpus, @@ -409,8 +420,18 @@ def run( else: net_d = MultiPeriodDiscriminatorV2(config.model.use_spectral_norm).to(device) - optim_g = torch.optim.AdamW(net_g.parameters(), config.train.learning_rate, betas=config.train.betas, eps=config.train.eps) - optim_d = torch.optim.AdamW(net_d.parameters(), config.train.learning_rate, betas=config.train.betas, eps=config.train.eps) + optim_g = torch.optim.AdamW( + net_g.parameters(), + config.train.learning_rate, + betas=config.train.betas, + eps=config.train.eps, + ) + optim_d = torch.optim.AdamW( + net_d.parameters(), + config.train.learning_rate, + betas=config.train.betas, + eps=config.train.eps, + ) # Wrap models with DDP for multi-gpu processing if n_gpus > 1 and device.type == "cuda": @@ -420,8 +441,12 @@ def run( # Load checkpoint if available try: print("Starting training...") - _, _, _, epoch_str = load_checkpoint(latest_checkpoint_path(experiment_dir, "D_*.pth"), net_d, optim_d) - _, _, _, epoch_str = load_checkpoint(latest_checkpoint_path(experiment_dir, "G_*.pth"), net_g, optim_g) + _, _, _, epoch_str = load_checkpoint( + latest_checkpoint_path(experiment_dir, "D_*.pth"), net_d, optim_d + ) + _, _, _, epoch_str = load_checkpoint( + latest_checkpoint_path(experiment_dir, "G_*.pth"), net_g, optim_g + ) epoch_str += 1 global_step = (epoch_str - 1) * len(train_loader) @@ -433,21 +458,33 @@ def run( verify_checkpoint_shapes(pretrainG, net_g) print(f"Loaded pretrained (G) '{pretrainG}'") if hasattr(net_g, "module"): - net_g.module.load_state_dict(torch.load(pretrainG, map_location="cpu")["model"]) + net_g.module.load_state_dict( + torch.load(pretrainG, map_location="cpu")["model"] + ) else: - net_g.load_state_dict(torch.load(pretrainG, map_location="cpu")["model"]) + net_g.load_state_dict( + torch.load(pretrainG, map_location="cpu")["model"] + ) if pretrainD != "": if rank == 0: print(f"Loaded pretrained (D) '{pretrainD}'") if hasattr(net_d, "module"): - net_d.module.load_state_dict(torch.load(pretrainD, map_location="cpu")["model"]) + net_d.module.load_state_dict( + torch.load(pretrainD, map_location="cpu")["model"] + ) else: - net_d.load_state_dict(torch.load(pretrainD, map_location="cpu")["model"]) + net_d.load_state_dict( + torch.load(pretrainD, map_location="cpu")["model"] + ) # Initialize schedulers and scaler - scheduler_g = torch.optim.lr_scheduler.ExponentialLR(optim_g, gamma=config.train.lr_decay, last_epoch=epoch_str - 2) - scheduler_d = torch.optim.lr_scheduler.ExponentialLR(optim_d, gamma=config.train.lr_decay, last_epoch=epoch_str - 2) + scheduler_g = torch.optim.lr_scheduler.ExponentialLR( + optim_g, gamma=config.train.lr_decay, last_epoch=epoch_str - 2 + ) + scheduler_d = torch.optim.lr_scheduler.ExponentialLR( + optim_d, gamma=config.train.lr_decay, last_epoch=epoch_str - 2 + ) optim_g.step() optim_d.step() @@ -489,6 +526,7 @@ def run( scheduler_g.step() scheduler_d.step() + def train_and_evaluate( rank, epoch, @@ -542,21 +580,39 @@ def train_and_evaluate( data_iterator = cache if cache == []: for batch_idx, info in enumerate(train_loader): - phone, phone_lengths, pitch, pitchf, spec, spec_lengths, wave, wave_lengths, sid = info + ( + phone, + phone_lengths, + pitch, + pitchf, + spec, + spec_lengths, + wave, + wave_lengths, + sid, + ) = info cache.append( ( batch_idx, ( phone.cuda(rank, non_blocking=True), - phone_lengths.cuda(rank, non_blocking=True), - pitch.cuda(rank, non_blocking=True) if pitch_guidance else None, - pitchf.cuda(rank, non_blocking=True) if pitch_guidance else None, + phone_lengths.cuda(rank, non_blocking=True), + ( + pitch.cuda(rank, non_blocking=True) + if pitch_guidance + else None + ), + ( + pitchf.cuda(rank, non_blocking=True) + if pitch_guidance + else None + ), spec.cuda(rank, non_blocking=True), spec_lengths.cuda(rank, non_blocking=True), wave.cuda(rank, non_blocking=True), wave_lengths.cuda(rank, non_blocking=True), - sid.cuda(rank, non_blocking=True) - ) + sid.cuda(rank, non_blocking=True), + ), ) ) else: @@ -567,12 +623,24 @@ def train_and_evaluate( epoch_recorder = EpochRecorder() with tqdm(total=len(train_loader), leave=False) as pbar: for batch_idx, info in data_iterator: - phone, phone_lengths, pitch, pitchf, spec, spec_lengths, wave, wave_lengths, sid = info + ( + phone, + phone_lengths, + pitch, + pitchf, + spec, + spec_lengths, + wave, + wave_lengths, + sid, + ) = info if device.type == "cuda" and not cache_data_in_gpu: phone = phone.cuda(rank, non_blocking=True) phone_lengths = phone_lengths.cuda(rank, non_blocking=True) pitch = pitch.cuda(rank, non_blocking=True) if pitch_guidance else None - pitchf = pitchf.cuda(rank, non_blocking=True) if pitch_guidance else None + pitchf = ( + pitchf.cuda(rank, non_blocking=True) if pitch_guidance else None + ) sid = sid.cuda(rank, non_blocking=True) spec = spec.cuda(rank, non_blocking=True) spec_lengths = spec_lengths.cuda(rank, non_blocking=True) @@ -592,16 +660,27 @@ def train_and_evaluate( # Forward pass use_amp = config.train.fp16_run and device.type == "cuda" with autocast(enabled=use_amp): - (y_hat, ids_slice, x_mask, z_mask, (z, z_p, m_p, logs_p, m_q, logs_q)) = net_g(phone, phone_lengths, pitch, pitchf, spec, spec_lengths, sid) + ( + y_hat, + ids_slice, + x_mask, + z_mask, + (z, z_p, m_p, logs_p, m_q, logs_q), + ) = net_g(phone, phone_lengths, pitch, pitchf, spec, spec_lengths, sid) mel = spec_to_mel_torch( spec, config.data.filter_length, config.data.n_mel_channels, config.data.sample_rate, config.data.mel_fmin, - config.data.mel_fmax + config.data.mel_fmax, + ) + y_mel = commons.slice_segments( + mel, + ids_slice, + config.train.segment_size // config.data.hop_length, + dim=3, ) - y_mel = commons.slice_segments(mel, ids_slice, config.train.segment_size // config.data.hop_length, dim=3) with autocast(enabled=False): y_hat_mel = mel_spectrogram_torch( y_hat.float().squeeze(1), @@ -611,14 +690,21 @@ def train_and_evaluate( config.data.hop_length, config.data.win_length, config.data.mel_fmin, - config.data.mel_fmax + config.data.mel_fmax, ) if use_amp: y_hat_mel = y_hat_mel.half() - wave = commons.slice_segments(wave, ids_slice * config.data.hop_length, config.train.segment_size, dim=3) + wave = commons.slice_segments( + wave, + ids_slice * config.data.hop_length, + config.train.segment_size, + dim=3, + ) y_d_hat_r, y_d_hat_g, _, _ = net_d(wave, y_hat.detach()) with autocast(enabled=False): - loss_disc, losses_disc_r, losses_disc_g = discriminator_loss(y_d_hat_r, y_d_hat_g) + loss_disc, losses_disc_r, losses_disc_g = discriminator_loss( + y_d_hat_r, y_d_hat_g + ) # Discriminator backward and update optim_d.zero_grad() scaler.scale(loss_disc).backward() @@ -631,7 +717,9 @@ def train_and_evaluate( y_d_hat_r, y_d_hat_g, fmap_r, fmap_g = net_d(wave, y_hat) with autocast(enabled=False): loss_mel = F.l1_loss(y_mel, y_hat_mel) * config.train.c_mel - loss_kl = (kl_loss(z_p, logs_q, m_p, logs_p, z_mask) * config.train.c_kl) + loss_kl = ( + kl_loss(z_p, logs_q, m_p, logs_p, z_mask) * config.train.c_kl + ) loss_fm = feature_loss(fmap_r, fmap_g) loss_gen, losses_gen = generator_loss(y_d_hat_g) loss_gen_all = loss_gen + loss_fm + loss_mel + loss_kl @@ -642,7 +730,9 @@ def train_and_evaluate( lowest_value["epoch"] = epoch # print(f'Lowest generator loss updated: {lowest_value["value"]} at epoch {epoch}, step {global_step}') if epoch > lowest_value["epoch"]: - print("Alert: The lower generating loss has been exceeded by a lower loss in a subsequent epoch.") + print( + "Alert: The lower generating loss has been exceeded by a lower loss in a subsequent epoch." + ) optim_g.zero_grad() scaler.scale(loss_gen_all).backward() @@ -666,13 +756,29 @@ def train_and_evaluate( "grad_norm_d": grad_norm_d, "grad_norm_g": grad_norm_g, } - scalar_dict.update({"loss/g/fm": loss_fm, "loss/g/mel": loss_mel, "loss/g/kl": loss_kl}) - scalar_dict.update({f"loss/g/{i}": v for i, v in enumerate(losses_gen)}) - scalar_dict.update({f"loss/d_r/{i}": v for i, v in enumerate(losses_disc_r)}) - scalar_dict.update({f"loss/d_g/{i}": v for i, v in enumerate(losses_disc_g)}) + scalar_dict.update( + { + "loss/g/fm": loss_fm, + "loss/g/mel": loss_mel, + "loss/g/kl": loss_kl, + } + ) + scalar_dict.update( + {f"loss/g/{i}": v for i, v in enumerate(losses_gen)} + ) + scalar_dict.update( + {f"loss/d_r/{i}": v for i, v in enumerate(losses_disc_r)} + ) + scalar_dict.update( + {f"loss/d_g/{i}": v for i, v in enumerate(losses_disc_g)} + ) image_dict = { - "slice/mel_org": plot_spectrogram_to_numpy(y_mel[0].data.cpu().numpy()), - "slice/mel_gen": plot_spectrogram_to_numpy(y_hat_mel[0].data.cpu().numpy()), + "slice/mel_org": plot_spectrogram_to_numpy( + y_mel[0].data.cpu().numpy() + ), + "slice/mel_gen": plot_spectrogram_to_numpy( + y_hat_mel[0].data.cpu().numpy() + ), "all/mel": plot_spectrogram_to_numpy(mel[0].data.cpu().numpy()), } summarize( @@ -689,15 +795,31 @@ def train_and_evaluate( model_add = [] model_del = [] done = False - + if rank == 0: # Save weights every N epochs if epoch % save_every_epoch == 0: - checkpoint_suffix = (f"{2333333 if save_only_latest else global_step}.pth") - save_checkpoint(net_g, optim_g, config.train.learning_rate, epoch, os.path.join(experiment_dir, "G_" + checkpoint_suffix)) - save_checkpoint(net_d, optim_d, config.train.learning_rate, epoch, os.path.join(experiment_dir, "D_" + checkpoint_suffix)) + checkpoint_suffix = f"{2333333 if save_only_latest else global_step}.pth" + save_checkpoint( + net_g, + optim_g, + config.train.learning_rate, + epoch, + os.path.join(experiment_dir, "G_" + checkpoint_suffix), + ) + save_checkpoint( + net_d, + optim_d, + config.train.learning_rate, + epoch, + os.path.join(experiment_dir, "D_" + checkpoint_suffix), + ) if custom_save_every_weights: - model_add.append(os.path.join(experiment_dir, f"{model_name}_{epoch}e_{global_step}s.pth")) + model_add.append( + os.path.join( + experiment_dir, f"{model_name}_{epoch}e_{global_step}s.pth" + ) + ) overtrain_info = "" # Check overtraining if overtraining_detector and rank == 0 and epoch > 1: @@ -705,9 +827,13 @@ def train_and_evaluate( current_loss_disc = float(loss_disc) loss_disc_history.append(current_loss_disc) # Update smoothed loss history with loss_disc - smoothed_value_disc = update_exponential_moving_average(smoothed_loss_disc_history, current_loss_disc) + smoothed_value_disc = update_exponential_moving_average( + smoothed_loss_disc_history, current_loss_disc + ) # Check overtraining with smoothed loss_disc - is_overtraining_disc = check_overtraining(smoothed_loss_disc_history, overtraining_threshold * 2) + is_overtraining_disc = check_overtraining( + smoothed_loss_disc_history, overtraining_threshold * 2 + ) if is_overtraining_disc: consecutive_increases_disc += 1 else: @@ -716,9 +842,13 @@ def train_and_evaluate( current_loss_gen = float(lowest_value["value"]) loss_gen_history.append(current_loss_gen) # Update the smoothed loss_gen history - smoothed_value_gen = update_exponential_moving_average(smoothed_loss_gen_history, current_loss_gen) + smoothed_value_gen = update_exponential_moving_average( + smoothed_loss_gen_history, current_loss_gen + ) # Check for overtraining with the smoothed loss_gen - is_overtraining_gen = check_overtraining(smoothed_loss_gen_history, overtraining_threshold, 0.01) + is_overtraining_gen = check_overtraining( + smoothed_loss_gen_history, overtraining_threshold, 0.01 + ) if is_overtraining_gen: consecutive_increases_gen += 1 else: @@ -726,24 +856,50 @@ def train_and_evaluate( overtrain_info = f"Smoothed loss_g {smoothed_value_gen:.3f} and loss_d {smoothed_value_disc:.3f}" # Save the data in the JSON file if the epoch is divisible by save_every_epoch if epoch % save_every_epoch == 0: - save_to_json(training_file_path, loss_disc_history, smoothed_loss_disc_history, loss_gen_history, smoothed_loss_gen_history) + save_to_json( + training_file_path, + loss_disc_history, + smoothed_loss_disc_history, + loss_gen_history, + smoothed_loss_gen_history, + ) - if is_overtraining_gen and consecutive_increases_gen == overtraining_threshold or is_overtraining_disc and consecutive_increases_disc == overtraining_threshold * 2: - print(f"Overtraining detected at epoch {epoch} with smoothed loss_g {smoothed_value_gen:.3f} and loss_d {smoothed_value_disc:.3f}") + if ( + is_overtraining_gen + and consecutive_increases_gen == overtraining_threshold + or is_overtraining_disc + and consecutive_increases_disc == overtraining_threshold * 2 + ): + print( + f"Overtraining detected at epoch {epoch} with smoothed loss_g {smoothed_value_gen:.3f} and loss_d {smoothed_value_disc:.3f}" + ) done = True else: - print(f"New best epoch {epoch} with smoothed loss_g {smoothed_value_gen:.3f} and loss_d {smoothed_value_disc:.3f}") - old_model_files = glob.glob(os.path.join(experiment_dir, f"{model_name}_*e_*s_best_epoch.pth")) + print( + f"New best epoch {epoch} with smoothed loss_g {smoothed_value_gen:.3f} and loss_d {smoothed_value_disc:.3f}" + ) + old_model_files = glob.glob( + os.path.join(experiment_dir, f"{model_name}_*e_*s_best_epoch.pth") + ) for file in old_model_files: model_del.append(file) - model_add.append(os.path.join(experiment_dir, f"{model_name}_{epoch}e_{global_step}s_best_epoch.pth")) + model_add.append( + os.path.join( + experiment_dir, + f"{model_name}_{epoch}e_{global_step}s_best_epoch.pth", + ) + ) # Check completion if epoch >= custom_total_epoch: lowest_value_rounded = float(lowest_value["value"]) lowest_value_rounded = round(lowest_value_rounded, 3) - print(f"Training has been successfully completed with {epoch} epoch, {global_step} steps and {round(loss_gen_all.item(), 3)} loss gen.") - print(f"Lowest generator loss: {lowest_value_rounded} at epoch {lowest_value['epoch']}, step {lowest_value['step']}") + print( + f"Training has been successfully completed with {epoch} epoch, {global_step} steps and {round(loss_gen_all.item(), 3)} loss gen." + ) + print( + f"Lowest generator loss: {lowest_value_rounded} at epoch {lowest_value['epoch']}, step {lowest_value['step']}" + ) pid_file_path = os.path.join(experiment_dir, "config.json") with open(pid_file_path, "r") as pid_file: @@ -752,11 +908,19 @@ def train_and_evaluate( pid_data.pop("process_pids", None) json.dump(pid_data, pid_file, indent=4) # Final model - model_add.append(os.path.join(experiment_dir, f"{model_name}_{epoch}e_{global_step}s.pth")) + model_add.append( + os.path.join( + experiment_dir, f"{model_name}_{epoch}e_{global_step}s.pth" + ) + ) done = True - + if model_add: - ckpt = net_g.module.state_dict() if hasattr(net_g, "module") else net_g.state_dict() + ckpt = ( + net_g.module.state_dict() + if hasattr(net_g, "module") + else net_g.state_dict() + ) for m in model_add: if not os.path.exists(m): extract_model( @@ -769,7 +933,7 @@ def train_and_evaluate( step=global_step, version=version, hps=hps, - overtrain_info=overtrain_info + overtrain_info=overtrain_info, ) # Clean-up old best epochs for m in model_del: @@ -781,18 +945,27 @@ def train_and_evaluate( record = f"{model_name} | epoch={epoch} | step={global_step} | {epoch_recorder.record()}" if epoch > 1: - record = record + f" | lowest_value={lowest_value_rounded} (epoch {lowest_value['epoch']} and step {lowest_value['step']})" - + record = ( + record + + f" | lowest_value={lowest_value_rounded} (epoch {lowest_value['epoch']} and step {lowest_value['step']})" + ) + if overtraining_detector: remaining_epochs_gen = overtraining_threshold - consecutive_increases_gen - remaining_epochs_disc = overtraining_threshold * 2 - consecutive_increases_disc - record = record + f" | Number of epochs remaining for overtraining: g/total: {remaining_epochs_gen} d/total: {remaining_epochs_disc} | smoothed_loss_gen={smoothed_value_gen:.3f} | smoothed_loss_disc={smoothed_value_disc:.3f}" + remaining_epochs_disc = ( + overtraining_threshold * 2 - consecutive_increases_disc + ) + record = ( + record + + f" | Number of epochs remaining for overtraining: g/total: {remaining_epochs_gen} d/total: {remaining_epochs_disc} | smoothed_loss_gen={smoothed_value_gen:.3f} | smoothed_loss_disc={smoothed_value_disc:.3f}" + ) print(record) last_loss_gen_all = loss_gen_all if done: os._exit(2333333) + def check_overtraining(smoothed_loss_history, threshold, epsilon=0.004): """ Checks for overtraining based on the smoothed loss history. @@ -812,7 +985,10 @@ def check_overtraining(smoothed_loss_history, threshold, epsilon=0.004): return False return True -def update_exponential_moving_average(smoothed_loss_history, new_value, smoothing=0.987): + +def update_exponential_moving_average( + smoothed_loss_history, new_value, smoothing=0.987 +): """ Updates the exponential moving average with a new value. @@ -822,19 +998,22 @@ def update_exponential_moving_average(smoothed_loss_history, new_value, smoothin smoothing (float): Smoothing factor. """ if smoothed_loss_history: - smoothed_value = smoothing * smoothed_loss_history[-1] + (1 - smoothing) * new_value + smoothed_value = ( + smoothing * smoothed_loss_history[-1] + (1 - smoothing) * new_value + ) else: smoothed_value = new_value smoothed_loss_history.append(smoothed_value) return smoothed_value + def save_to_json( file_path, loss_disc_history, smoothed_loss_disc_history, loss_gen_history, smoothed_loss_gen_history, - ): +): """ Save the training history to a JSON file. """ @@ -847,6 +1026,7 @@ def save_to_json( with open(file_path, "w") as f: json.dump(data, f) + if __name__ == "__main__": torch.multiprocessing.set_start_method("spawn") main()