Skip to content

Commit

Permalink
Break text into sentences for tts.
Browse files Browse the repository at this point in the history 
This is for models that are not using piper-phonemize as their front-end.
  • Loading branch information
@csukuangfj
csukuangfj committed Dec 3, 2023
1 parent 99ff6a8 commit 8872ea9
Show file tree
Hide file tree
Showing 3 changed files with 71 additions and 76 deletions.
113 changes: 65 additions & 48 deletions sherpa-onnx/csrc/lexicon.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,8 @@ static std::vector<int32_t> ConvertTokensToIds(

Lexicon::Lexicon(const std::string &lexicon, const std::string &tokens,
const std::string &punctuations, const std::string &language,
bool debug /*= false*/, bool is_piper /*= false*/)
: debug_(debug), is_piper_(is_piper) {
bool debug /*= false*/)
: debug_(debug) {
InitLanguage(language);

{
Expand All @@ -108,9 +108,9 @@ Lexicon::Lexicon(const std::string &lexicon, const std::string &tokens,
#if __ANDROID_API__ >= 9
Lexicon::Lexicon(AAssetManager *mgr, const std::string &lexicon,
const std::string &tokens, const std::string &punctuations,
const std::string &language, bool debug /*= false*/,
bool is_piper /*= false*/)
: debug_(debug), is_piper_(is_piper) {
const std::string &language, bool debug /*= false*/
)
: debug_(debug) {
InitLanguage(language);

{
Expand All @@ -132,16 +132,10 @@ Lexicon::Lexicon(AAssetManager *mgr, const std::string &lexicon,
std::vector<std::vector<int64_t>> Lexicon::ConvertTextToTokenIds(
const std::string &text, const std::string & /*voice*/ /*= ""*/) const {
switch (language_) {
case Language::kEnglish:
return ConvertTextToTokenIdsEnglish(text);
case Language::kGerman:
return ConvertTextToTokenIdsGerman(text);
case Language::kSpanish:
return ConvertTextToTokenIdsSpanish(text);
case Language::kFrench:
return ConvertTextToTokenIdsFrench(text);
case Language::kChinese:
return ConvertTextToTokenIdsChinese(text);
case Language::kNotChinese:
return ConvertTextToTokenIdsNotChinese(text);
default:
SHERPA_ONNX_LOGE("Unknown language: %d", static_cast<int32_t>(language_));
exit(-1);
Expand Down Expand Up @@ -197,7 +191,8 @@ std::vector<std::vector<int64_t>> Lexicon::ConvertTextToTokenIdsChinese(
fprintf(stderr, "\n");
}

std::vector<int64_t> ans;
std::vector<std::vector<int64_t>> ans;
std::vector<int64_t> this_sentence;

int32_t blank = -1;
if (token2id_.count(" ")) {
Expand All @@ -212,15 +207,32 @@ std::vector<std::vector<int64_t>> Lexicon::ConvertTextToTokenIdsChinese(
}

if (sil != -1) {
ans.push_back(sil);
this_sentence.push_back(sil);
}

for (const auto &w : words) {
if (punctuations_.count(w)) {
if (token2id_.count(w)) {
ans.push_back(token2id_.at(w));
} else if (sil != -1) {
ans.push_back(sil);
if (w == "." || w == ";" || w == "!" || w == "?" || w == "-" || w == ":" ||
w == "。" || w == "；" || w == "！" || w == "？" || w == "：" ||
w == "”" ||
// not sentence break
w == "," || w == "“" || w == "，" || w == "、") {
if (punctuations_.count(w)) {
if (token2id_.count(w)) {
this_sentence.push_back(token2id_.at(w));
} else if (sil != -1) {
this_sentence.push_back(sil);
}
}

if (w != "," && w != "“" && w != "，" && w != "、") {
if (eos != -1) {
this_sentence.push_back(eos);
}
ans.push_back(std::move(this_sentence));

if (sil != -1) {
this_sentence.push_back(sil);
}
}
continue;
}
Expand All @@ -231,24 +243,26 @@ std::vector<std::vector<int64_t>> Lexicon::ConvertTextToTokenIdsChinese(
}

const auto &token_ids = word2ids_.at(w);
ans.insert(ans.end(), token_ids.begin(), token_ids.end());
this_sentence.insert(this_sentence.end(), token_ids.begin(),
token_ids.end());
if (blank != -1) {
ans.push_back(blank);
this_sentence.push_back(blank);
}
}

if (sil != -1) {
ans.push_back(sil);
this_sentence.push_back(sil);
}

if (eos != -1) {
ans.push_back(eos);
this_sentence.push_back(eos);
}
ans.push_back(std::move(this_sentence));

return {ans};
return ans;
}

std::vector<std::vector<int64_t>> Lexicon::ConvertTextToTokenIdsEnglish(
std::vector<std::vector<int64_t>> Lexicon::ConvertTextToTokenIdsNotChinese(
const std::string &_text) const {
std::string text(_text);
ToLowerCase(&text);
Expand All @@ -271,14 +285,22 @@ std::vector<std::vector<int64_t>> Lexicon::ConvertTextToTokenIdsEnglish(

int32_t blank = token2id_.at(" ");

std::vector<int64_t> ans;
if (is_piper_ && token2id_.count("^")) {
ans.push_back(token2id_.at("^")); // sos
}
std::vector<std::vector<int64_t>> ans;
std::vector<int64_t> this_sentence;

for (const auto &w : words) {
if (punctuations_.count(w)) {
ans.push_back(token2id_.at(w));
if (w == "." || w == ";" || w == "!" || w == "?" || w == "-" || w == ":" ||
// not sentence break
w == ",") {
if (punctuations_.count(w)) {
this_sentence.push_back(token2id_.at(w));
}

if (w != ",") {
this_sentence.push_back(blank);
ans.push_back(std::move(this_sentence));
}

continue;
}

Expand All @@ -288,37 +310,32 @@ std::vector<std::vector<int64_t>> Lexicon::ConvertTextToTokenIdsEnglish(
}

const auto &token_ids = word2ids_.at(w);
ans.insert(ans.end(), token_ids.begin(), token_ids.end());
ans.push_back(blank);
this_sentence.insert(this_sentence.end(), token_ids.begin(),
token_ids.end());
this_sentence.push_back(blank);
}

if (!ans.empty()) {
if (!this_sentence.empty()) {
// remove the last blank
ans.resize(ans.size() - 1);
this_sentence.resize(this_sentence.size() - 1);
}

if (is_piper_ && token2id_.count("$")) {
ans.push_back(token2id_.at("$")); // eos
if (!this_sentence.empty()) {
ans.push_back(std::move(this_sentence));
}

return {ans};
return ans;
}

void Lexicon::InitTokens(std::istream &is) { token2id_ = ReadTokens(is); }

void Lexicon::InitLanguage(const std::string &_lang) {
std::string lang(_lang);
ToLowerCase(&lang);
if (lang == "english") {
language_ = Language::kEnglish;
} else if (lang == "german") {
language_ = Language::kGerman;
} else if (lang == "spanish") {
language_ = Language::kSpanish;
} else if (lang == "french") {
language_ = Language::kFrench;
} else if (lang == "chinese") {
if (lang == "chinese") {
language_ = Language::kChinese;
} else if (!lang.empty()) {
language_ = Language::kNotChinese;
} else {
SHERPA_ONNX_LOGE("Unknown language: %s", _lang.c_str());
exit(-1);
Expand Down
28 changes: 4 additions & 24 deletions sherpa-onnx/csrc/lexicon.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,35 +29,19 @@ class Lexicon : public OfflineTtsFrontend {
// Note: for models from piper, we won't use this class.
Lexicon(const std::string &lexicon, const std::string &tokens,
const std::string &punctuations, const std::string &language,
bool debug = false, bool is_piper = false);
bool debug = false);

#if __ANDROID_API__ >= 9
Lexicon(AAssetManager *mgr, const std::string &lexicon,
const std::string &tokens, const std::string &punctuations,
const std::string &language, bool debug = false,
bool is_piper = false);
const std::string &language, bool debug = false);
#endif

std::vector<std::vector<int64_t>> ConvertTextToTokenIds(
const std::string &text, const std::string &voice = "") const override;

private:
std::vector<std::vector<int64_t>> ConvertTextToTokenIdsGerman(
const std::string &text) const {
return ConvertTextToTokenIdsEnglish(text);
}

std::vector<std::vector<int64_t>> ConvertTextToTokenIdsSpanish(
const std::string &text) const {
return ConvertTextToTokenIdsEnglish(text);
}

std::vector<std::vector<int64_t>> ConvertTextToTokenIdsFrench(
const std::string &text) const {
return ConvertTextToTokenIdsEnglish(text);
}

std::vector<std::vector<int64_t>> ConvertTextToTokenIdsEnglish(
std::vector<std::vector<int64_t>> ConvertTextToTokenIdsNotChinese(
const std::string &text) const;

std::vector<std::vector<int64_t>> ConvertTextToTokenIdsChinese(
Expand All @@ -70,10 +54,7 @@ class Lexicon : public OfflineTtsFrontend {

private:
enum class Language {
kEnglish,
kGerman,
kSpanish,
kFrench,
kNotChinese,
kChinese,
kUnknown,
};
Expand All @@ -84,7 +65,6 @@ class Lexicon : public OfflineTtsFrontend {
std::unordered_map<std::string, int32_t> token2id_;
Language language_;
bool debug_;
bool is_piper_;

// for Chinese polyphones
std::unique_ptr<std::regex> pattern_;
Expand Down
6 changes: 2 additions & 4 deletions sherpa-onnx/csrc/offline-tts-vits-impl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,8 +195,7 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl {
} else {
frontend_ = std::make_unique<Lexicon>(
mgr, config_.model.vits.lexicon, config_.model.vits.tokens,
model_->Punctuations(), model_->Language(), config_.model.debug,
model_->IsPiper());
model_->Punctuations(), model_->Language(), config_.model.debug);
}
}
#endif
Expand All @@ -208,8 +207,7 @@ class OfflineTtsVitsImpl : public OfflineTtsImpl {
} else {
frontend_ = std::make_unique<Lexicon>(
config_.model.vits.lexicon, config_.model.vits.tokens,
model_->Punctuations(), model_->Language(), config_.model.debug,
model_->IsPiper());
model_->Punctuations(), model_->Language(), config_.model.debug);
}
}

Expand Down

0 comments on commit 8872ea9

Please sign in to comment.