Move vocabulary to a dedicated module

dottxt-ai · Nov 11, 2024 · 5eb350d · 5eb350d
1 parent e85ce90
commit 5eb350d
Show file tree

Hide file tree

Showing 4 changed files with 6 additions and 5 deletions.
diff --git a/src/lib.rs b/src/lib.rs
@@ -5,9 +5,6 @@ pub mod primitives;
 pub mod regex;
 pub mod vocabulary;
 
-mod locator;
-mod processor;
-
 #[cfg(feature = "python-bindings")]
 mod python_bindings;
 

diff --git a/src/locator.rs → src/vocabulary/locator.rs b/src/locator.rs → src/vocabulary/locator.rs
diff --git a/src/vocabulary.rs → src/vocabulary/mod.rs b/src/vocabulary.rs → src/vocabulary/mod.rs
@@ -3,11 +3,15 @@ use std::collections::HashMap;
 use tokenizers::normalizers::Sequence;
 use tokenizers::{FromPretrainedParameters, NormalizerWrapper, Tokenizer};
 
-use crate::locator::EosTokenLocator;
 use crate::prelude::*;
-use crate::processor::TokenProcessor;
 use crate::VocabularyError;
 
+use locator::EosTokenLocator;
+use processor::TokenProcessor;
+
+mod locator;
+mod processor;
+
 /// Vocabulary of an LLM.
 ///
 /// ## Examples

diff --git a/src/processor.rs → src/vocabulary/processor.rs b/src/processor.rs → src/vocabulary/processor.rs