Move vocabulary and token processor to dedicated module

dottxt-ai · Nov 8, 2024 · 6610052 · 6610052
1 parent e85ce90
commit 6610052
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 2 deletions.
diff --git a/src/lib.rs b/src/lib.rs
@@ -6,7 +6,6 @@ pub mod regex;
 pub mod vocabulary;
 
 mod locator;
-mod processor;
 
 #[cfg(feature = "python-bindings")]
 mod python_bindings;

diff --git a/src/vocabulary.rs → src/vocabulary/mod.rs b/src/vocabulary.rs → src/vocabulary/mod.rs
@@ -5,9 +5,12 @@ use tokenizers::{FromPretrainedParameters, NormalizerWrapper, Tokenizer};
 
 use crate::locator::EosTokenLocator;
 use crate::prelude::*;
-use crate::processor::TokenProcessor;
 use crate::VocabularyError;
 
+use processor::TokenProcessor;
+
+mod processor;
+
 /// Vocabulary of an LLM.
 ///
 /// ## Examples

diff --git a/src/processor.rs → src/vocabulary/processor.rs b/src/processor.rs → src/vocabulary/processor.rs