From 66100526e390df43c85a869852c8d167aa1c3a0e Mon Sep 17 00:00:00 2001
From: "Victoria Terenina (torymur)" <torymur@gmail.com>
Date: Fri, 8 Nov 2024 17:47:26 +0000
Subject: [PATCH] Move vocabulary and token processor to dedicated module

---
 src/lib.rs                               | 1 -
 src/{vocabulary.rs => vocabulary/mod.rs} | 5 ++++-
 src/{ => vocabulary}/processor.rs        | 0
 3 files changed, 4 insertions(+), 2 deletions(-)
 rename src/{vocabulary.rs => vocabulary/mod.rs} (99%)
 rename src/{ => vocabulary}/processor.rs (100%)

diff --git a/src/lib.rs b/src/lib.rs
index 2cba82ad..85cd301b 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -6,7 +6,6 @@ pub mod regex;
 pub mod vocabulary;
 
 mod locator;
-mod processor;
 
 #[cfg(feature = "python-bindings")]
 mod python_bindings;
diff --git a/src/vocabulary.rs b/src/vocabulary/mod.rs
similarity index 99%
rename from src/vocabulary.rs
rename to src/vocabulary/mod.rs
index 438e94b5..7972ab3c 100644
--- a/src/vocabulary.rs
+++ b/src/vocabulary/mod.rs
@@ -5,9 +5,12 @@ use tokenizers::{FromPretrainedParameters, NormalizerWrapper, Tokenizer};
 
 use crate::locator::EosTokenLocator;
 use crate::prelude::*;
-use crate::processor::TokenProcessor;
 use crate::VocabularyError;
 
+use processor::TokenProcessor;
+
+mod processor;
+
 /// Vocabulary of an LLM.
 ///
 /// ## Examples
diff --git a/src/processor.rs b/src/vocabulary/processor.rs
similarity index 100%
rename from src/processor.rs
rename to src/vocabulary/processor.rs