diff --git a/crates/bpe/src/interval_encoding.rs b/crates/bpe/src/interval_encoding.rs index c296bd0..d17b22b 100644 --- a/crates/bpe/src/interval_encoding.rs +++ b/crates/bpe/src/interval_encoding.rs @@ -11,7 +11,7 @@ use crate::byte_pair_encoding::BytePairEncoding; /// is the predecessor of another node. /// With the `tree_depth` field the number of path length (which is equivalent to the number of encoded tokens) can be determined /// in O(1) as well. -/// +/// /// Note: the fields `tree_end` and `tree_depth` could also be represented by succinct data structures, reducing their size drastically. /// Since we still need the `tree_id` and `last_token` fields, this would in total reduce memory footprint by a bit less than 50%. pub struct IntervalEncoding<'a> {