chore: use one mod for rate limit

crates/http-api-bindings/src/chat/mod.rs

@@ -1,12 +1,10 @@
-mod rate_limit;
-
-use std::{sync::Arc, time::Duration};
+use std::sync::Arc;
 
 use async_openai::config::OpenAIConfig;
-use ratelimit::Ratelimiter;
 use tabby_common::config::HttpModelConfig;
 use tabby_inference::{ChatCompletionStream, ExtendedOpenAIConfig};
 
+use super::rate_limit;
 use crate::create_reqwest_client;
 
 pub async fn create(model: &HttpModelConfig) -> Arc<dyn ChatCompletionStream> {
@@ -19,6 +17,7 @@ pub async fn create(model: &HttpModelConfig) -> Arc<dyn ChatCompletionStream> {
         .with_api_key(model.api_key.clone().unwrap_or_default());
 
     let mut builder = ExtendedOpenAIConfig::builder();
+
     builder
         .base(config)
         .supported_models(model.supported_models.clone())
@@ -39,11 +38,8 @@ pub async fn create(model: &HttpModelConfig) -> Arc<dyn ChatCompletionStream> {
             .with_http_client(create_reqwest_client(api_endpoint)),
     );
 
-    let ratelimiter =
-        Ratelimiter::builder(model.rate_limit.request_per_minute, Duration::from_secs(60))
-            .max_tokens(model.rate_limit.request_per_minute)
-            .build()
-            .expect("Failed to create ratelimiter, please check the rate limit configuration");
-
-    Arc::new(rate_limit::RateLimitedChatStream::new(engine, ratelimiter))
+    Arc::new(rate_limit::RateLimitedChatStream::new(
+        engine,
+        model.rate_limit.request_per_minute,
+    ))
 }

crates/http-api-bindings/src/completion/mod.rs

@@ -1,17 +1,17 @@
 mod llama;
 mod mistral;
 mod openai;
-mod rate_limit;
 
-use std::{sync::Arc, time::Duration};
+use std::sync::Arc;
 
 use llama::LlamaCppEngine;
 use mistral::MistralFIMEngine;
 use openai::OpenAICompletionEngine;
-use ratelimit::Ratelimiter;
 use tabby_common::config::HttpModelConfig;
 use tabby_inference::CompletionStream;
 
+use super::rate_limit;
+
 pub async fn create(model: &HttpModelConfig) -> Arc<dyn CompletionStream> {
     let engine = match model.kind.as_str() {
         "llama.cpp/completion" => LlamaCppEngine::create(
@@ -51,13 +51,10 @@ pub async fn create(model: &HttpModelConfig) -> Arc<dyn CompletionStream> {
         ),
     };
 
-    let ratelimiter =
-        Ratelimiter::builder(model.rate_limit.request_per_minute, Duration::from_secs(60))
-            .max_tokens(model.rate_limit.request_per_minute)
-            .build()
-            .expect("Failed to create ratelimiter, please check the rate limit configuration");
-
-    Arc::new(rate_limit::RateLimitedCompletion::new(engine, ratelimiter))
+    Arc::new(rate_limit::RateLimitedCompletion::new(
+        engine,
+        model.rate_limit.request_per_minute,
+    ))
 }
 
 const FIM_TOKEN: &str = "<|FIM|>";

crates/http-api-bindings/src/embedding/mod.rs

@@ -1,17 +1,16 @@
 mod llama;
 mod openai;
-mod rate_limit;
 mod voyage;
 
 use core::panic;
-use std::{sync::Arc, time::Duration};
+use std::sync::Arc;
 
 use llama::LlamaCppEngine;
-use ratelimit::Ratelimiter;
 use tabby_common::config::HttpModelConfig;
 use tabby_inference::Embedding;
 
 use self::{openai::OpenAIEmbeddingEngine, voyage::VoyageEmbeddingEngine};
+use super::rate_limit;
 
 pub async fn create(config: &HttpModelConfig) -> Arc<dyn Embedding> {
     let engine = match config.kind.as_str() {
@@ -48,13 +47,8 @@ pub async fn create(config: &HttpModelConfig) -> Arc<dyn Embedding> {
         ),
     };
 
-    let ratelimiter = Ratelimiter::builder(
+    Arc::new(rate_limit::RateLimitedEmbedding::new(
+        engine,
         config.rate_limit.request_per_minute,
-        Duration::from_secs(60),
-    )
-    .max_tokens(config.rate_limit.request_per_minute)
-    .build()
-    .expect("Failed to create ratelimiter, please check the rate limit configuration");
-
-    Arc::new(rate_limit::RateLimitedEmbedding::new(engine, ratelimiter))
+    ))
 }

crates/http-api-bindings/src/lib.rs

@@ -1,6 +1,7 @@
 mod chat;
 mod completion;
 mod embedding;
+mod rate_limit;
 
 pub use chat::create as create_chat;
 pub use completion::{build_completion_prompt, create};