chore: use rps for rate limit

Signed-off-by: Wei Zhang <[email protected]>
TabbyML · Nov 29, 2024 · 0ab60bc · 0ab60bc
1 parent 0f8a5a0
commit 0ab60bc
Showing 1 changed file with 4 additions and 3 deletions.
diff --git a/crates/http-api-bindings/src/rate_limit.rs b/crates/http-api-bindings/src/rate_limit.rs
@@ -12,9 +12,10 @@ use ratelimit::Ratelimiter;
 use tabby_inference::{ChatCompletionStream, CompletionOptions, CompletionStream, Embedding};
 
 fn new_rate_limiter(rpm: u64) -> Ratelimiter {
-    Ratelimiter::builder(rpm, Duration::from_secs(60))
-        .max_tokens(rpm)
-        .initial_available(rpm)
+    let rps = rpm / 60;
+    Ratelimiter::builder(rps, Duration::from_secs(1))
+        .max_tokens(rps)
+        .initial_available(rps)
         .build()
         .expect("Failed to create RateLimiter, please check the HttpModelConfig.rate_limit configuration")
 }