CD-159 - speed up simulated streaming.

lm-sys · Oct 28, 2024 · a6b5c5c · a6b5c5c
1 parent 3060aad
commit a6b5c5c
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/fastchat/serve/api_provider.py b/fastchat/serve/api_provider.py
@@ -1308,7 +1308,7 @@ def jab_api_stream_iter(
         pos = 0
         while pos < len(text):
             # simulate token streaming
-            pos += 3
+            pos += 5
             time.sleep(0.001)
             data = {
                 "text": text[:pos],