Make benchmark_guided_serving.py

Signed-off-by: mgoin <[email protected]>
vllm-project · Dec 3, 2024 · 7854d0e · 7854d0e
1 parent 7090c27
commit 7854d0e
Show file tree

Hide file tree

Showing 2 changed files with 508 additions and 0 deletions.
diff --git a/benchmarks/backend_request_func.py b/benchmarks/backend_request_func.py
@@ -26,6 +26,7 @@ class RequestFuncInput:
     logprobs: Optional[int] = None
     multi_modal_content: Optional[dict] = None
     ignore_eos: bool = False
+    extra_body: Optional[dict[str, str]] = None
 
 
 @dataclass
@@ -242,6 +243,8 @@ async def async_request_openai_completions(
             "stream": True,
             "ignore_eos": request_func_input.ignore_eos,
         }
+        if request_func_input.extra_body:
+            payload.update(request_func_input.extra_body)
         headers = {
             "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"
         }