async metrics

examples/openai_completion_client.py

@@ -2,7 +2,7 @@
 
 # Modify OpenAI's API key and API base to use vLLM's API server.
 openai_api_key = "EMPTY"
-openai_api_base = "http://localhost:8000/v1"
+openai_api_base = "http://localhost:8001/v1"
 
 client = OpenAI(
     # defaults to os.environ.get("OPENAI_API_KEY")

vllm/engine/async_llm_engine.py

@@ -33,6 +33,7 @@
 logger = init_logger(__name__)
 ENGINE_ITERATION_TIMEOUT_S = envs.VLLM_ENGINE_ITERATION_TIMEOUT_S
 
+_running_tasks = set()
 
 class AsyncEngineDeadError(RuntimeError):
     pass
@@ -251,6 +252,10 @@ def has_new_requests(self):
 class _AsyncLLMEngine(LLMEngine):
     """Extension of LLMEngine to add async methods."""
 
+    async def do_log_stats_async(self, scheduler_outputs, model_output):
+        self.do_log_stats(scheduler_outputs, model_output)
+
+
     async def step_async(
         self, virtual_engine: int
     ) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
@@ -289,7 +294,11 @@ async def step_async(
             scheduler_outputs.ignored_seq_groups, seq_group_metadata_list)
 
         # Log stats.
-        self.do_log_stats(scheduler_outputs, output)
+        log_task = asyncio.create_task(self.do_log_stats_async(
+            scheduler_outputs, output))
+        _running_tasks.add(log_task)
+        log_task.add_done_callback(_running_tasks.discard)
+        # self.do_log_stats(scheduler_outputs, output)
 
         # Tracing
         self.do_tracing(scheduler_outputs)
@@ -1068,7 +1077,8 @@ async def do_log_stats(
             await self.engine.do_log_stats.remote(  # type: ignore
                 scheduler_outputs, model_output)
         else:
-            self.engine.do_log_stats()
+            self.engine.do_log_stats(scheduler_outputs,
+                                     model_output)
 
     async def check_health(self) -> None:
         """Raises an error if engine is unhealthy."""