From b921f1616f1153d9706eaf989ea35fd11eeae518 Mon Sep 17 00:00:00 2001 From: Rayrtfr <2384172887@qq.com> Date: Tue, 12 Sep 2023 01:56:34 +0800 Subject: [PATCH] bugfix of openai_api_server for fastchat.serve.vllm_worker (#2398) Co-authored-by: wuyongyu --- fastchat/serve/openai_api_server.py | 54 +++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/fastchat/serve/openai_api_server.py b/fastchat/serve/openai_api_server.py index e399345d8..5ffcc0448 100644 --- a/fastchat/serve/openai_api_server.py +++ b/fastchat/serve/openai_api_server.py @@ -399,15 +399,28 @@ async def create_chat_completion(request: ChatCompletionRequest): for i, content in enumerate(all_tasks): if content["error_code"] != 0: return create_error_response(content["error_code"], content["text"]) - choices.append( - ChatCompletionResponseChoice( - index=i, - message=ChatMessage(role="assistant", content=content["text"]), - finish_reason=content.get("finish_reason", "stop"), + if isinstance(content["text"], list): + for t in content["text"]: + choices.append( + ChatCompletionResponseChoice( + index=i, + message=ChatMessage(role="assistant", content=t), + finish_reason=content.get("finish_reason", "stop"), + ) + ) + else: + choices.append( + ChatCompletionResponseChoice( + index=i, + message=ChatMessage(role="assistant", content=content["text"]), + finish_reason=content.get("finish_reason", "stop"), + ) ) - ) if "usage" in content: - task_usage = UsageInfo.parse_obj(content["usage"]) + if isinstance(content["usage"], list): + task_usage = UsageInfo.parse_obj(content["usage"][0]) + else: + task_usage = UsageInfo.parse_obj(content["usage"]) for usage_key, usage_value in task_usage.dict().items(): setattr(usage, usage_key, getattr(usage, usage_key) + usage_value) @@ -775,14 +788,27 @@ async def create_chat_completion(request: APIChatCompletionRequest): for i, content in enumerate(all_tasks): if content["error_code"] != 0: return create_error_response(content["error_code"], content["text"]) - choices.append( - ChatCompletionResponseChoice( - index=i, - message=ChatMessage(role="assistant", content=content["text"]), - finish_reason=content.get("finish_reason", "stop"), + if isinstance(content["text"], list): + for t in content["text"]: + choices.append( + ChatCompletionResponseChoice( + index=i, + message=ChatMessage(role="assistant", content=t), + finish_reason=content.get("finish_reason", "stop"), + ) + ) + else: + choices.append( + ChatCompletionResponseChoice( + index=i, + message=ChatMessage(role="assistant", content=content["text"]), + finish_reason=content.get("finish_reason", "stop"), + ) ) - ) - task_usage = UsageInfo.parse_obj(content["usage"]) + if isinstance(content["usage"], list): + task_usage = UsageInfo.parse_obj(content["usage"][0]) + else: + task_usage = UsageInfo.parse_obj(content["usage"]) for usage_key, usage_value in task_usage.dict().items(): setattr(usage, usage_key, getattr(usage, usage_key) + usage_value)