From 757d770c5c1bbf8ecba1fad9e78927f04539ecd8 Mon Sep 17 00:00:00 2001 From: Akash Pathak Date: Fri, 12 Apr 2024 17:59:00 +0530 Subject: [PATCH] added tools support --- vllm/entrypoints/openai/protocol.py | 1 + vllm/entrypoints/openai/serving_chat.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/vllm/entrypoints/openai/protocol.py b/vllm/entrypoints/openai/protocol.py index f94d22d279cc4..e4ce94fa7fe5a 100644 --- a/vllm/entrypoints/openai/protocol.py +++ b/vllm/entrypoints/openai/protocol.py @@ -64,6 +64,7 @@ class ChatCompletionRequest(BaseModel): # https://platform.openai.com/docs/api-reference/chat/create messages: List[Dict[str, str]] model: str + tools: Optional[List[Dict]] = [] frequency_penalty: Optional[float] = 0.0 logit_bias: Optional[Dict[str, float]] = None logprobs: Optional[bool] = False diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py index a03c5dc88108f..72e1f597012ff 100644 --- a/vllm/entrypoints/openai/serving_chat.py +++ b/vllm/entrypoints/openai/serving_chat.py @@ -52,8 +52,9 @@ async def create_chat_completion( return error_check_ret try: - prompt = self.tokenizer.apply_chat_template( + prompt = self.tokenizer.apply_tool_use_template( conversation=request.messages, + tools=request.tools, tokenize=False, add_generation_prompt=request.add_generation_prompt) except Exception as e: