Add Claude tool support.

json_str/claude/tool_use.json

@@ -0,0 +1,47 @@
+data: {"type":"message_start","message":{"id":"msg_01Jp7JVrr2MFfTzUBL9hrgoH","type":"message","role":"assistant","model":"claude-3-5-sonnet-20240620","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":558,"output_tokens":1}}             }
+data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}         }
+data: {"type": "ping"}
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"I"}           }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" apolog"}   }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"ize, but I"}  }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"'ll"}            }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" need to"}             }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" respon"}    }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"d in"}            }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" English as that"}     }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"'s"}     }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" the language I've"}  }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" been instruct"}            }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"ed to use."} }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" Let"}               }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" me"} }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" r"}               }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"ephrase your"} }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" request"}  }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" an"}             }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"d procee"}               }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"d with searching"}    }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" for today"}  }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"'s news."}  }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"\n\nTo"}              }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" search for today"}   }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"'s news, I"}      }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"'ll"}    }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" use the Google search"}         }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" function"}              }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":". Here"}     }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"'s"}   }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" how"}  }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" I'll do that"}             }
+data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":":"}           }
+data: {"type":"content_block_stop","index":0               }
+data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"toolu_01M17un8HfqkS3uDKBPuBr35","name":"get_search_results","input":{}}    }
+data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":""}              }
+data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"promp"}              }
+data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"t\""}            }
+data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":": \"toda"}    }
+data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"y's "}             }
+data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"top news\"}"}        }
+data: {"type":"content_block_stop","index":1      }
+data: {"type":"message_delta","delta":{"stop_reason":"tool_use","stop_sequence":null},"usage":{"output_tokens":124}  }
+data: {"type":"message_stop"              }

json_str/claude/tools.json

@@ -0,0 +1,95 @@
+{
+    "model": "claude-3-5-sonnet-20240620",
+    "messages": [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "text",
+                    "text": "搜索今天的新闻"
+                }
+            ]
+        },
+        {
+            "role": "assistant",
+            "content": [
+                {
+                    "type": "tool_use",
+                    "id": "toolu_01RofFmKHUKsEaZvqESG5Hwz",
+                    "name": "get_search_results",
+                    "input": {
+                        "prompt": "latest news today"
+                    }
+                }
+            ]
+        },
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "tool_result",
+                    "tool_use_id": "toolu_01RofFmKHUKsEaZvqESG5Hwz",
+                    "content": "latest news today"
+                }
+            ]
+        }
+    ],
+    "temperature": 0.5,
+    "top_p": 0.7,
+    "max_tokens": 4096,
+    "stream": true,
+    "system": "You are Claude, a large language model trained by Anthropic. Use simple characters to represent mathematical symbols. Do not use LaTeX commands. Respond conversationally in English.",
+    "tools": [
+        {
+            "name": "get_search_results",
+            "description": "Search Google to enhance knowledge.",
+            "input_schema": {
+                "type": "object",
+                "properties": {
+                    "prompt": {
+                        "type": "string",
+                        "description": "The prompt to search."
+                    }
+                },
+                "required": [
+                    "prompt"
+                ]
+            }
+        },
+        {
+            "name": "get_url_content",
+            "description": "Get the webpage content of a URL",
+            "input_schema": {
+                "type": "object",
+                "properties": {
+                    "url": {
+                        "type": "string",
+                        "description": "the URL to request"
+                    }
+                },
+                "required": [
+                    "url"
+                ]
+            }
+        },
+        {
+            "name": "download_read_arxiv_pdf",
+            "description": "Get the content of the paper corresponding to the arXiv ID",
+            "input_schema": {
+                "type": "object",
+                "properties": {
+                    "prompt": {
+                        "type": "string",
+                        "description": "the arXiv ID of the paper"
+                    }
+                },
+                "required": [
+                    "prompt"
+                ]
+            }
+        }
+    ],
+    "tool_choice": {
+        "type": "auto"
+    }
+}

json_str/gpt/tool_use.json

@@ -0,0 +1,8 @@
+data: {"id":"chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV","object":"chat.completion.chunk","created":1720463853,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_abc28019ad","choices":[{"index":0,"delta":{"role":"assistant","content":null,"tool_calls":[{"index":0,"id":"call_hbFDbIHYbimw1J0v9d1qvpgl","type":"function","function":{"name":"get_search_results","arguments":""}}]},"logprobs":null,"finish_reason":null}]}
+data: {"id":"chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV","object":"chat.completion.chunk","created":1720463853,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_abc28019ad","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\""}}]},"logprobs":null,"finish_reason":null}]}
+data: {"id":"chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV","object":"chat.completion.chunk","created":1720463853,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_abc28019ad","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"prompt"}}]},"logprobs":null,"finish_reason":null}]}
+data: {"id":"chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV","object":"chat.completion.chunk","created":1720463853,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_abc28019ad","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\":\""}}]},"logprobs":null,"finish_reason":null}]}
+data: {"id":"chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV","object":"chat.completion.chunk","created":1720463853,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_abc28019ad","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"today"}}]},"logprobs":null,"finish_reason":null}]}
+data: {"id":"chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV","object":"chat.completion.chunk","created":1720463853,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_abc28019ad","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"'s"}}]},"logprobs":null,"finish_reason":null}]}
+data: {"id":"chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV","object":"chat.completion.chunk","created":1720463853,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_abc28019ad","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":" news"}}]},"logprobs":null,"finish_reason":null}]}
+data: {"id":"chatcmpl-9inWv0yEtgn873CxMBzHeCeiHctTV","object":"chat.completion.chunk","created":1720463853,"model":"gpt-4o-2024-05-13","system_fingerprint":"fp_abc28019ad","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"}"}}]},"logprobs":null,"finish_reason":null}]}

json_str/gpt/tools.json

@@ -0,0 +1,91 @@
+{
+    "model": "gpt-4o",
+    "messages": [
+        {
+            "role": "system",
+            "content": "You are ChatGPT, a large language model trained by OpenAI. Respond conversationally in English. Use simple characters to represent mathematical symbols. Do not use LaTeX commands. Knowledge cutoff: 2023-12. Current date: [ 2024-07-09 ]"
+        },
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "text",
+                    "text": "搜索今天的新闻"
+                }
+            ]
+        },
+        {
+            "role": "function",
+            "name": "get_search_results",
+            "content": "latest news today"
+        }
+    ],
+    "max_tokens": 4096,
+    "stream": true,
+    "temperature": 0.5,
+    "top_p": 1.0,
+    "presence_penalty": 0.0,
+    "frequency_penalty": 0.0,
+    "n": 1,
+    "user": "function",
+    "tools": [
+        {
+            "type": "function",
+            "function": {
+                "name": "get_search_results",
+                "description": "Search Google to enhance knowledge.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "prompt": {
+                            "type": "string",
+                            "description": "The prompt to search."
+                        }
+                    },
+                    "required": [
+                        "prompt"
+                    ]
+                }
+            }
+        },
+        {
+            "type": "function",
+            "function": {
+                "name": "get_url_content",
+                "description": "Get the webpage content of a URL",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "url": {
+                            "type": "string",
+                            "description": "the URL to request"
+                        }
+                    },
+                    "required": [
+                        "url"
+                    ]
+                }
+            }
+        },
+        {
+            "type": "function",
+            "function": {
+                "name": "download_read_arxiv_pdf",
+                "description": "Get the content of the paper corresponding to the arXiv ID",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "prompt": {
+                            "type": "string",
+                            "description": "the arXiv ID of the paper"
+                        }
+                    },
+                    "required": [
+                        "prompt"
+                    ]
+                }
+            }
+        }
+    ],
+    "tool_choice": "auto"
+}

main.py

@@ -69,7 +69,7 @@ async def process_request(request: RequestModel, provider: Dict):
         "headers": headers,
         "payload": payload
     }
-    print(f"Request details: {json.dumps(request_info, indent=2, ensure_ascii=False)}")
+    print(f"Request details: {json.dumps(request_info, indent=4, ensure_ascii=False)}")
 
     if request.stream:
         return StreamingResponse(fetch_response_stream(app.state.client, url, headers, payload, engine, request.model), media_type="text/event-stream")
@@ -86,7 +86,7 @@ class ModelRequestHandler:
     async def request_model(self, request: RequestModel, token: str):
         model_name = request.model
         matching_providers = self.get_matching_providers(model_name)
-        # print("matching_providers", json.dumps(matching_providers, indent=2, ensure_ascii=False))
+        # print("matching_providers", json.dumps(matching_providers, indent=4, ensure_ascii=False))
 
         if not matching_providers:
             raise HTTPException(status_code=404, detail="No matching model found")

request.py

@@ -191,7 +191,32 @@ async def get_claude_payload(request, engine, provider):
             content = msg.content
             name = msg.name
         if name:
-            messages.append({"role": msg.role, "name": name, "content": content})
+            # messages.append({"role": "assistant", "name": name, "content": content})
+            messages.append(
+                {
+                    "role": "assistant",
+                    "content": [
+                        {
+                            "type": "tool_use",
+                            "id": "toolu_01RofFmKHUKsEaZvqESG5Hwz",
+                            "name": name,
+                            "input": {"text": messages[-1]["content"][0]["text"]},
+                        }
+                    ]
+                }
+            )
+            messages.append(
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "tool_result",
+                            "tool_use_id": "toolu_01RofFmKHUKsEaZvqESG5Hwz",
+                            "content": content
+                        }
+                    ]
+                }
+            )
         elif msg.role != "system":
             messages.append({"role": msg.role, "content": content})
         elif msg.role == "system":
@@ -228,19 +253,18 @@ async def get_claude_payload(request, engine, provider):
         if field not in miss_fields and value is not None:
             payload[field] = value
 
-    tools = []
-    for tool in request.tools:
-        print("tool", type(tool), tool)
+    if request.tools:
+        tools = []
+        for tool in request.tools:
+            print("tool", type(tool), tool)
 
-        json_tool = await gpt2claude_tools_json(tool.dict()["function"])
-        tools.append(json_tool)
-    payload["tools"] = tools
-        # del payload["type"]
-        # del payload["function"]
-    if "tool_choice" in payload:
-        payload["tool_choice"] = {
-            "type": "auto"
-        }
+            json_tool = await gpt2claude_tools_json(tool.dict()["function"])
+            tools.append(json_tool)
+        payload["tools"] = tools
+        if "tool_choice" in payload:
+            payload["tool_choice"] = {
+                "type": "auto"
+            }
     import json
     print("payload", json.dumps(payload, indent=2, ensure_ascii=False))
 

requirements.txt

@@ -0,0 +1 @@
+fastapi

response.py

@@ -2,7 +2,7 @@ from datetime import datetime
 import json
 import httpx
 
-async def generate_sse_response(timestamp, model, content):
+async def generate_sse_response(timestamp, model, content=None, tools_id=None, function_call_name=None, function_call_content=None):
     sample_data = {
         "id": "chatcmpl-9ijPeRHa0wtyA2G8wq5z8FC3wGMzc",
         "object": "chat.completion.chunk",
@@ -19,6 +19,11 @@ async def generate_sse_response(timestamp, model, content):
         ],
         "usage": None
     }
+    if function_call_content:
+        sample_data["choices"][0]["delta"] = {"tool_calls":[{"index":0,"function":{"arguments": function_call_content}}]}
+    if tools_id and function_call_name:
+        sample_data["choices"][0]["delta"] = {"tool_calls":[{"index":0,"id":tools_id,"type":"function","function":{"name":function_call_name,"arguments":""}}]}
+        # sample_data["choices"][0]["delta"] = {"tool_calls":[{"index":0,"function":{"id": tools_id, "name": function_call_name}}]}
     json_data = json.dumps(sample_data, ensure_ascii=False)
 
     # 构建SSE响应
@@ -81,7 +86,7 @@ async def fetch_claude_response_stream(client, url, headers, payload, model):
                 for chunk in chunk_line:
                     if chunk.startswith("data:"):
                         line = chunk[6:]
-                        # print(line)
+                        print(line)
                         resp: dict = json.loads(line)
                         message = resp.get("message")
                         if message:
@@ -89,24 +94,29 @@ async def fetch_claude_response_stream(client, url, headers, payload, model):
                             if tokens_use:
                                 total_tokens = tokens_use["input_tokens"] + tokens_use["output_tokens"]
                                 # print("\n\rtotal_tokens", total_tokens)
-                        # tool_use = resp.get("content_block")
-                        # if tool_use and "tool_use" == tool_use['type']:
-                        #     # print("tool_use", tool_use)
-                        #     tools_id = tool_use["id"]
-                        #     need_function_call = True
-                        #     if "name" in tool_use:
-                        #         function_call_name = tool_use["name"]
+                        tool_use = resp.get("content_block")
+                        tools_id = None
+                        function_call_name = None
+                        if tool_use and "tool_use" == tool_use['type']:
+                            # print("tool_use", tool_use)
+                            tools_id = tool_use["id"]
+                            if "name" in tool_use:
+                                function_call_name = tool_use["name"]
+                                sse_string = await generate_sse_response(timestamp, model, None, tools_id, function_call_name, None)
+                                yield sse_string
                         delta = resp.get("delta")
                         # print("delta", delta)
                         if not delta:
                             continue
                         if "text" in delta:
                             content = delta["text"]
-                            sse_string = await generate_sse_response(timestamp, model, content)
-                            print(sse_string)
+                            sse_string = await generate_sse_response(timestamp, model, content, None, None)
+                            yield sse_string
+                        if "partial_json" in delta:
+                            # {"type":"input_json_delta","partial_json":""}
+                            function_call_content = delta["partial_json"]
+                            sse_string = await generate_sse_response(timestamp, model, None, None, None, function_call_content)
                             yield sse_string
-                        # if "partial_json" in delta:
-                        #     function_call_content = delta["partial_json"]
             yield "data: [DONE]\n\n"
     except httpx.ConnectError as e:
         print(f"连接错误： {e}")