From b832ca2cf1f249501b8cf55516cf43c77e72b3f1 Mon Sep 17 00:00:00 2001 From: Zishan Date: Tue, 11 Feb 2025 15:39:22 +0100 Subject: [PATCH 01/14] add handle streming response for anthropic --- proxy/routes/anthropic.py | 106 +++++++++++++++++- .../anthropic/test_claude_agent_streaming.py | 51 +++++++++ 2 files changed, 152 insertions(+), 5 deletions(-) create mode 100644 tests/anthropic/test_claude_agent_streaming.py diff --git a/proxy/routes/anthropic.py b/proxy/routes/anthropic.py index 86706f9..a9516ad 100644 --- a/proxy/routes/anthropic.py +++ b/proxy/routes/anthropic.py @@ -5,6 +5,7 @@ import json import httpx from typing import Any from utils.explorer import push_trace +from starlette.responses import StreamingResponse # from .open_ai import push_to_explorer proxy = APIRouter() @@ -20,6 +21,7 @@ IGNORED_HEADERS = [ "x-forwarded-proto", "x-forwarded-server", "x-real-ip", + "content-length" ] MISSING_INVARIANT_AUTH_HEADER = "Missing invariant-authorization header" @@ -32,6 +34,13 @@ END_REASONS = [ "stop_sequence" ] +MESSAGE_START = "message_start" +MESSGAE_DELTA = "message_delta" +MESSAGE_STOP = "message_stop" +CONTENT_BLOCK_START = "content_block_start" +CONTENT_BLOCK_DELTA = "content_block_delta" +CONTENT_BLOCK_STOP = "content_block_stop" + def validate_headers( invariant_authorization: str = Header(None), x_api_key: str = Header(None) ): @@ -56,6 +65,7 @@ async def anthropic_proxy( headers = { k: v for k, v in request.headers.items() if k.lower() not in IGNORED_HEADERS } + headers["accept-encoding"] = "identity" request_body = await request.body() @@ -63,18 +73,26 @@ async def anthropic_proxy( anthropic_url = f"https://api.anthropic.com/{endpoint}" client = httpx.AsyncClient() - + anthropic_request = client.build_request( "POST", anthropic_url, headers=headers, data=request_body ) - invariant_authorization = request.headers.get("invariant-authorization") - async with client: + response = await client.send(anthropic_request) + + if request_body_json.get("stream"): + return await handle_streaming_response(client, anthropic_request, dataset_name, invariant_authorization) + else: response = await client.send(anthropic_request) + if response.status_code != 200: + raise HTTPException( + status_code=response.status_code, + detail=f"Failed to fetch response: {response.text}", + ) await handle_non_streaming_response( response, dataset_name, request_body_json, invariant_authorization ) @@ -85,13 +103,14 @@ async def push_to_explorer( merged_response: dict[str, Any], request_body: dict[str, Any], invariant_authorization: str, + reformat: bool = True, ) -> None: """Pushes the full trace to the Invariant Explorer""" # Combine the messages from the request body and Anthropic response messages = request_body.get("messages", []) messages += [merged_response] - - messages = anthropic_to_invariant_messages(messages) + if reformat: + messages = anthropic_to_invariant_messages(messages) _ = await push_trace( dataset_name=dataset_name, messages=[messages], @@ -115,6 +134,83 @@ async def handle_non_streaming_response( invariant_authorization, ) +async def handle_streaming_response( + client: httpx.AsyncClient, + anthropic_request: httpx.Request, + dataset_name: str, + invariant_authorization: str +) -> StreamingResponse: + + format_invariant_response = [] + + async def event_generator() -> Any: + async with client.stream( + "POST", + anthropic_request.url, + headers=anthropic_request.headers, + content=anthropic_request.content, + ) as response: + if response.status_code != 200: + yield json.dumps( + {"error": f"Failed to fetch response: {response.status_code}"} + ).encode() + return + async for chunk in response.aiter_bytes(): + yield chunk + + process_chunk_text( + chunk, + format_invariant_response + ) + + if len(format_invariant_response) > 0 and format_invariant_response[-1].get("stop_reason") in END_REASONS: + await push_to_explorer( + dataset_name, + format_invariant_response[-1], + json.loads(anthropic_request.content), + invariant_authorization, + reformat = False + ) + + generator = event_generator() + + return StreamingResponse(generator, media_type="text/event-stream") + + +def process_chunk_text(chunk, format_invariant_response): + """ + Process the chunk of text and update the format_invariant_response + Example of chunk list: + chunk_list = [b'event: message_start\ndata: {"type":"message_start","message":{"id":"msg_012KWB6kiKvzx7r1SKs5nGA1","type":"message","role":"assistant","model":"claude-3-5-sonnet-20241022","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":20,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":5}} }\n\nevent: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} }\n\n' + ,b'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" making it an attractive destination for both business"} }\n\n' + , b'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" and leisure."} }\n\n' + , b'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n' + , b'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":301} }\n\n' + , b'event: message_stop\ndata: {"type":"message_stop" }\n\n'] + """ + + text_decode = chunk.decode().strip() + for text_block in text_decode.split("\n\n"): + text_data = text_block.split("\ndata:")[1] + text_json = json.loads(text_data) + update_format_invariant_response(text_json, format_invariant_response) + +def update_format_invariant_response(text_json, format_invariant_response): + if text_json.get("type") == MESSAGE_START: + message = text_json.get("message") + format_invariant_response.append({ + "id": message.get("id"), + "role": message.get("role"), + "content": "", + "model": message.get("model"), + "stop_reason": message.get("stop_reason"), + "stop_sequence": message.get("stop_sequence"), + }) + elif text_json.get("type") == CONTENT_BLOCK_DELTA and len(format_invariant_response) > 0: + format_invariant_response[-1]["content"] += text_json.get("delta").get("text") + elif text_json.get("type") == MESSGAE_DELTA and len(format_invariant_response) > 0: + format_invariant_response[-1]["stop_reason"] = text_json.get("delta").get("stop_reason") + def anthropic_to_invariant_messages( messages: list[dict], keep_empty_tool_response: bool = False ) -> list[dict]: diff --git a/tests/anthropic/test_claude_agent_streaming.py b/tests/anthropic/test_claude_agent_streaming.py new file mode 100644 index 0000000..0df1339 --- /dev/null +++ b/tests/anthropic/test_claude_agent_streaming.py @@ -0,0 +1,51 @@ +import anthropic +from typing import Dict +import os +from tavily import TavilyClient +import anthropic +from httpx import Client +import os +# from invariant import testing +tavily_client = TavilyClient(api_key=os.getenv("TAVILY_API_KEY")) +import datetime + +def test_streaming_response_without_toolcall(): + # Example queries + dataset_name = "claude_streaming_agent_test" + str(datetime.datetime.now().strftime("%Y%m%d%H%M%S")) + invariant_api_key = os.environ.get("INVARIANT_API_KEY") + + client = anthropic.Anthropic( + http_client=Client( + headers={ + "Invariant-Authorization": f"Bearer {invariant_api_key}" + }, + ), + base_url=f"http://localhost/api/v1/proxy/{dataset_name}/anthropic", + ) + + cities = ["Zurich", "New York", "London"] + queries = [ + "Can you introduce Zurich city within 200 words?", + "Tell me the history of New York", + "How's the weather in London next week?" + ] + # Process each query + for index,query in enumerate(queries): + messages = [ + { + "role": "user", + "content": query + } + ] + response_text = "" + with client.messages.stream( + model="claude-3-5-sonnet-20241022", + max_tokens=1024, + messages=messages, + # stream = True + ) as response: + for reply in response.text_stream: + response_text += reply + print(reply, end="", flush=True) + assert reply != "" + assert cities[index] in response_text \ No newline at end of file From a02091edcb3e8c2c5212ea5b62ca0141c0a8a753 Mon Sep 17 00:00:00 2001 From: Zishan Date: Wed, 12 Feb 2025 11:41:16 +0100 Subject: [PATCH 02/14] format change --- proxy/routes/anthropic.py | 37 +++++++++----------- resources/streaming_chunk_text/anthropic.txt | 6 ++++ 2 files changed, 22 insertions(+), 21 deletions(-) create mode 100644 resources/streaming_chunk_text/anthropic.txt diff --git a/proxy/routes/anthropic.py b/proxy/routes/anthropic.py index a9516ad..d6c5671 100644 --- a/proxy/routes/anthropic.py +++ b/proxy/routes/anthropic.py @@ -26,7 +26,7 @@ IGNORED_HEADERS = [ MISSING_INVARIANT_AUTH_HEADER = "Missing invariant-authorization header" MISSING_ANTHROPIC_AUTH_HEADER = "Missing athropic authorization header" -NOT_SUPPORTED_ENDPOINT = "Not supported OpenAI endpoint" +NOT_SUPPORTED_ENDPOINT = "Not supported Anthropic endpoint" FAILED_TO_PUSH_TRACE = "Failed to push trace to the dataset: " END_REASONS = [ "end_turn", @@ -141,7 +141,7 @@ async def handle_streaming_response( invariant_authorization: str ) -> StreamingResponse: - format_invariant_response = [] + formatted_invariant_response = [] async def event_generator() -> Any: async with client.stream( @@ -160,13 +160,13 @@ async def handle_streaming_response( process_chunk_text( chunk, - format_invariant_response + formatted_invariant_response ) - if len(format_invariant_response) > 0 and format_invariant_response[-1].get("stop_reason") in END_REASONS: + if formatted_invariant_response and formatted_invariant_response[-1].get("stop_reason") in END_REASONS: await push_to_explorer( dataset_name, - format_invariant_response[-1], + formatted_invariant_response[-1], json.loads(anthropic_request.content), invariant_authorization, reformat = False @@ -177,28 +177,23 @@ async def handle_streaming_response( return StreamingResponse(generator, media_type="text/event-stream") -def process_chunk_text(chunk, format_invariant_response): +def process_chunk_text(chunk, formatted_invariant_response): """ - Process the chunk of text and update the format_invariant_response - Example of chunk list: - chunk_list = [b'event: message_start\ndata: {"type":"message_start","message":{"id":"msg_012KWB6kiKvzx7r1SKs5nGA1","type":"message","role":"assistant","model":"claude-3-5-sonnet-20241022","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":20,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":5}} }\n\nevent: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} }\n\n' - ,b'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" making it an attractive destination for both business"} }\n\n' - , b'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" and leisure."} }\n\n' - , b'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n' - , b'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":301} }\n\n' - , b'event: message_stop\ndata: {"type":"message_stop" }\n\n'] + Process the chunk of text and update the formatted_invariant_response + Example of chunk list can be find in: + ../../resources/streaming_chunk_text/anthropic.txt """ text_decode = chunk.decode().strip() for text_block in text_decode.split("\n\n"): text_data = text_block.split("\ndata:")[1] text_json = json.loads(text_data) - update_format_invariant_response(text_json, format_invariant_response) + update_formatted_invariant_response(text_json, formatted_invariant_response) -def update_format_invariant_response(text_json, format_invariant_response): +def update_formatted_invariant_response(text_json, formatted_invariant_response): if text_json.get("type") == MESSAGE_START: message = text_json.get("message") - format_invariant_response.append({ + formatted_invariant_response.append({ "id": message.get("id"), "role": message.get("role"), "content": "", @@ -206,10 +201,10 @@ def update_format_invariant_response(text_json, format_invariant_response): "stop_reason": message.get("stop_reason"), "stop_sequence": message.get("stop_sequence"), }) - elif text_json.get("type") == CONTENT_BLOCK_DELTA and len(format_invariant_response) > 0: - format_invariant_response[-1]["content"] += text_json.get("delta").get("text") - elif text_json.get("type") == MESSGAE_DELTA and len(format_invariant_response) > 0: - format_invariant_response[-1]["stop_reason"] = text_json.get("delta").get("stop_reason") + elif text_json.get("type") == CONTENT_BLOCK_DELTA: + formatted_invariant_response[-1]["content"] += text_json.get("delta").get("text") + elif text_json.get("type") == MESSGAE_DELTA: + formatted_invariant_response[-1]["stop_reason"] = text_json.get("delta").get("stop_reason") def anthropic_to_invariant_messages( messages: list[dict], keep_empty_tool_response: bool = False diff --git a/resources/streaming_chunk_text/anthropic.txt b/resources/streaming_chunk_text/anthropic.txt new file mode 100644 index 0000000..c7f5def --- /dev/null +++ b/resources/streaming_chunk_text/anthropic.txt @@ -0,0 +1,6 @@ +anthropic_chunk_list = [b'event: message_start\ndata: {"type":"message_start","message":{"id":"msg_012KWB6kiKvzx7r1SKs5nGA1","type":"message","role":"assistant","model":"claude-3-5-sonnet-20241022","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":20,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":5}} }\n\nevent: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} }\n\n' + ,b'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" making it an attractive destination for both business"} }\n\n' + , b'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" and leisure."} }\n\n' + , b'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n' + , b'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":301} }\n\n' + , b'event: message_stop\ndata: {"type":"message_stop" }\n\n'] \ No newline at end of file From 89da7aadabbfd33e9b428eab8478897a66654f84 Mon Sep 17 00:00:00 2001 From: Zishan Date: Wed, 12 Feb 2025 11:53:31 +0100 Subject: [PATCH 03/14] remove useless code --- tests/anthropic/test_claude_agent_streaming.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/anthropic/test_claude_agent_streaming.py b/tests/anthropic/test_claude_agent_streaming.py index 0df1339..38eae7a 100644 --- a/tests/anthropic/test_claude_agent_streaming.py +++ b/tests/anthropic/test_claude_agent_streaming.py @@ -1,12 +1,9 @@ import anthropic -from typing import Dict import os -from tavily import TavilyClient import anthropic from httpx import Client import os # from invariant import testing -tavily_client = TavilyClient(api_key=os.getenv("TAVILY_API_KEY")) import datetime def test_streaming_response_without_toolcall(): From 86895c61cea0ed722601c027c3cbde71c3209c34 Mon Sep 17 00:00:00 2001 From: Zishan Date: Wed, 12 Feb 2025 14:40:57 +0100 Subject: [PATCH 04/14] add anthropic api key and remove tavily use --- .github/workflows/tests_ci.yml | 1 + proxy/routes/anthropic.py | 1 - run.sh | 1 + .../anthropic/test_claude_agent_streaming.py | 4 ++- tests/anthropic/test_claude_weather_agent.py | 26 ++++++++----------- 5 files changed, 16 insertions(+), 17 deletions(-) diff --git a/.github/workflows/tests_ci.yml b/.github/workflows/tests_ci.yml index fa97fb5..f5c135a 100644 --- a/.github/workflows/tests_ci.yml +++ b/.github/workflows/tests_ci.yml @@ -19,4 +19,5 @@ jobs: - name: Run tests env: OPENAI_API_KEY: ${{ secrets.INVARIANT_TESTING_OPENAI_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.INVARIANT_TESTING_ANTHROPIC_KEY}} run: ./run.sh tests -s -vv diff --git a/proxy/routes/anthropic.py b/proxy/routes/anthropic.py index d6c5671..a7f8173 100644 --- a/proxy/routes/anthropic.py +++ b/proxy/routes/anthropic.py @@ -6,7 +6,6 @@ import httpx from typing import Any from utils.explorer import push_trace from starlette.responses import StreamingResponse -# from .open_ai import push_to_explorer proxy = APIRouter() diff --git a/run.sh b/run.sh index 7b7a251..0815f14 100755 --- a/run.sh +++ b/run.sh @@ -68,6 +68,7 @@ tests() { --mount type=bind,source=./tests,target=/tests \ --network invariant-proxy-web-test \ -e OPENAI_API_KEY="$OPENAI_API_KEY" \ + -e ANTHROPIC_API_KEY="$ANTHROPIC_API_KEY" --env-file ./tests/.env.test \ explorer-proxy-test $@ } diff --git a/tests/anthropic/test_claude_agent_streaming.py b/tests/anthropic/test_claude_agent_streaming.py index 38eae7a..cc77249 100644 --- a/tests/anthropic/test_claude_agent_streaming.py +++ b/tests/anthropic/test_claude_agent_streaming.py @@ -5,7 +5,9 @@ from httpx import Client import os # from invariant import testing import datetime - +import pytest + +@pytest.mark.skipif(not os.getenv("ANTHROPIC_API_KEY"),reason="Anthropic API keys not set") def test_streaming_response_without_toolcall(): # Example queries dataset_name = "claude_streaming_agent_test" + str(datetime.datetime.now().strftime("%Y%m%d%H%M%S")) diff --git a/tests/anthropic/test_claude_weather_agent.py b/tests/anthropic/test_claude_weather_agent.py index 9315225..0792636 100644 --- a/tests/anthropic/test_claude_weather_agent.py +++ b/tests/anthropic/test_claude_weather_agent.py @@ -5,24 +5,23 @@ from typing import Dict import anthropic import pytest from httpx import Client -from tavily import TavilyClient class WeatherAgent: - def __init__(self, api_key: str): - self.tavily_client = TavilyClient(api_key=os.getenv("TAVILY_API_KEY")) + def __init__(self): dataset_name = "claude_weather_agent_test" + str( datetime.datetime.now().strftime("%Y%m%d%H%M%S") ) + invariant_api_key = os.environ.get("INVARIANT_API_KEY") self.client = anthropic.Anthropic( http_client=Client( - headers={"Invariant-Authorization": "Bearer "}, + headers={"Invariant-Authorization": f"Bearer {invariant_api_key}"}, ), base_url=f"http://localhost/api/v1/proxy/{dataset_name}/anthropic", ) self.get_weather_function = { "name": "get_weather", - "description": "Get the current weather in a given location", + "description": "Get the current weather in a given locatiofn", "input_schema": { "type": "object", "properties": { @@ -81,21 +80,17 @@ class WeatherAgent: def get_weather(self, location: str): """Get the current weather in a given location using latitude and longitude.""" - query = f"What is the weather in {location}?" - response = self.tavily_client.search(query) - response_content = response["results"][0]["content"] - return response["results"][0]["title"] + ":\n" + response_content + response = f'''Weather in {location}: + Good morning! Expect overcast skies with intermittent showers throughout the day. Temperatures will range from a cool 15°C in the early hours to around 19°C by mid-afternoon. Light winds from the northeast at about 10 km/h will keep conditions mild. It might be a good idea to carry an umbrella if you’re heading out. Stay dry and have a great day! + ''' + return response -@pytest.mark.skipif( - not os.getenv("ANTHROPIC_API_KEY") or not os.getenv("TAVILY_API_KEY"), - reason="API keys not set", -) +@pytest.mark.skipif(not os.getenv("ANTHROPIC_API_KEY"),reason="Anthropic API keys not set") def test_proxy_response(): """Test the proxy response for the weather agent.""" # Initialize agent with Anthropic API key - anthropic_api_key = os.getenv("ANTHROPIC_API_KEY") - weather_agent = WeatherAgent(anthropic_api_key) + weather_agent = WeatherAgent() # Example queries queries = [ @@ -107,5 +102,6 @@ def test_proxy_response(): # Process each query for index, query in enumerate(queries): response = weather_agent.get_response(query) + print("response:",response) assert response is not None assert cities[index] in response From 1aee0234fc45fe28f5424fe6f754598cf0677b04 Mon Sep 17 00:00:00 2001 From: Zishan Date: Wed, 12 Feb 2025 14:52:45 +0100 Subject: [PATCH 05/14] minor fix --- run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run.sh b/run.sh index 0815f14..3f20699 100755 --- a/run.sh +++ b/run.sh @@ -68,7 +68,7 @@ tests() { --mount type=bind,source=./tests,target=/tests \ --network invariant-proxy-web-test \ -e OPENAI_API_KEY="$OPENAI_API_KEY" \ - -e ANTHROPIC_API_KEY="$ANTHROPIC_API_KEY" + -e ANTHROPIC_API_KEY="$ANTHROPIC_API_KEY"\ --env-file ./tests/.env.test \ explorer-proxy-test $@ } From a032bbaba3cee9aa8227bf671c6669551992441e Mon Sep 17 00:00:00 2001 From: Zishan Date: Wed, 12 Feb 2025 15:04:10 +0100 Subject: [PATCH 06/14] test api key --- tests/anthropic/test_claude_weather_agent.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/anthropic/test_claude_weather_agent.py b/tests/anthropic/test_claude_weather_agent.py index 0792636..53cbccc 100644 --- a/tests/anthropic/test_claude_weather_agent.py +++ b/tests/anthropic/test_claude_weather_agent.py @@ -54,9 +54,10 @@ class WeatherAgent: tools=[self.get_weather_function], model="claude-3-5-sonnet-20241022", max_tokens=1024, - messages=messages, + messages=messages ) + # If there's tool call, Extract the tool call parameters from the response if len(response.content) > 1 and response.content[1].type == "tool_use": tool_call_params = response.content[1].input From 768e6559479eedf44c02389312d789a14d4a1cbf Mon Sep 17 00:00:00 2001 From: Zishan Date: Wed, 12 Feb 2025 17:32:10 +0100 Subject: [PATCH 07/14] adjust pytest --- proxy/routes/anthropic.py | 2 +- .../anthropic/test_claude_agent_streaming.py | 11 ++++++++--- tests/anthropic/test_claude_weather_agent.py | 19 +++++++++++++------ 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/proxy/routes/anthropic.py b/proxy/routes/anthropic.py index a7f8173..e320ac9 100644 --- a/proxy/routes/anthropic.py +++ b/proxy/routes/anthropic.py @@ -52,7 +52,7 @@ def validate_headers( @proxy.post( "/{dataset_name}/anthropic/{endpoint:path}", dependencies=[Depends(validate_headers)], -) +) async def anthropic_proxy( dataset_name: str, endpoint: str, diff --git a/tests/anthropic/test_claude_agent_streaming.py b/tests/anthropic/test_claude_agent_streaming.py index cc77249..cb7e3a8 100644 --- a/tests/anthropic/test_claude_agent_streaming.py +++ b/tests/anthropic/test_claude_agent_streaming.py @@ -6,9 +6,14 @@ import os # from invariant import testing import datetime import pytest +import sys +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -@pytest.mark.skipif(not os.getenv("ANTHROPIC_API_KEY"),reason="Anthropic API keys not set") -def test_streaming_response_without_toolcall(): +from util import * # needed for pytest fixtures + +pytest_plugins = ("pytest_asyncio") +@pytest.mark.skipif(not os.getenv("ANTHROPIC_API_KEY"), reason="No ANTHROPIC_API_KEY set") +def test_streaming_response_without_toolcall(proxy_url): # Example queries dataset_name = "claude_streaming_agent_test" + str(datetime.datetime.now().strftime("%Y%m%d%H%M%S")) invariant_api_key = os.environ.get("INVARIANT_API_KEY") @@ -19,7 +24,7 @@ def test_streaming_response_without_toolcall(): "Invariant-Authorization": f"Bearer {invariant_api_key}" }, ), - base_url=f"http://localhost/api/v1/proxy/{dataset_name}/anthropic", + base_url=f"{proxy_url}/api/v1/proxy/{dataset_name}/anthropic", ) cities = ["Zurich", "New York", "London"] diff --git a/tests/anthropic/test_claude_weather_agent.py b/tests/anthropic/test_claude_weather_agent.py index 53cbccc..ad3d2ee 100644 --- a/tests/anthropic/test_claude_weather_agent.py +++ b/tests/anthropic/test_claude_weather_agent.py @@ -5,10 +5,17 @@ from typing import Dict import anthropic import pytest from httpx import Client +import sys +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from util import * # needed for pytest fixtures + +pytest_plugins = ("pytest_asyncio",) class WeatherAgent: - def __init__(self): + def __init__(self,proxy_url): + # proxy_url = "http://localhost" dataset_name = "claude_weather_agent_test" + str( datetime.datetime.now().strftime("%Y%m%d%H%M%S") ) @@ -17,11 +24,11 @@ class WeatherAgent: http_client=Client( headers={"Invariant-Authorization": f"Bearer {invariant_api_key}"}, ), - base_url=f"http://localhost/api/v1/proxy/{dataset_name}/anthropic", + base_url=f"{proxy_url}/api/v1/proxy/{dataset_name}/anthropic", ) self.get_weather_function = { "name": "get_weather", - "description": "Get the current weather in a given locatiofn", + "description": "Get the current weather in a given location", "input_schema": { "type": "object", "properties": { @@ -87,11 +94,11 @@ class WeatherAgent: return response -@pytest.mark.skipif(not os.getenv("ANTHROPIC_API_KEY"),reason="Anthropic API keys not set") -def test_proxy_response(): +@pytest.mark.skipif(not os.getenv("ANTHROPIC_API_KEY"), reason="No ANTHROPIC_API_KEY set") +def test_proxy_response(proxy_url): """Test the proxy response for the weather agent.""" # Initialize agent with Anthropic API key - weather_agent = WeatherAgent() + weather_agent = WeatherAgent(proxy_url) # Example queries queries = [ From 4276aad7be261411ccd3b4f93fc5a5c036540fa1 Mon Sep 17 00:00:00 2001 From: Zishan Date: Thu, 13 Feb 2025 10:54:29 +0100 Subject: [PATCH 08/14] change prompt and add try/except for streming call --- .../anthropic/test_claude_agent_streaming.py | 34 ++++++++++++------- tests/anthropic/test_claude_weather_agent.py | 7 ++-- tests/open_ai/test_chat_with_tool_call.py | 16 ++++++--- tests/open_ai/test_chat_without_tool_calls.py | 19 ++++++++--- 4 files changed, 50 insertions(+), 26 deletions(-) diff --git a/tests/anthropic/test_claude_agent_streaming.py b/tests/anthropic/test_claude_agent_streaming.py index cb7e3a8..0d36ee7 100644 --- a/tests/anthropic/test_claude_agent_streaming.py +++ b/tests/anthropic/test_claude_agent_streaming.py @@ -7,6 +7,7 @@ import os import datetime import pytest import sys +import httpx sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from util import * # needed for pytest fixtures @@ -27,10 +28,10 @@ def test_streaming_response_without_toolcall(proxy_url): base_url=f"{proxy_url}/api/v1/proxy/{dataset_name}/anthropic", ) - cities = ["Zurich", "New York", "London"] + cities = ["zurich", "new york", "london"] queries = [ "Can you introduce Zurich city within 200 words?", - "Tell me the history of New York", + "Tell me the history of New York within 100 words?", "How's the weather in London next week?" ] # Process each query @@ -42,14 +43,21 @@ def test_streaming_response_without_toolcall(proxy_url): } ] response_text = "" - with client.messages.stream( - model="claude-3-5-sonnet-20241022", - max_tokens=1024, - messages=messages, - # stream = True - ) as response: - for reply in response.text_stream: - response_text += reply - print(reply, end="", flush=True) - assert reply != "" - assert cities[index] in response_text \ No newline at end of file + attempt = 0 + while attempt<3: + try: + with client.messages.stream( + model="claude-3-5-sonnet-20241022", + max_tokens=1024, + messages=messages, + # stream = True + ) as response: + for reply in response.text_stream: + response_text += reply + assert cities[index] in response_text.lower() + break + except httpx.RemoteProtocolError as e: + attempt += 1 + print(f"Streming error on attempt {attempt}: {e}") + else: + print("Max retries reached. Exiting.") \ No newline at end of file diff --git a/tests/anthropic/test_claude_weather_agent.py b/tests/anthropic/test_claude_weather_agent.py index ad3d2ee..0fac053 100644 --- a/tests/anthropic/test_claude_weather_agent.py +++ b/tests/anthropic/test_claude_weather_agent.py @@ -103,13 +103,12 @@ def test_proxy_response(proxy_url): # Example queries queries = [ "What's the weather like in Zurich city?", - "Tell me the forecast for New York", + "Tell me the weather for New York", "How's the weather in London next week?", ] - cities = ["Zurich", "New York", "London"] + cities = ["zurich", "new york", "london"] # Process each query for index, query in enumerate(queries): response = weather_agent.get_response(query) - print("response:",response) assert response is not None - assert cities[index] in response + assert cities[index] in response.lower() diff --git a/tests/open_ai/test_chat_with_tool_call.py b/tests/open_ai/test_chat_with_tool_call.py index 422e5ff..5d02ffa 100644 --- a/tests/open_ai/test_chat_with_tool_call.py +++ b/tests/open_ai/test_chat_with_tool_call.py @@ -205,10 +205,18 @@ async def test_chat_completion_with_tool_call_with_streaming( model="gpt-4o", messages=history, stream=True ) final_response = {"role": "assistant", "content": ""} - for chunk in chat_response_final: - if chunk.choices and chunk.choices[0].delta.content: - final_response["content"] += chunk.choices[0].delta.content - + attempt = 0 + while attempt<3: + try: + for chunk in chat_response_final: + if chunk.choices and chunk.choices[0].delta.content: + final_response["content"] += chunk.choices[0].delta.content + break + except httpx.RemoteProtocolError as e: + attempt += 1 + print(f"Streming error on attempt {attempt}: {e}") + else: + print("Max retries reached. Exiting.") # Fetch the trace ids for the dataset traces_response = await context.request.get( f"{explorer_api_url}/api/v1/dataset/byuser/developer/{dataset_name}/traces" diff --git a/tests/open_ai/test_chat_without_tool_calls.py b/tests/open_ai/test_chat_without_tool_calls.py index 130b597..d06aa48 100644 --- a/tests/open_ai/test_chat_without_tool_calls.py +++ b/tests/open_ai/test_chat_without_tool_calls.py @@ -6,7 +6,7 @@ import uuid import pytest from httpx import Client - +import httpx # add tests folder (parent) to sys.path from openai import OpenAI @@ -44,10 +44,19 @@ async def test_chat_completion(context, explorer_api_url, proxy_url, do_stream): expected_assistant_message = chat_response.choices[0].message.content else: full_response = "" - for chunk in chat_response: - if chunk.choices and chunk.choices[0].delta.content: - full_response += chunk.choices[0].delta.content - assert "PARIS" in full_response.upper() + attempt = 0 + while attempt<3: + try: + for chunk in chat_response: + if chunk.choices and chunk.choices[0].delta.content: + full_response += chunk.choices[0].delta.content + assert "PARIS" in full_response.upper() + break + except httpx.RemoteProtocolError as e: + attempt += 1 + print(f"Streming error on attempt {attempt}: {e}") + else: + print("Max retries reached. Exiting.") expected_assistant_message = full_response # Fetch the trace ids for the dataset From 3c0637b9467d7787721721e4f397d1a8a043e065 Mon Sep 17 00:00:00 2001 From: Zishan Date: Mon, 17 Feb 2025 15:09:03 +0100 Subject: [PATCH 09/14] organize anthropic test --- proxy/routes/anthropic.py | 83 ++++--- resources/streaming_chunk_text/anthropic.txt | 2 +- tests/anthropic/test_chat_with_tool_call.py | 219 ++++++++++++++++++ .../anthropic/test_chat_without_tool_call.py | 82 +++++++ .../anthropic/test_claude_agent_streaming.py | 63 ----- tests/anthropic/test_claude_weather_agent.py | 114 --------- tests/open_ai/test_chat_with_tool_call.py | 17 +- 7 files changed, 358 insertions(+), 222 deletions(-) create mode 100644 tests/anthropic/test_chat_with_tool_call.py create mode 100644 tests/anthropic/test_chat_without_tool_call.py delete mode 100644 tests/anthropic/test_claude_agent_streaming.py delete mode 100644 tests/anthropic/test_claude_weather_agent.py diff --git a/proxy/routes/anthropic.py b/proxy/routes/anthropic.py index e320ac9..1070af1 100644 --- a/proxy/routes/anthropic.py +++ b/proxy/routes/anthropic.py @@ -71,7 +71,7 @@ async def anthropic_proxy( request_body_json = json.loads(request_body) anthropic_url = f"https://api.anthropic.com/{endpoint}" - client = httpx.AsyncClient() + client = httpx.AsyncClient(timeout=httpx.Timeout(60)) anthropic_request = client.build_request( "POST", @@ -81,17 +81,19 @@ async def anthropic_proxy( ) invariant_authorization = request.headers.get("invariant-authorization") - response = await client.send(anthropic_request) - + print("is stream:", request_body_json.get("stream")) if request_body_json.get("stream"): return await handle_streaming_response(client, anthropic_request, dataset_name, invariant_authorization) else: - response = await client.send(anthropic_request) - if response.status_code != 200: + print("anthropic_request:", anthropic_request) + try: + response = await client.send(anthropic_request) + except httpx.HTTPStatusError as e: raise HTTPException( status_code=response.status_code, - detail=f"Failed to fetch response: {response.text}", + detail=f"Failed to fetch response: {response.text}, got error{e}", ) + print("response:", response.status_code) await handle_non_streaming_response( response, dataset_name, request_body_json, invariant_authorization ) @@ -108,6 +110,7 @@ async def push_to_explorer( # Combine the messages from the request body and Anthropic response messages = request_body.get("messages", []) messages += [merged_response] + if reformat: messages = anthropic_to_invariant_messages(messages) _ = await push_trace( @@ -168,7 +171,6 @@ async def handle_streaming_response( formatted_invariant_response[-1], json.loads(anthropic_request.content), invariant_authorization, - reformat = False ) generator = event_generator() @@ -182,12 +184,14 @@ def process_chunk_text(chunk, formatted_invariant_response): Example of chunk list can be find in: ../../resources/streaming_chunk_text/anthropic.txt """ - text_decode = chunk.decode().strip() for text_block in text_decode.split("\n\n"): - text_data = text_block.split("\ndata:")[1] - text_json = json.loads(text_data) - update_formatted_invariant_response(text_json, formatted_invariant_response) + # might be empty block + + if len(text_block.split("\ndata:"))>1: + text_data = text_block.split("\ndata:")[1] + text_json = json.loads(text_data) + update_formatted_invariant_response(text_json, formatted_invariant_response) def update_formatted_invariant_response(text_json, formatted_invariant_response): if text_json.get("type") == MESSAGE_START: @@ -200,8 +204,20 @@ def update_formatted_invariant_response(text_json, formatted_invariant_response) "stop_reason": message.get("stop_reason"), "stop_sequence": message.get("stop_sequence"), }) + elif text_json.get("type") == CONTENT_BLOCK_START and text_json.get("content_block").get("type")=="tool_use": + content_block = text_json.get("content_block") + formatted_invariant_response.append( + { + "role": "tool", + "tool_id": content_block.get("id"), + "content": "", + } + ) elif text_json.get("type") == CONTENT_BLOCK_DELTA: - formatted_invariant_response[-1]["content"] += text_json.get("delta").get("text") + if formatted_invariant_response[-1]["role"]=="assistant": + formatted_invariant_response[-1]["content"] += text_json.get("delta").get("text") + elif formatted_invariant_response[-1]["role"]=="tool": + formatted_invariant_response[-1]["content"] += text_json.get("delta").get("partial_json") elif text_json.get("type") == MESSGAE_DELTA: formatted_invariant_response[-1]["stop_reason"] = text_json.get("delta").get("stop_reason") @@ -253,24 +269,27 @@ def handle_user_message(message, keep_empty_tool_response): def handle_assistant_message(message): output = [] - for sub_message in message["content"]: - if sub_message["type"] == "text": - output.append({"role": "assistant", "content": sub_message.get("text")}) - elif sub_message["type"] == "tool_use": - output.append( - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "tool_id": sub_message.get("id"), - "type": "function", - "function": { - "name": sub_message.get("name"), - "arguments": sub_message.get("input"), - }, - } - ], - } - ) + if isinstance(message["content"], list): + for sub_message in message["content"]: + if sub_message["type"] == "text": + output.append({"role": "assistant", "content": sub_message.get("text")}) + elif sub_message["type"] == "tool_use": + output.append( + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "tool_id": sub_message.get("id"), + "type": "function", + "function": { + "name": sub_message.get("name"), + "arguments": sub_message.get("input"), + }, + } + ], + } + ) + else: + output.append({"role": "assistant", "content": message["content"]}) return output diff --git a/resources/streaming_chunk_text/anthropic.txt b/resources/streaming_chunk_text/anthropic.txt index c7f5def..c8c4480 100644 --- a/resources/streaming_chunk_text/anthropic.txt +++ b/resources/streaming_chunk_text/anthropic.txt @@ -3,4 +3,4 @@ anthropic_chunk_list = [b'event: message_start\ndata: {"type":"message_start","m , b'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" and leisure."} }\n\n' , b'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n' , b'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":301} }\n\n' - , b'event: message_stop\ndata: {"type":"message_stop" }\n\n'] \ No newline at end of file + , b'event: message_stop\ndata: {"type":"message_stop" }\n\n'] diff --git a/tests/anthropic/test_chat_with_tool_call.py b/tests/anthropic/test_chat_with_tool_call.py new file mode 100644 index 0000000..ece1348 --- /dev/null +++ b/tests/anthropic/test_chat_with_tool_call.py @@ -0,0 +1,219 @@ +import datetime +import os +from typing import Dict +import json +import anthropic +import pytest +from httpx import Client +import sys +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from util import * # needed for pytest fixtures + +pytest_plugins = ("pytest_asyncio",) + + +class WeatherAgent: + def __init__(self,proxy_url): + dataset_name = "claude_weather_agent_test" + str( + datetime.datetime.now().strftime("%Y%m%d%H%M%S") + ) + invariant_api_key = os.environ.get("INVARIANT_API_KEY","None") + self.client = anthropic.Anthropic( + http_client=Client( + headers={"Invariant-Authorization": f"Bearer {invariant_api_key}"}, + ), + base_url=f"{proxy_url}/api/v1/proxy/{dataset_name}/anthropic", + ) + self.get_weather_function = { + "name": "get_weather", + "description": "Get the current weather in a given location", + "input_schema": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": 'The unit of temperature, either "celsius" or "fahrenheit"', + }, + }, + "required": ["location"], + }, + } + + # self.system_prompt = """You are an assistant that can perform weather searches using function calls. + # When a user asks for weather information, respond with a JSON object specifying + # the function name `get_weather` and the arguments latitude and longitude are needed.""" + + def get_response(self, user_query: str) -> Dict: + """ + Get the response from the agent for a given user query for weather. + """ + messages = [{"role": "user", "content": user_query}] + response_list = [] + while True: + response = self.client.messages.create( + # system=self.system_prompt, + tools=[self.get_weather_function], + model="claude-3-5-sonnet-20241022", + max_tokens=1024, + messages=messages + ) + response_list.append(response) + # If there's tool call, Extract the tool call parameters from the response + if len(response.content) > 1 and response.content[1].type == "tool_use": + tool_call_params = response.content[1].input + tool_call_result = self.get_weather(tool_call_params["location"]) + tool_call_id = response.content[1].id + messages.append({"role": response.role, "content": response.content}) + messages.append( + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": tool_call_id, + "content": tool_call_result, + } + ], + } + ) + else: + return response_list + + def get_streaming_response(self, user_query: str) -> Dict: + messages = [{"role": "user", "content": user_query}] + response_list = [] + + def clean_quotes(text): + # Convert \' to ' + text = text.replace("\'", "'") + # Convert \" to " + text = text.replace('\"', '"') + text = text.replace("\n", " ") + return text + + while True: + json_data = "" + content = [] + response = [] + with self.client.messages.stream( + tools=[self.get_weather_function], + model="claude-3-5-sonnet-20241022", + max_tokens=1024, + messages=messages, + ) as stream: + for event in stream: + if isinstance(event, anthropic.types.RawContentBlockStartEvent): + # Start a new block + current_block = event.content_block + current_text = "" + elif isinstance(event, anthropic.types.RawContentBlockDeltaEvent): + if hasattr(event.delta, 'text'): + # Accumulate text for TextBlock + current_text += clean_quotes(event.delta.text) + elif hasattr(event.delta, 'partial_json'): + # Accumulate JSON for ToolUseBlock + json_data += clean_quotes(event.delta.partial_json) + current_text += clean_quotes(event.delta.partial_json) + elif isinstance(event, anthropic.types.RawContentBlockStopEvent): + # Block is complete, add it to content + if current_block.type == 'text': + content.append(anthropic.types.TextBlock(citations=None, text=current_text, type="text")) + elif current_block.type == 'tool_use': + content.append( + anthropic.types.ToolUseBlock(id=current_block.id, + input=json.loads(current_text), + name=current_block.name, + type="tool_use") + ) + response_list.append(content) + if isinstance(event, anthropic.types.RawMessageStopEvent) and event.message.stop_reason == "tool_use": + tool_call_params = json.loads(json_data) + tool_call_result = self.get_weather(tool_call_params["location"]) + messages.append({"role": "assistant", "content": content}) + messages.append( + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": content[-1].id, + "content": tool_call_result, + } + ], + } + ) + else: + return response_list + + def get_weather(self, location: str): + """Get the current weather in a given location using latitude and longitude.""" + response = f'''Weather in {location}: + Good morning! Expect overcast skies with intermittent showers throughout the day. + Temperatures will range from a cool 15°C in the early hours to around 19°C by mid-afternoon. + Light winds from the northeast at about 10 km/h will keep conditions mild. + It might be a good idea to carry an umbrella if you’re heading out. Stay dry and have a great day! + ''' + return response + +@pytest.mark.skipif(not os.getenv("ANTHROPIC_API_KEY"), reason="No ANTHROPIC_API_KEY set") +async def test_chat_completion_without_streaming( + context, explorer_api_url, proxy_url +): + """Test the chat completion without streaming for the weather agent.""" + + weather_agent = WeatherAgent(proxy_url) + + queries = [ + "What's the weather like in Zurich city?", + # "Tell me the weather for New York", + # "How's the weather in London next week?", + ] + cities = ["zurich", "new york", "london"] + # Process each query + for index, query in enumerate(queries): + response = weather_agent.get_response(query) + print("non streaming response:",response) + assert response is not None + assert response[0].role == "assistant" + assert response[0].stop_reason == "tool_use" + assert response[0].content[0].type == "text" + assert response[0].content[1].type == "tool_use" + assert cities[index] in response[0].content[1].input["location"].lower() + + assert response[1].role == "assistant" + assert response[1].stop_reason == "end_turn" + assert cities[index] in response[1].content[0].text.lower() + + +@pytest.mark.skipif(not os.getenv("ANTHROPIC_API_KEY"), reason="No ANTHROPIC_API_KEY set") +async def test_chat_completion_with_streaming( + context, explorer_api_url, proxy_url +): + """Test the chat completion with streaming for the weather agent.""" + weather_agent = WeatherAgent(proxy_url) + + queries = [ + "What's the weather like in Zurich city?", + # "Tell me the weather for New York", + # "How's the weather in London next week?", + ] + cities = ["zurich", "new york", "london"] + + + for index, query in enumerate(queries): + response = weather_agent.get_streaming_response(query) + print("streaming response:",response) + assert response is not None + assert response[0][0].type == "text" + assert response[0][1].type == "tool_use" + assert response[0][1].name == "get_weather" + assert cities[index] in response[0][1].input["location"].lower() + + assert response[1][0].type == "text" + assert cities[index] in response[1][0].text.lower() diff --git a/tests/anthropic/test_chat_without_tool_call.py b/tests/anthropic/test_chat_without_tool_call.py new file mode 100644 index 0000000..51d2405 --- /dev/null +++ b/tests/anthropic/test_chat_without_tool_call.py @@ -0,0 +1,82 @@ +import anthropic +import os +import anthropic +from httpx import Client +import os +# from invariant import testing +import datetime +import pytest +import sys +import httpx +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from util import * # needed for pytest fixtures + +@pytest.fixture +def client(proxy_url): + dataset_name = "claude_streaming_response_without_toolcall_test" + str(datetime.datetime.now().strftime("%Y%m%d%H%M%S")) + invariant_api_key = os.environ.get("INVARIANT_API_KEY","None") + + client = anthropic.Anthropic( + http_client=Client( + headers={"Invariant-Authorization": f"Bearer {invariant_api_key}"}, + ), + base_url=f"{proxy_url}/api/v1/proxy/{dataset_name}/anthropic", + ) + return client + + +pytest_plugins = ("pytest_asyncio") +@pytest.mark.skipif(not os.getenv("ANTHROPIC_API_KEY"), reason="No ANTHROPIC_API_KEY set") +async def test_chat_completion_without_streaming(client): + + cities = ["zurich", "new york", "london"] + queries = [ + "Can you introduce Zurich city within 200 words?", + "Tell me the history of New York within 100 words?", + "How's the weather in London next week?" + ] + + # Process each query + for query in queries: + response = client.messages.create( + # system=self.system_prompt, + model="claude-3-5-sonnet-20241022", + max_tokens=1024, + messages=[{"role": "user", "content": query}], + ) + response_text = response.content[0].text + assert response_text is not None + assert cities[queries.index(query)] in response_text.lower() + +@pytest.mark.skipif(not os.getenv("ANTHROPIC_API_KEY"), reason="No ANTHROPIC_API_KEY set") +def test_streaming_response_without_toolcall(proxy_url,client): + + cities = ["zurich", "new york", "london"] + queries = [ + "Can you introduce Zurich city within 200 words?", + "Tell me the history of New York within 100 words?", + "How's the weather in London next week?" + ] + # Process each query + for index,query in enumerate(queries): + messages = [ + { + "role": "user", + "content": query + } + ] + response_text = "" + attempt = 0 + + with client.messages.stream( + model="claude-3-5-sonnet-20241022", + max_tokens=1024, + messages=messages, + ) as response: + for reply in response.text_stream: + response_text += reply + assert cities[index] in response_text.lower() + + assert response_text is not None + assert cities[queries.index(query)] in response_text.lower() diff --git a/tests/anthropic/test_claude_agent_streaming.py b/tests/anthropic/test_claude_agent_streaming.py deleted file mode 100644 index 0d36ee7..0000000 --- a/tests/anthropic/test_claude_agent_streaming.py +++ /dev/null @@ -1,63 +0,0 @@ -import anthropic -import os -import anthropic -from httpx import Client -import os -# from invariant import testing -import datetime -import pytest -import sys -import httpx -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from util import * # needed for pytest fixtures - -pytest_plugins = ("pytest_asyncio") -@pytest.mark.skipif(not os.getenv("ANTHROPIC_API_KEY"), reason="No ANTHROPIC_API_KEY set") -def test_streaming_response_without_toolcall(proxy_url): - # Example queries - dataset_name = "claude_streaming_agent_test" + str(datetime.datetime.now().strftime("%Y%m%d%H%M%S")) - invariant_api_key = os.environ.get("INVARIANT_API_KEY") - - client = anthropic.Anthropic( - http_client=Client( - headers={ - "Invariant-Authorization": f"Bearer {invariant_api_key}" - }, - ), - base_url=f"{proxy_url}/api/v1/proxy/{dataset_name}/anthropic", - ) - - cities = ["zurich", "new york", "london"] - queries = [ - "Can you introduce Zurich city within 200 words?", - "Tell me the history of New York within 100 words?", - "How's the weather in London next week?" - ] - # Process each query - for index,query in enumerate(queries): - messages = [ - { - "role": "user", - "content": query - } - ] - response_text = "" - attempt = 0 - while attempt<3: - try: - with client.messages.stream( - model="claude-3-5-sonnet-20241022", - max_tokens=1024, - messages=messages, - # stream = True - ) as response: - for reply in response.text_stream: - response_text += reply - assert cities[index] in response_text.lower() - break - except httpx.RemoteProtocolError as e: - attempt += 1 - print(f"Streming error on attempt {attempt}: {e}") - else: - print("Max retries reached. Exiting.") \ No newline at end of file diff --git a/tests/anthropic/test_claude_weather_agent.py b/tests/anthropic/test_claude_weather_agent.py deleted file mode 100644 index 0fac053..0000000 --- a/tests/anthropic/test_claude_weather_agent.py +++ /dev/null @@ -1,114 +0,0 @@ -import datetime -import os -from typing import Dict - -import anthropic -import pytest -from httpx import Client -import sys -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from util import * # needed for pytest fixtures - -pytest_plugins = ("pytest_asyncio",) - - -class WeatherAgent: - def __init__(self,proxy_url): - # proxy_url = "http://localhost" - dataset_name = "claude_weather_agent_test" + str( - datetime.datetime.now().strftime("%Y%m%d%H%M%S") - ) - invariant_api_key = os.environ.get("INVARIANT_API_KEY") - self.client = anthropic.Anthropic( - http_client=Client( - headers={"Invariant-Authorization": f"Bearer {invariant_api_key}"}, - ), - base_url=f"{proxy_url}/api/v1/proxy/{dataset_name}/anthropic", - ) - self.get_weather_function = { - "name": "get_weather", - "description": "Get the current weather in a given location", - "input_schema": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": 'The unit of temperature, either "celsius" or "fahrenheit"', - }, - }, - "required": ["location"], - }, - } - - # self.system_prompt = """You are an assistant that can perform weather searches using function calls. - # When a user asks for weather information, respond with a JSON object specifying - # the function name `get_weather` and the arguments latitude and longitude are needed.""" - - def get_response(self, user_query: str) -> Dict: - """ - Get the response from the agent for a given user query for weather. - """ - messages = [{"role": "user", "content": user_query}] - while True: - response = self.client.messages.create( - # system=self.system_prompt, - tools=[self.get_weather_function], - model="claude-3-5-sonnet-20241022", - max_tokens=1024, - messages=messages - ) - - - # If there's tool call, Extract the tool call parameters from the response - if len(response.content) > 1 and response.content[1].type == "tool_use": - tool_call_params = response.content[1].input - tool_call_result = self.get_weather(tool_call_params["location"]) - tool_call_id = response.content[1].id - messages.append({"role": response.role, "content": response.content}) - messages.append( - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": tool_call_id, - "content": tool_call_result, - } - ], - } - ) - else: - return response.content[0].text - - def get_weather(self, location: str): - """Get the current weather in a given location using latitude and longitude.""" - response = f'''Weather in {location}: - Good morning! Expect overcast skies with intermittent showers throughout the day. Temperatures will range from a cool 15°C in the early hours to around 19°C by mid-afternoon. Light winds from the northeast at about 10 km/h will keep conditions mild. It might be a good idea to carry an umbrella if you’re heading out. Stay dry and have a great day! - ''' - return response - - -@pytest.mark.skipif(not os.getenv("ANTHROPIC_API_KEY"), reason="No ANTHROPIC_API_KEY set") -def test_proxy_response(proxy_url): - """Test the proxy response for the weather agent.""" - # Initialize agent with Anthropic API key - weather_agent = WeatherAgent(proxy_url) - - # Example queries - queries = [ - "What's the weather like in Zurich city?", - "Tell me the weather for New York", - "How's the weather in London next week?", - ] - cities = ["zurich", "new york", "london"] - # Process each query - for index, query in enumerate(queries): - response = weather_agent.get_response(query) - assert response is not None - assert cities[index] in response.lower() diff --git a/tests/open_ai/test_chat_with_tool_call.py b/tests/open_ai/test_chat_with_tool_call.py index 5d02ffa..a8c3018 100644 --- a/tests/open_ai/test_chat_with_tool_call.py +++ b/tests/open_ai/test_chat_with_tool_call.py @@ -7,7 +7,6 @@ import uuid import pytest from httpx import Client - # add tests folder (parent) to sys.path from openai import OpenAI @@ -206,17 +205,11 @@ async def test_chat_completion_with_tool_call_with_streaming( ) final_response = {"role": "assistant", "content": ""} attempt = 0 - while attempt<3: - try: - for chunk in chat_response_final: - if chunk.choices and chunk.choices[0].delta.content: - final_response["content"] += chunk.choices[0].delta.content - break - except httpx.RemoteProtocolError as e: - attempt += 1 - print(f"Streming error on attempt {attempt}: {e}") - else: - print("Max retries reached. Exiting.") + + for chunk in chat_response_final: + if chunk.choices and chunk.choices[0].delta.content: + final_response["content"] += chunk.choices[0].delta.content + # Fetch the trace ids for the dataset traces_response = await context.request.get( f"{explorer_api_url}/api/v1/dataset/byuser/developer/{dataset_name}/traces" From 5765dac7647b1e02aa73a3a7ea02547fad316ec6 Mon Sep 17 00:00:00 2001 From: Zishan Date: Mon, 17 Feb 2025 15:27:11 +0100 Subject: [PATCH 10/14] rename anthropic test file names --- ..._call.py => anthropic_test_chat_with_tool_call.py} | 11 ++++++----- ...ll.py => anthropic_test_chat_without_tool_call.py} | 0 2 files changed, 6 insertions(+), 5 deletions(-) rename tests/anthropic/{test_chat_with_tool_call.py => anthropic_test_chat_with_tool_call.py} (97%) rename tests/anthropic/{test_chat_without_tool_call.py => anthropic_test_chat_without_tool_call.py} (100%) diff --git a/tests/anthropic/test_chat_with_tool_call.py b/tests/anthropic/anthropic_test_chat_with_tool_call.py similarity index 97% rename from tests/anthropic/test_chat_with_tool_call.py rename to tests/anthropic/anthropic_test_chat_with_tool_call.py index ece1348..6f22b77 100644 --- a/tests/anthropic/test_chat_with_tool_call.py +++ b/tests/anthropic/anthropic_test_chat_with_tool_call.py @@ -100,7 +100,6 @@ class WeatherAgent: while True: json_data = "" content = [] - response = [] with self.client.messages.stream( tools=[self.get_weather_function], model="claude-3-5-sonnet-20241022", @@ -171,8 +170,8 @@ async def test_chat_completion_without_streaming( queries = [ "What's the weather like in Zurich city?", - # "Tell me the weather for New York", - # "How's the weather in London next week?", + "Tell me the weather for New York", + "How's the weather in London next week?", ] cities = ["zurich", "new york", "london"] # Process each query @@ -189,6 +188,8 @@ async def test_chat_completion_without_streaming( assert response[1].role == "assistant" assert response[1].stop_reason == "end_turn" assert cities[index] in response[1].content[0].text.lower() + + @pytest.mark.skipif(not os.getenv("ANTHROPIC_API_KEY"), reason="No ANTHROPIC_API_KEY set") @@ -200,8 +201,8 @@ async def test_chat_completion_with_streaming( queries = [ "What's the weather like in Zurich city?", - # "Tell me the weather for New York", - # "How's the weather in London next week?", + "Tell me the weather for New York", + "How's the weather in London next week?", ] cities = ["zurich", "new york", "london"] diff --git a/tests/anthropic/test_chat_without_tool_call.py b/tests/anthropic/anthropic_test_chat_without_tool_call.py similarity index 100% rename from tests/anthropic/test_chat_without_tool_call.py rename to tests/anthropic/anthropic_test_chat_without_tool_call.py From b3c5fb4d474b411696e7644526eef71449b87b91 Mon Sep 17 00:00:00 2001 From: Zishan Date: Tue, 18 Feb 2025 14:54:09 +0100 Subject: [PATCH 11/14] complete anthropic test --- proxy/routes/anthropic.py | 3 - proxy/utils/explorer.py | 1 - ...ll.py => test_anthropic_with_tool_call.py} | 67 ++++++++++++-- ...py => test_anthropic_without_tool_call.py} | 89 +++++++++++++++---- tests/open_ai/test_chat_without_tool_calls.py | 17 +--- 5 files changed, 135 insertions(+), 42 deletions(-) rename tests/anthropic/{anthropic_test_chat_with_tool_call.py => test_anthropic_with_tool_call.py} (77%) rename tests/anthropic/{anthropic_test_chat_without_tool_call.py => test_anthropic_without_tool_call.py} (52%) diff --git a/proxy/routes/anthropic.py b/proxy/routes/anthropic.py index 67d484e..c5f3c32 100644 --- a/proxy/routes/anthropic.py +++ b/proxy/routes/anthropic.py @@ -68,11 +68,9 @@ async def anthropic_proxy( ) invariant_authorization = request.headers.get("invariant-authorization") - print("is stream:", request_body_json.get("stream")) if request_body_json.get("stream"): return await handle_streaming_response(client, anthropic_request, dataset_name, invariant_authorization) else: - print("anthropic_request:", anthropic_request) try: response = await client.send(anthropic_request) except httpx.HTTPStatusError as e: @@ -80,7 +78,6 @@ async def anthropic_proxy( status_code=response.status_code, detail=f"Failed to fetch response: {response.text}, got error{e}", ) - print("response:", response.status_code) await handle_non_streaming_response( response, dataset_name, request_body_json, invariant_authorization ) diff --git a/proxy/utils/explorer.py b/proxy/utils/explorer.py index 71108d7..1d6a18f 100644 --- a/proxy/utils/explorer.py +++ b/proxy/utils/explorer.py @@ -8,7 +8,6 @@ from invariant_sdk.types.push_traces import PushTracesRequest, PushTracesRespons DEFAULT_API_URL = "https://explorer.invariantlabs.ai" - async def push_trace( messages: List[List[Dict[str, Any]]], dataset_name: str, diff --git a/tests/anthropic/anthropic_test_chat_with_tool_call.py b/tests/anthropic/test_anthropic_with_tool_call.py similarity index 77% rename from tests/anthropic/anthropic_test_chat_with_tool_call.py rename to tests/anthropic/test_anthropic_with_tool_call.py index 6f22b77..6af4836 100644 --- a/tests/anthropic/anthropic_test_chat_with_tool_call.py +++ b/tests/anthropic/test_anthropic_with_tool_call.py @@ -15,7 +15,7 @@ pytest_plugins = ("pytest_asyncio",) class WeatherAgent: def __init__(self,proxy_url): - dataset_name = "claude_weather_agent_test" + str( + self.dataset_name = "claude_weather_agent_test" + str( datetime.datetime.now().strftime("%Y%m%d%H%M%S") ) invariant_api_key = os.environ.get("INVARIANT_API_KEY","None") @@ -23,7 +23,7 @@ class WeatherAgent: http_client=Client( headers={"Invariant-Authorization": f"Bearer {invariant_api_key}"}, ), - base_url=f"{proxy_url}/api/v1/proxy/{dataset_name}/anthropic", + base_url=f"{proxy_url}/api/v1/proxy/{self.dataset_name}/anthropic", ) self.get_weather_function = { "name": "get_weather", @@ -171,13 +171,12 @@ async def test_chat_completion_without_streaming( queries = [ "What's the weather like in Zurich city?", "Tell me the weather for New York", - "How's the weather in London next week?", ] - cities = ["zurich", "new york", "london"] + cities = ["zurich", "new york"] # Process each query + responses = [] for index, query in enumerate(queries): response = weather_agent.get_response(query) - print("non streaming response:",response) assert response is not None assert response[0].role == "assistant" assert response[0].stop_reason == "tool_use" @@ -188,7 +187,33 @@ async def test_chat_completion_without_streaming( assert response[1].role == "assistant" assert response[1].stop_reason == "end_turn" assert cities[index] in response[1].content[0].text.lower() - + responses.append(response) + + traces_response = await context.request.get( + f"{explorer_api_url}/api/v1/dataset/byuser/developer/{weather_agent.dataset_name}/traces" + ) + traces = await traces_response.json() + assert len(traces) == len(queries) + + for index,trace in enumerate(traces): + trace_id = trace["id"] + # Fetch the trace + trace_response = await context.request.get( + f"{explorer_api_url}/api/v1/trace/{trace_id}" + ) + trace = await trace_response.json() + trace_messages = trace["messages"] + + assert trace_messages[0]["role"] == "user" + assert trace_messages[0]["content"] == queries[index] + assert trace_messages[1]["role"] == "assistant" + assert cities[index] in trace_messages[1]["content"].lower() + assert trace_messages[2]["role"] == "assistant" + assert trace_messages[2]["tool_calls"][0]["function"]["name"] == "get_weather" + assert cities[index] in trace_messages[2]["tool_calls"][0]["function"]["arguments"]["location"].lower() + assert trace_messages[3]["role"] == "tool" + assert trace_messages[4]["role"] == "assistant" + assert cities[index] in trace_messages[4]["content"].lower() @@ -202,14 +227,12 @@ async def test_chat_completion_with_streaming( queries = [ "What's the weather like in Zurich city?", "Tell me the weather for New York", - "How's the weather in London next week?", ] - cities = ["zurich", "new york", "london"] + cities = ["zurich", "new york"] for index, query in enumerate(queries): response = weather_agent.get_streaming_response(query) - print("streaming response:",response) assert response is not None assert response[0][0].type == "text" assert response[0][1].type == "tool_use" @@ -218,3 +241,29 @@ async def test_chat_completion_with_streaming( assert response[1][0].type == "text" assert cities[index] in response[1][0].text.lower() + + traces_response = await context.request.get( + f"{explorer_api_url}/api/v1/dataset/byuser/developer/{weather_agent.dataset_name}/traces" + ) + traces = await traces_response.json() + assert len(traces) == len(queries) + + for index,trace in enumerate(traces): + trace_id = trace["id"] + # Fetch the trace + trace_response = await context.request.get( + f"{explorer_api_url}/api/v1/trace/{trace_id}" + ) + trace = await trace_response.json() + trace_messages = trace["messages"] + assert trace_messages[0]["role"] == "user" + assert trace_messages[0]["content"] == queries[index] + assert trace_messages[1]["role"] == "assistant" + assert cities[index] in trace_messages[1]["content"].lower() + assert trace_messages[2]["role"] == "assistant" + assert trace_messages[2]["tool_calls"][0]["function"]["name"] == "get_weather" + assert cities[index] in trace_messages[2]["tool_calls"][0]["function"]["arguments"]["location"].lower() + assert trace_messages[3]["role"] == "tool" + assert trace_messages[4]["role"] == "assistant" + assert cities[index] in trace_messages[4]["content"].lower() + diff --git a/tests/anthropic/anthropic_test_chat_without_tool_call.py b/tests/anthropic/test_anthropic_without_tool_call.py similarity index 52% rename from tests/anthropic/anthropic_test_chat_without_tool_call.py rename to tests/anthropic/test_anthropic_without_tool_call.py index 51d2405..eed14be 100644 --- a/tests/anthropic/anthropic_test_chat_without_tool_call.py +++ b/tests/anthropic/test_anthropic_without_tool_call.py @@ -1,19 +1,19 @@ import anthropic import os -import anthropic from httpx import Client -import os # from invariant import testing import datetime import pytest import sys -import httpx sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from util import * # needed for pytest fixtures -@pytest.fixture -def client(proxy_url): +pytest_plugins = ("pytest_asyncio") +@pytest.mark.skipif(not os.getenv("ANTHROPIC_API_KEY"), reason="No ANTHROPIC_API_KEY set") +async def test_chat_completion_without_streaming( + context, explorer_api_url,proxy_url +): dataset_name = "claude_streaming_response_without_toolcall_test" + str(datetime.datetime.now().strftime("%Y%m%d%H%M%S")) invariant_api_key = os.environ.get("INVARIANT_API_KEY","None") @@ -23,13 +23,7 @@ def client(proxy_url): ), base_url=f"{proxy_url}/api/v1/proxy/{dataset_name}/anthropic", ) - return client - - -pytest_plugins = ("pytest_asyncio") -@pytest.mark.skipif(not os.getenv("ANTHROPIC_API_KEY"), reason="No ANTHROPIC_API_KEY set") -async def test_chat_completion_without_streaming(client): - + cities = ["zurich", "new york", "london"] queries = [ "Can you introduce Zurich city within 200 words?", @@ -38,6 +32,7 @@ async def test_chat_completion_without_streaming(client): ] # Process each query + responses = [] for query in queries: response = client.messages.create( # system=self.system_prompt, @@ -46,12 +41,50 @@ async def test_chat_completion_without_streaming(client): messages=[{"role": "user", "content": query}], ) response_text = response.content[0].text + responses.append(response_text) assert response_text is not None assert cities[queries.index(query)] in response_text.lower() -@pytest.mark.skipif(not os.getenv("ANTHROPIC_API_KEY"), reason="No ANTHROPIC_API_KEY set") -def test_streaming_response_without_toolcall(proxy_url,client): + traces_response = await context.request.get( + f"{explorer_api_url}/api/v1/dataset/byuser/developer/{dataset_name}/traces" + ) + traces = await traces_response.json() + assert len(traces) == len(queries) + for index,trace in enumerate(traces): + trace_id = trace["id"] + # Fetch the trace + trace_response = await context.request.get( + f"{explorer_api_url}/api/v1/trace/{trace_id}" + ) + trace = await trace_response.json() + assert trace["messages"] == [ + { + "role": "user", + "content": queries[index] + }, + { + "role": "assistant", + "content": responses[index] + } + ] + +@pytest.mark.skipif(not os.getenv("ANTHROPIC_API_KEY"), reason="No ANTHROPIC_API_KEY set") +async def test_streaming_response_without_toolcall( + context, + explorer_api_url, + proxy_url): + + dataset_name = "claude_streaming_response_without_toolcall_test" + str(datetime.datetime.now().strftime("%Y%m%d%H%M%S")) + invariant_api_key = os.environ.get("INVARIANT_API_KEY","None") + + client = anthropic.Anthropic( + http_client=Client( + headers={"Invariant-Authorization": f"Bearer {invariant_api_key}"}, + ), + base_url=f"{proxy_url}/api/v1/proxy/{dataset_name}/anthropic", + ) + cities = ["zurich", "new york", "london"] queries = [ "Can you introduce Zurich city within 200 words?", @@ -59,6 +92,7 @@ def test_streaming_response_without_toolcall(proxy_url,client): "How's the weather in London next week?" ] # Process each query + responses = [] for index,query in enumerate(queries): messages = [ { @@ -67,7 +101,6 @@ def test_streaming_response_without_toolcall(proxy_url,client): } ] response_text = "" - attempt = 0 with client.messages.stream( model="claude-3-5-sonnet-20241022", @@ -77,6 +110,30 @@ def test_streaming_response_without_toolcall(proxy_url,client): for reply in response.text_stream: response_text += reply assert cities[index] in response_text.lower() - + responses.append(response_text) assert response_text is not None assert cities[queries.index(query)] in response_text.lower() + + traces_response = await context.request.get( + f"{explorer_api_url}/api/v1/dataset/byuser/developer/{dataset_name}/traces" + ) + traces = await traces_response.json() + assert len(traces) == len(queries) + + for index,trace in enumerate(traces): + trace_id = trace["id"] + # Fetch the trace + trace_response = await context.request.get( + f"{explorer_api_url}/api/v1/trace/{trace_id}" + ) + trace = await trace_response.json() + assert trace["messages"] == [ + { + "role": "user", + "content": queries[index] + }, + { + "role": "assistant", + "content": responses[index] + } + ] \ No newline at end of file diff --git a/tests/open_ai/test_chat_without_tool_calls.py b/tests/open_ai/test_chat_without_tool_calls.py index d06aa48..245ace2 100644 --- a/tests/open_ai/test_chat_without_tool_calls.py +++ b/tests/open_ai/test_chat_without_tool_calls.py @@ -44,19 +44,10 @@ async def test_chat_completion(context, explorer_api_url, proxy_url, do_stream): expected_assistant_message = chat_response.choices[0].message.content else: full_response = "" - attempt = 0 - while attempt<3: - try: - for chunk in chat_response: - if chunk.choices and chunk.choices[0].delta.content: - full_response += chunk.choices[0].delta.content - assert "PARIS" in full_response.upper() - break - except httpx.RemoteProtocolError as e: - attempt += 1 - print(f"Streming error on attempt {attempt}: {e}") - else: - print("Max retries reached. Exiting.") + for chunk in chat_response: + if chunk.choices and chunk.choices[0].delta.content: + full_response += chunk.choices[0].delta.content + assert "PARIS" in full_response.upper() expected_assistant_message = full_response # Fetch the trace ids for the dataset From fd0dd4c69ea92b61f914cb93c944fdc5e9d2528e Mon Sep 17 00:00:00 2001 From: Zishan Date: Tue, 18 Feb 2025 16:12:14 +0100 Subject: [PATCH 12/14] add another way for authorization --- proxy/routes/anthropic.py | 26 ++++++++++++++++++-------- run.sh | 5 +++-- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/proxy/routes/anthropic.py b/proxy/routes/anthropic.py index c5f3c32..3d86bf7 100644 --- a/proxy/routes/anthropic.py +++ b/proxy/routes/anthropic.py @@ -14,13 +14,12 @@ proxy = APIRouter() ALLOWED_ANTHROPIC_ENDPOINTS = {"v1/messages"} -MISSING_INVARIANT_AUTH_HEADER = "Missing invariant-authorization header" -MISSING_ANTHROPIC_AUTH_HEADER = "Missing athropic authorization header" +MISSING_INVARIANT_AUTH_API_KEY = "Missing invariant authorization header" +MISSING_ANTHROPIC_AUTH_HEADER = "Missing anthropic authorization header" NOT_SUPPORTED_ENDPOINT = "Not supported Anthropic endpoint" FAILED_TO_PUSH_TRACE = "Failed to push trace to the dataset: " END_REASONS = ["end_turn", "max_tokens", "stop_sequence"] - MESSAGE_START = "message_start" MESSGAE_DELTA = "message_delta" MESSAGE_STOP = "message_stop" @@ -29,11 +28,9 @@ CONTENT_BLOCK_DELTA = "content_block_delta" CONTENT_BLOCK_STOP = "content_block_stop" def validate_headers( - invariant_authorization: str = Header(None), x_api_key: str = Header(None) + x_api_key: str = Header(None) ): - """Require the invariant-authorization and authorization headers to be present""" - if invariant_authorization is None: - raise HTTPException(status_code=400, detail=MISSING_INVARIANT_AUTH_HEADER) + """Require the headers to be present""" if x_api_key is None: raise HTTPException(status_code=400, detail=MISSING_ANTHROPIC_AUTH_HEADER) @@ -55,6 +52,20 @@ async def anthropic_proxy( } headers["accept-encoding"] = "identity" + if request.headers.get( + "invariant-authorization" + ) is None and "|invariant-auth:" not in request.headers.get("authorization"): + raise HTTPException(status_code=400, detail=MISSING_INVARIANT_AUTH_API_KEY) + + if request.headers.get("invariant-authorization"): + invariant_authorization = request.headers.get("invariant-authorization") + else: + authorization = request.headers.get("x-api-key") + api_keys = authorization.split("|invariant-auth: ") + invariant_authorization = f"Bearer {api_keys[1].strip()}" + # Update the authorization header to pass the OpenAI API Key to the OpenAI API + headers["x-api-key"] = f"{api_keys[0].strip()}" + request_body = await request.body() request_body_json = json.loads(request_body) @@ -66,7 +77,6 @@ async def anthropic_proxy( anthropic_request = client.build_request( "POST", anthropic_url, headers=headers, data=request_body ) - invariant_authorization = request.headers.get("invariant-authorization") if request_body_json.get("stream"): return await handle_streaming_response(client, anthropic_request, dataset_name, invariant_authorization) diff --git a/run.sh b/run.sh index 01f1bb4..fd6c075 100755 --- a/run.sh +++ b/run.sh @@ -6,8 +6,8 @@ up() { # Start your local docker-compose services docker compose -f docker-compose.local.yml up -d - echo "Proxy started at http://localhost:8005/api/v1/proxy/" - echo "See http://localhost:8005/api/v1/proxy/docs for API documentation" + echo "Proxy started at http://localhost/api/v1/proxy/" + echo "See http://localhost/api/v1/proxy/docs for API documentation" } build() { @@ -65,6 +65,7 @@ tests() { docker build -t 'explorer-proxy-tests' -f ./tests/Dockerfile.test ./tests docker run \ + -it \ --mount type=bind,source=./tests,target=/tests \ --network invariant-proxy-web-test \ -e OPENAI_API_KEY="$OPENAI_API_KEY" \ From 8fa4beb3b32d87fa7a8c9f47f9daa050ba0b41ee Mon Sep 17 00:00:00 2001 From: Zishan Date: Tue, 18 Feb 2025 16:18:44 +0100 Subject: [PATCH 13/14] remove -it from run.sh --- proxy/routes/anthropic.py | 2 +- run.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/proxy/routes/anthropic.py b/proxy/routes/anthropic.py index 3d86bf7..fd0f0fa 100644 --- a/proxy/routes/anthropic.py +++ b/proxy/routes/anthropic.py @@ -15,7 +15,7 @@ proxy = APIRouter() ALLOWED_ANTHROPIC_ENDPOINTS = {"v1/messages"} MISSING_INVARIANT_AUTH_API_KEY = "Missing invariant authorization header" -MISSING_ANTHROPIC_AUTH_HEADER = "Missing anthropic authorization header" +MISSING_ANTHROPIC_AUTH_HEADER = "Missing Anthropic authorization header" NOT_SUPPORTED_ENDPOINT = "Not supported Anthropic endpoint" FAILED_TO_PUSH_TRACE = "Failed to push trace to the dataset: " END_REASONS = ["end_turn", "max_tokens", "stop_sequence"] diff --git a/run.sh b/run.sh index fd6c075..6561b48 100755 --- a/run.sh +++ b/run.sh @@ -65,7 +65,6 @@ tests() { docker build -t 'explorer-proxy-tests' -f ./tests/Dockerfile.test ./tests docker run \ - -it \ --mount type=bind,source=./tests,target=/tests \ --network invariant-proxy-web-test \ -e OPENAI_API_KEY="$OPENAI_API_KEY" \ From 0adf3729c22d9d71ecd147f87c32e4a1788105c9 Mon Sep 17 00:00:00 2001 From: Zishan Date: Tue, 18 Feb 2025 16:46:51 +0100 Subject: [PATCH 14/14] remove useless code --- proxy/utils/constants.py | 1 - run.sh | 4 ++-- tests/anthropic/test_anthropic_with_tool_call.py | 5 ----- tests/anthropic/test_anthropic_without_tool_call.py | 2 -- tests/open_ai/test_chat_with_tool_call.py | 1 - tests/open_ai/test_chat_without_tool_calls.py | 1 - 6 files changed, 2 insertions(+), 12 deletions(-) diff --git a/proxy/utils/constants.py b/proxy/utils/constants.py index a8b61d3..858a971 100644 --- a/proxy/utils/constants.py +++ b/proxy/utils/constants.py @@ -10,7 +10,6 @@ IGNORED_HEADERS = [ "x-forwarded-proto", "x-forwarded-server", "x-real-ip", - "content-length" ] CLIENT_TIMEOUT = 60.0 diff --git a/run.sh b/run.sh index 6561b48..01f1bb4 100755 --- a/run.sh +++ b/run.sh @@ -6,8 +6,8 @@ up() { # Start your local docker-compose services docker compose -f docker-compose.local.yml up -d - echo "Proxy started at http://localhost/api/v1/proxy/" - echo "See http://localhost/api/v1/proxy/docs for API documentation" + echo "Proxy started at http://localhost:8005/api/v1/proxy/" + echo "See http://localhost:8005/api/v1/proxy/docs for API documentation" } build() { diff --git a/tests/anthropic/test_anthropic_with_tool_call.py b/tests/anthropic/test_anthropic_with_tool_call.py index 6af4836..56be242 100644 --- a/tests/anthropic/test_anthropic_with_tool_call.py +++ b/tests/anthropic/test_anthropic_with_tool_call.py @@ -45,10 +45,6 @@ class WeatherAgent: }, } - # self.system_prompt = """You are an assistant that can perform weather searches using function calls. - # When a user asks for weather information, respond with a JSON object specifying - # the function name `get_weather` and the arguments latitude and longitude are needed.""" - def get_response(self, user_query: str) -> Dict: """ Get the response from the agent for a given user query for weather. @@ -57,7 +53,6 @@ class WeatherAgent: response_list = [] while True: response = self.client.messages.create( - # system=self.system_prompt, tools=[self.get_weather_function], model="claude-3-5-sonnet-20241022", max_tokens=1024, diff --git a/tests/anthropic/test_anthropic_without_tool_call.py b/tests/anthropic/test_anthropic_without_tool_call.py index eed14be..91995f6 100644 --- a/tests/anthropic/test_anthropic_without_tool_call.py +++ b/tests/anthropic/test_anthropic_without_tool_call.py @@ -1,7 +1,6 @@ import anthropic import os from httpx import Client -# from invariant import testing import datetime import pytest import sys @@ -35,7 +34,6 @@ async def test_chat_completion_without_streaming( responses = [] for query in queries: response = client.messages.create( - # system=self.system_prompt, model="claude-3-5-sonnet-20241022", max_tokens=1024, messages=[{"role": "user", "content": query}], diff --git a/tests/open_ai/test_chat_with_tool_call.py b/tests/open_ai/test_chat_with_tool_call.py index a8c3018..72f7f8f 100644 --- a/tests/open_ai/test_chat_with_tool_call.py +++ b/tests/open_ai/test_chat_with_tool_call.py @@ -204,7 +204,6 @@ async def test_chat_completion_with_tool_call_with_streaming( model="gpt-4o", messages=history, stream=True ) final_response = {"role": "assistant", "content": ""} - attempt = 0 for chunk in chat_response_final: if chunk.choices and chunk.choices[0].delta.content: diff --git a/tests/open_ai/test_chat_without_tool_calls.py b/tests/open_ai/test_chat_without_tool_calls.py index 245ace2..b96ae43 100644 --- a/tests/open_ai/test_chat_without_tool_calls.py +++ b/tests/open_ai/test_chat_without_tool_calls.py @@ -6,7 +6,6 @@ import uuid import pytest from httpx import Client -import httpx # add tests folder (parent) to sys.path from openai import OpenAI