From 094970f6b8eec3040015203e097b7567479cc7d8 Mon Sep 17 00:00:00 2001 From: Hemang Date: Mon, 10 Feb 2025 05:43:27 +0100 Subject: [PATCH] Add openai test for chat completions without streaming and without tool calls. --- .gitignore | 1 + run.sh | 7 ++- tests/.env.test | 1 + tests/Dockerfile.test | 3 +- tests/docker-compose.test.yml | 2 +- tests/open_ai/test_chat_without_tool_calls.py | 57 ++++++++++++++++--- tests/requirements.txt | 4 +- tests/util.py | 15 ++++- 8 files changed, 76 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index 3f5d309..c62d361 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ __pycache__/ .pytest_cache/ .py[oc] data/ +tests/results/ # Coverage and build artifacts .coverage diff --git a/run.sh b/run.sh index 2a5211d..86fc5d1 100755 --- a/run.sh +++ b/run.sh @@ -58,11 +58,16 @@ tests() { echo "app-api and proxy are available. Running tests..." + # Make call to signup endpoint + curl -k -X POST http://127.0.0.1/api/v1/user/signup + docker build -t 'explorer-proxy-test' -f ./tests/Dockerfile.test ./tests docker run \ --mount type=bind,source=./tests,target=/tests \ - --network host \ + --network invariant-proxy-web-test \ + -e OPENAI_API_KEY="$OPENAI_API_KEY" \ + --env-file ./tests/.env.test \ explorer-proxy-test $@ } diff --git a/tests/.env.test b/tests/.env.test index f52d757..e9e47a0 100644 --- a/tests/.env.test +++ b/tests/.env.test @@ -2,6 +2,7 @@ # from the proxy. Both proxy and app-api are on the same network: # invariant-proxy-web-test INVARIANT_API_URL=http://explorer-proxy-test-app-api:8000 +INVARIANT_PROXY_API_URL=http://explorer-proxy-test:8000 POSTGRES_USER=postgres POSTGRES_PASSWORD=postgres diff --git a/tests/Dockerfile.test b/tests/Dockerfile.test index 54476b4..54514e0 100644 --- a/tests/Dockerfile.test +++ b/tests/Dockerfile.test @@ -3,6 +3,7 @@ FROM mcr.microsoft.com/playwright/python:v1.50.0-noble RUN mkdir -p /tests COPY ./requirements.txt /tests/requirements.txt WORKDIR /tests +RUN pip install --upgrade pip RUN pip install --no-cache-dir -r requirements.txt -ENTRYPOINT ["pytest", "-s", "-vv"] \ No newline at end of file +ENTRYPOINT ["pytest", "--capture=tee-sys", "--tracing", "off", "--junit-xml=/tests/results/test-results-all.xml", "-s", "-vv"] \ No newline at end of file diff --git a/tests/docker-compose.test.yml b/tests/docker-compose.test.yml index dd46370..acbed5b 100644 --- a/tests/docker-compose.test.yml +++ b/tests/docker-compose.test.yml @@ -83,7 +83,7 @@ services: - "traefik.http.services.explorer-test-api.loadbalancer.server.port=8000" - "traefik.docker.network=invariant-proxy-web-test" healthcheck: - test: curl -X GET -I http://localhost:8000/api/v1 --fail + test: curl -X GET -I http://localhost:8000/api/v1/ --fail interval: 1s timeout: 5s diff --git a/tests/open_ai/test_chat_without_tool_calls.py b/tests/open_ai/test_chat_without_tool_calls.py index 5f6c7b1..5ad439a 100644 --- a/tests/open_ai/test_chat_without_tool_calls.py +++ b/tests/open_ai/test_chat_without_tool_calls.py @@ -1,9 +1,14 @@ """Test the chat completions proxy calls without tool calling.""" import os +import sys +import uuid + +import pytest +from httpx import Client # add tests folder (parent) to sys.path -import sys +from openai import OpenAI sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -12,11 +17,47 @@ from util import * # needed for pytest fixtures pytest_plugins = ("pytest_asyncio",) -async def test_hello_world(context, url): - """Demo test""" - response = await context.request.get( - f"{url}/api/v1/dataset/byuser/developer/Welcome-to-Explorer" +@pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="No OPENAI_API_KEY set") +async def test_chat_completion_without_streaming(context, explorer_api_url, proxy_url): + """Test the chat completions proxy calls without tool calling.""" + dataset_name = "test-dataset-open-ai-" + str(uuid.uuid4()) + + client = OpenAI( + http_client=Client( + headers={ + "Invariant-Authorization": "Bearer " + }, # This key is not used for local tests + ), + base_url=f"{proxy_url}/api/v1/proxy/{dataset_name}/openai", ) - dataset = await response.json() - assert dataset["name"] == "Welcome-to-Explorer" - assert dataset["user"]["username"] == "developer" + + chat_response = client.chat.completions.create( + model="gpt-4o", + messages=[{"role": "user", "content": "What is the capital of France?"}], + ) + + # Verify the chat response + assert "PARIS" in chat_response.choices[0].message.content.upper() + + # Fetch the trace ids for the dataset + traces_response = await context.request.get( + f"{explorer_api_url}/api/v1/dataset/byuser/developer/{dataset_name}/traces" + ) + traces = await traces_response.json() + assert len(traces) == 1 + trace_id = traces[0]["id"] + + # Fetch the trace + trace_response = await context.request.get( + f"{explorer_api_url}/api/v1/trace/{trace_id}" + ) + trace = await trace_response.json() + + # Verify the trace messages + assert len(trace["messages"]) == 2 + assert trace["messages"][0] == { + "role": "user", + "content": "What is the capital of France?", + } + assert trace["messages"][1]["role"] == "assistant" + assert "PARIS" in trace["messages"][1]["content"].upper() diff --git a/tests/requirements.txt b/tests/requirements.txt index 9160ba9..9f2ec8a 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,4 +1,6 @@ +anthropic openai pytest pytest-asyncio -pytest-playwright \ No newline at end of file +pytest-playwright +tavily-python \ No newline at end of file diff --git a/tests/util.py b/tests/util.py index d8bfb23..30826aa 100644 --- a/tests/util.py +++ b/tests/util.py @@ -1,12 +1,23 @@ """Util functions for tests""" +import os + import pytest from playwright.async_api import async_playwright @pytest.fixture -def url(): - return "http://127.0.0.1" +def proxy_url(): + if "INVARIANT_PROXY_API_URL" in os.environ: + return os.environ["INVARIANT_PROXY_API_URL"] + raise ValueError("Please set the INVARIANT_PROXY_API_URL environment variable") + + +@pytest.fixture +def explorer_api_url(): + if "INVARIANT_API_URL" in os.environ: + return os.environ["INVARIANT_API_URL"] + raise ValueError("Please set the INVARIANT_API_URL environment variable") @pytest.fixture