Add openai test for chat completions without streaming and without tool calls.

.gitignore

@@ -3,6 +3,7 @@ __pycache__/
 .pytest_cache/
 .py[oc]
 data/
+tests/results/
 
 # Coverage and build artifacts
 .coverage

run.sh

@@ -58,11 +58,16 @@ tests() {
 
   echo "app-api and proxy are available. Running tests..."
 
+  # Make call to signup endpoint
+  curl -k -X POST http://127.0.0.1/api/v1/user/signup
+
   docker build -t 'explorer-proxy-test' -f ./tests/Dockerfile.test ./tests
 
   docker run \
     --mount type=bind,source=./tests,target=/tests \
-    --network host \
+    --network invariant-proxy-web-test \
+    -e OPENAI_API_KEY="$OPENAI_API_KEY" \
+    --env-file ./tests/.env.test \
     explorer-proxy-test $@
 }
 

tests/.env.test

@@ -2,6 +2,7 @@
 # from the proxy. Both proxy and app-api are on the same network:
 # invariant-proxy-web-test
 INVARIANT_API_URL=http://explorer-proxy-test-app-api:8000
+INVARIANT_PROXY_API_URL=http://explorer-proxy-test:8000
 
 POSTGRES_USER=postgres
 POSTGRES_PASSWORD=postgres

tests/Dockerfile.test

@@ -3,6 +3,7 @@ FROM mcr.microsoft.com/playwright/python:v1.50.0-noble
 RUN mkdir -p /tests
 COPY ./requirements.txt /tests/requirements.txt
 WORKDIR /tests
+RUN pip install --upgrade pip
 RUN pip install --no-cache-dir -r requirements.txt
 
-ENTRYPOINT ["pytest", "-s", "-vv"]
+ENTRYPOINT ["pytest",  "--capture=tee-sys", "--tracing", "off", "--junit-xml=/tests/results/test-results-all.xml", "-s", "-vv"]

tests/docker-compose.test.yml

@@ -83,7 +83,7 @@ services:
       - "traefik.http.services.explorer-test-api.loadbalancer.server.port=8000"
       - "traefik.docker.network=invariant-proxy-web-test"
     healthcheck:
-      test: curl -X GET -I http://localhost:8000/api/v1 --fail
+      test: curl -X GET -I http://localhost:8000/api/v1/ --fail
       interval: 1s
       timeout: 5s
 

tests/open_ai/test_chat_without_tool_calls.py

@@ -1,9 +1,14 @@
 """Test the chat completions proxy calls without tool calling."""
 
 import os
+import sys
+import uuid
+
+import pytest
+from httpx import Client
 
 # add tests folder (parent) to sys.path
-import sys
+from openai import OpenAI
 
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
@@ -12,11 +17,47 @@ from util import *  # needed for pytest fixtures
 pytest_plugins = ("pytest_asyncio",)
 
 
-async def test_hello_world(context, url):
-    """Demo test"""
-    response = await context.request.get(
-        f"{url}/api/v1/dataset/byuser/developer/Welcome-to-Explorer"
+@pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="No OPENAI_API_KEY set")
+async def test_chat_completion_without_streaming(context, explorer_api_url, proxy_url):
+    """Test the chat completions proxy calls without tool calling."""
+    dataset_name = "test-dataset-open-ai-" + str(uuid.uuid4())
+
+    client = OpenAI(
+        http_client=Client(
+            headers={
+                "Invariant-Authorization": "Bearer <some-key>"
+            },  # This key is not used for local tests
+        ),
+        base_url=f"{proxy_url}/api/v1/proxy/{dataset_name}/openai",
     )
-    dataset = await response.json()
-    assert dataset["name"] == "Welcome-to-Explorer"
-    assert dataset["user"]["username"] == "developer"
+
+    chat_response = client.chat.completions.create(
+        model="gpt-4o",
+        messages=[{"role": "user", "content": "What is the capital of France?"}],
+    )
+
+    # Verify the chat response
+    assert "PARIS" in chat_response.choices[0].message.content.upper()
+
+    # Fetch the trace ids for the dataset
+    traces_response = await context.request.get(
+        f"{explorer_api_url}/api/v1/dataset/byuser/developer/{dataset_name}/traces"
+    )
+    traces = await traces_response.json()
+    assert len(traces) == 1
+    trace_id = traces[0]["id"]
+
+    # Fetch the trace
+    trace_response = await context.request.get(
+        f"{explorer_api_url}/api/v1/trace/{trace_id}"
+    )
+    trace = await trace_response.json()
+
+    # Verify the trace messages
+    assert len(trace["messages"]) == 2
+    assert trace["messages"][0] == {
+        "role": "user",
+        "content": "What is the capital of France?",
+    }
+    assert trace["messages"][1]["role"] == "assistant"
+    assert "PARIS" in trace["messages"][1]["content"].upper()

tests/requirements.txt

@@ -1,4 +1,6 @@
+anthropic
 openai
 pytest
 pytest-asyncio
-pytest-playwright
+pytest-playwright
+tavily-python

tests/util.py

@@ -1,12 +1,23 @@
 """Util functions for tests"""
 
+import os
+
 import pytest
 from playwright.async_api import async_playwright
 
 
 @pytest.fixture
-def url():
-    return "http://127.0.0.1"
+def proxy_url():
+    if "INVARIANT_PROXY_API_URL" in os.environ:
+        return os.environ["INVARIANT_PROXY_API_URL"]
+    raise ValueError("Please set the INVARIANT_PROXY_API_URL environment variable")
+
+
+@pytest.fixture
+def explorer_api_url():
+    if "INVARIANT_API_URL" in os.environ:
+        return os.environ["INVARIANT_API_URL"]
+    raise ValueError("Please set the INVARIANT_API_URL environment variable")
 
 
 @pytest.fixture