From 6558f2360474352586032299ed2698f988d39848 Mon Sep 17 00:00:00 2001 From: Hemang Date: Wed, 12 Mar 2025 22:43:12 +0100 Subject: [PATCH] Add unit tests and move the current integration tests into a separate directory. --- .github/workflows/tests_ci.yml | 21 +- .gitignore | 1 + gateway/converters/anthropic_to_invariant.py | 1 - gateway/converters/gemini_to_invariant.py | 3 - run.sh | 31 ++- tests/{ => integration}/.env.test | 0 tests/{ => integration}/Dockerfile.test | 2 +- ...est_anthropic_header_with_invariant_key.py | 2 +- .../test_anthropic_with_tool_call.py | 4 +- .../test_anthropic_without_tool_call.py | 2 +- .../{ => integration}/docker-compose.test.yml | 6 +- .../test_generate_content_with_tool_calls.py | 2 +- ...est_generate_content_without_tool_calls.py | 4 +- .../open_ai/test_chat_with_tool_call.py | 2 +- .../open_ai/test_chat_without_tool_calls.py | 4 +- tests/{ => integration}/pytest.ini | 0 tests/{ => integration}/requirements.txt | 0 .../resources}/images/new-york.jpeg | Bin .../resources}/images/two-cats.png | Bin tests/{ => integration}/util.py | 0 .../converters/test_gemini_to_invariant.py | 211 ++++++++++++++++++ 21 files changed, 264 insertions(+), 32 deletions(-) rename tests/{ => integration}/.env.test (100%) rename tests/{ => integration}/Dockerfile.test (83%) rename tests/{ => integration}/anthropic/test_anthropic_header_with_invariant_key.py (98%) rename tests/{ => integration}/anthropic/test_anthropic_with_tool_call.py (99%) rename tests/{ => integration}/anthropic/test_anthropic_without_tool_call.py (99%) rename tests/{ => integration}/docker-compose.test.yml (96%) rename tests/{ => integration}/gemini/test_generate_content_with_tool_calls.py (99%) rename tests/{ => integration}/gemini/test_generate_content_without_tool_calls.py (98%) rename tests/{ => integration}/open_ai/test_chat_with_tool_call.py (99%) rename tests/{ => integration}/open_ai/test_chat_without_tool_calls.py (98%) rename tests/{ => integration}/pytest.ini (100%) rename tests/{ => integration}/requirements.txt (100%) rename tests/{ => integration/resources}/images/new-york.jpeg (100%) rename tests/{ => integration/resources}/images/two-cats.png (100%) rename tests/{ => integration}/util.py (100%) create mode 100644 tests/unit_tests/converters/test_gemini_to_invariant.py diff --git a/.github/workflows/tests_ci.yml b/.github/workflows/tests_ci.yml index ee09d7c..b81a8fb 100644 --- a/.github/workflows/tests_ci.yml +++ b/.github/workflows/tests_ci.yml @@ -15,10 +15,25 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v1 - - name: Run tests + uses: actions/checkout@v4 + + - name: Set Up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install pytest + + - name: Run unit tests + run: ./run.sh unit-tests -s -vv + continue-on-error: true + + - name: Run integration tests env: OPENAI_API_KEY: ${{ secrets.INVARIANT_TESTING_OPENAI_KEY }} ANTHROPIC_API_KEY: ${{ secrets.INVARIANT_TESTING_ANTHROPIC_KEY}} GEMINI_API_KEY: ${{ secrets.INVARIANT_TESTING_GEMINI_KEY }} - run: ./run.sh tests -s -vv + run: ./run.sh integration-tests -s -vv diff --git a/.gitignore b/.gitignore index c62d361..a0fcfc5 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ __pycache__/ .py[oc] data/ tests/results/ +tests/integration/results/ # Coverage and build artifacts .coverage diff --git a/gateway/converters/anthropic_to_invariant.py b/gateway/converters/anthropic_to_invariant.py index 09263c9..4f6274e 100644 --- a/gateway/converters/anthropic_to_invariant.py +++ b/gateway/converters/anthropic_to_invariant.py @@ -1,6 +1,5 @@ """Converts the request and response formats from Anthropic to Invariant API format.""" - def convert_anthropic_to_invariant_message_format( messages: list[dict], keep_empty_tool_response: bool = False ) -> list[dict]: diff --git a/gateway/converters/gemini_to_invariant.py b/gateway/converters/gemini_to_invariant.py index b7d8f12..74960da 100644 --- a/gateway/converters/gemini_to_invariant.py +++ b/gateway/converters/gemini_to_invariant.py @@ -1,8 +1,5 @@ """Converts the request and response formats from Gemini to Invariant API format.""" -import base64 - - def convert_request(request: dict) -> list[dict]: """Converts the request from Gemini API to Invariant API format.""" openai_messages = [] diff --git a/run.sh b/run.sh index 149efd3..074f843 100755 --- a/run.sh +++ b/run.sh @@ -45,11 +45,16 @@ build() { down() { # Bring down local services docker compose -f docker-compose.local.yml down - docker compose -f tests/docker-compose.test.yml down + docker compose -f tests/integration/docker-compose.test.yml down } +unit_tests() { + echo "Running unit tests..." + + pytest tests/unit_tests $@ +} -tests() { +integration_tests() { echo "Setting up test environment to run integration tests..." # Ensure test network exists @@ -70,9 +75,9 @@ tests() { echo "File successfully downloaded: $FILE" # Start containers - docker compose -f tests/docker-compose.test.yml down - docker compose -f tests/docker-compose.test.yml build - docker compose -f tests/docker-compose.test.yml up -d + GATEWAY_PATH=$(pwd)/gateway docker compose -f tests/integration/docker-compose.test.yml down + GATEWAY_PATH=$(pwd)/gateway docker compose -f tests/integration/docker-compose.test.yml build + GATEWAY_PATH=$(pwd)/gateway docker compose -f tests/integration/docker-compose.test.yml up -d until [ "$(docker inspect -f '{{.State.Health.Status}}' invariant-gateway-test-explorer-app-api)" = "healthy" ]; do echo "Explorer backend app-api instance container starting..." @@ -100,15 +105,15 @@ tests() { # Make call to signup endpoint curl -k -X POST http://127.0.0.1/api/v1/user/signup - docker build -t 'invariant-gateway-tests' -f ./tests/Dockerfile.test ./tests + docker build -t 'invariant-gateway-tests' -f ./tests/integration/Dockerfile.test ./tests docker run \ - --mount type=bind,source=./tests,target=/tests \ + --mount type=bind,source=./tests/integration,target=/tests \ --network invariant-gateway-web-test \ -e OPENAI_API_KEY="$OPENAI_API_KEY" \ -e ANTHROPIC_API_KEY="$ANTHROPIC_API_KEY"\ -e GEMINI_API_KEY="$GEMINI_API_KEY" \ - --env-file ./tests/.env.test \ + --env-file ./tests/integration/.env.test \ invariant-gateway-tests $@ } @@ -129,12 +134,16 @@ case "$1" in "logs") docker compose -f docker-compose.local.yml logs -f ;; - "tests") + "unit-tests") shift - tests $@ + unit_tests $@ + ;; + "integration-tests") + shift + integration_tests $@ ;; *) - echo "Usage: $0 [up|build|down|logs|tests]" + echo "Usage: $0 [up|build|down|logs|unit-tests|integration-tests]" exit 1 ;; esac \ No newline at end of file diff --git a/tests/.env.test b/tests/integration/.env.test similarity index 100% rename from tests/.env.test rename to tests/integration/.env.test diff --git a/tests/Dockerfile.test b/tests/integration/Dockerfile.test similarity index 83% rename from tests/Dockerfile.test rename to tests/integration/Dockerfile.test index 54514e0..5a001e4 100644 --- a/tests/Dockerfile.test +++ b/tests/integration/Dockerfile.test @@ -1,7 +1,7 @@ FROM mcr.microsoft.com/playwright/python:v1.50.0-noble RUN mkdir -p /tests -COPY ./requirements.txt /tests/requirements.txt +COPY ./integration/requirements.txt /tests/requirements.txt WORKDIR /tests RUN pip install --upgrade pip RUN pip install --no-cache-dir -r requirements.txt diff --git a/tests/anthropic/test_anthropic_header_with_invariant_key.py b/tests/integration/anthropic/test_anthropic_header_with_invariant_key.py similarity index 98% rename from tests/anthropic/test_anthropic_header_with_invariant_key.py rename to tests/integration/anthropic/test_anthropic_header_with_invariant_key.py index 1b95900..ea10d7f 100644 --- a/tests/anthropic/test_anthropic_header_with_invariant_key.py +++ b/tests/integration/anthropic/test_anthropic_header_with_invariant_key.py @@ -6,7 +6,7 @@ import sys import time from unittest.mock import patch -# Add tests folder (parent) to sys.path +# Add integration folder (parent) to sys.path sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import anthropic diff --git a/tests/anthropic/test_anthropic_with_tool_call.py b/tests/integration/anthropic/test_anthropic_with_tool_call.py similarity index 99% rename from tests/anthropic/test_anthropic_with_tool_call.py rename to tests/integration/anthropic/test_anthropic_with_tool_call.py index 8e063d8..59f6d1d 100644 --- a/tests/anthropic/test_anthropic_with_tool_call.py +++ b/tests/integration/anthropic/test_anthropic_with_tool_call.py @@ -9,7 +9,7 @@ import time from pathlib import Path from typing import Dict, List -# Add tests folder (parent) to sys.path +# Add integration folder (parent) to sys.path sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import anthropic @@ -311,7 +311,7 @@ async def test_response_with_tool_call_with_image( """Test the chat completion with image for the weather agent.""" weather_agent = WeatherAgent(gateway_url, push_to_explorer) - image_path = Path(__file__).parent.parent / "images" / "new-york.jpeg" + image_path = Path(__file__).parent.parent / "resources" / "images" / "new-york.jpeg" with image_path.open("rb") as image_file: base64_image = base64.b64encode(image_file.read()).decode("utf-8") diff --git a/tests/anthropic/test_anthropic_without_tool_call.py b/tests/integration/anthropic/test_anthropic_without_tool_call.py similarity index 99% rename from tests/anthropic/test_anthropic_without_tool_call.py rename to tests/integration/anthropic/test_anthropic_without_tool_call.py index 8baf955..e759adf 100644 --- a/tests/anthropic/test_anthropic_without_tool_call.py +++ b/tests/integration/anthropic/test_anthropic_without_tool_call.py @@ -5,7 +5,7 @@ import os import sys import time -# Add tests folder (parent) to sys.path +# Add integration folder (parent) to sys.path sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import anthropic diff --git a/tests/docker-compose.test.yml b/tests/integration/docker-compose.test.yml similarity index 96% rename from tests/docker-compose.test.yml rename to tests/integration/docker-compose.test.yml index e1c1275..5d58b20 100644 --- a/tests/docker-compose.test.yml +++ b/tests/integration/docker-compose.test.yml @@ -24,8 +24,8 @@ services: invariant-gateway: container_name: invariant-gateway-test build: - context: ../gateway - dockerfile: ../gateway/Dockerfile.gateway + context: ${GATEWAY_PATH} + dockerfile: ${GATEWAY_PATH}/Dockerfile.gateway depends_on: app-api: condition: service_healthy @@ -36,7 +36,7 @@ services: - DEV_MODE=true volumes: - type: bind - source: ../gateway + source: ${GATEWAY_PATH} target: /srv/gateway networks: - invariant-gateway-web-test diff --git a/tests/gemini/test_generate_content_with_tool_calls.py b/tests/integration/gemini/test_generate_content_with_tool_calls.py similarity index 99% rename from tests/gemini/test_generate_content_with_tool_calls.py rename to tests/integration/gemini/test_generate_content_with_tool_calls.py index 9700376..677eea2 100644 --- a/tests/gemini/test_generate_content_with_tool_calls.py +++ b/tests/integration/gemini/test_generate_content_with_tool_calls.py @@ -5,7 +5,7 @@ import sys import time import uuid -# Add tests folder (parent) to sys.path +# Add integration folder (parent) to sys.path sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import pytest diff --git a/tests/gemini/test_generate_content_without_tool_calls.py b/tests/integration/gemini/test_generate_content_without_tool_calls.py similarity index 98% rename from tests/gemini/test_generate_content_without_tool_calls.py rename to tests/integration/gemini/test_generate_content_without_tool_calls.py index cd94e7c..331bec3 100644 --- a/tests/gemini/test_generate_content_without_tool_calls.py +++ b/tests/integration/gemini/test_generate_content_without_tool_calls.py @@ -7,7 +7,7 @@ import uuid from pathlib import Path from unittest.mock import patch -# Add tests folder (parent) to sys.path +# Add integration folder (parent) to sys.path sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import pytest @@ -128,7 +128,7 @@ async def test_generate_content_with_image( }, ) - image_path = Path(__file__).parent.parent / "images" / "two-cats.png" + image_path = Path(__file__).parent.parent / "resources" / "images" / "two-cats.png" image = PIL.Image.open(image_path) chat_response = client.models.generate_content( diff --git a/tests/open_ai/test_chat_with_tool_call.py b/tests/integration/open_ai/test_chat_with_tool_call.py similarity index 99% rename from tests/open_ai/test_chat_with_tool_call.py rename to tests/integration/open_ai/test_chat_with_tool_call.py index ed5ffee..7813e1d 100644 --- a/tests/open_ai/test_chat_with_tool_call.py +++ b/tests/integration/open_ai/test_chat_with_tool_call.py @@ -6,7 +6,7 @@ import sys import time import uuid -# Add tests folder (parent) to sys.path +# Add integration folder (parent) to sys.path sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import pytest diff --git a/tests/open_ai/test_chat_without_tool_calls.py b/tests/integration/open_ai/test_chat_without_tool_calls.py similarity index 98% rename from tests/open_ai/test_chat_without_tool_calls.py rename to tests/integration/open_ai/test_chat_without_tool_calls.py index 34e9daa..d6cbf4f 100644 --- a/tests/open_ai/test_chat_without_tool_calls.py +++ b/tests/integration/open_ai/test_chat_without_tool_calls.py @@ -8,7 +8,7 @@ import uuid from pathlib import Path from unittest.mock import patch -# Add tests folder (parent) to sys.path +# Add integration folder (parent) to sys.path sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import pytest @@ -113,7 +113,7 @@ async def test_chat_completion_with_image( if push_to_explorer else f"{gateway_url}/api/v1/gateway/openai", ) - image_path = Path(__file__).parent.parent / "images" / "two-cats.png" + image_path = Path(__file__).parent.parent / "resources" / "images" / "two-cats.png" with image_path.open("rb") as image_file: base64_image = base64.b64encode(image_file.read()).decode("utf-8") diff --git a/tests/pytest.ini b/tests/integration/pytest.ini similarity index 100% rename from tests/pytest.ini rename to tests/integration/pytest.ini diff --git a/tests/requirements.txt b/tests/integration/requirements.txt similarity index 100% rename from tests/requirements.txt rename to tests/integration/requirements.txt diff --git a/tests/images/new-york.jpeg b/tests/integration/resources/images/new-york.jpeg similarity index 100% rename from tests/images/new-york.jpeg rename to tests/integration/resources/images/new-york.jpeg diff --git a/tests/images/two-cats.png b/tests/integration/resources/images/two-cats.png similarity index 100% rename from tests/images/two-cats.png rename to tests/integration/resources/images/two-cats.png diff --git a/tests/util.py b/tests/integration/util.py similarity index 100% rename from tests/util.py rename to tests/integration/util.py diff --git a/tests/unit_tests/converters/test_gemini_to_invariant.py b/tests/unit_tests/converters/test_gemini_to_invariant.py new file mode 100644 index 0000000..8e8008f --- /dev/null +++ b/tests/unit_tests/converters/test_gemini_to_invariant.py @@ -0,0 +1,211 @@ +"""Tests for the gemini_to_invariant converters.""" + +import os +import sys + +# Add root folder (parent) to sys.path +sys.path.append( + os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + ) +) + +from gateway.converters.gemini_to_invariant import convert_request, convert_response + + +def test_convert_request_with_tool_call(): + """Test the convert_request function.""" + gemini_request = { + "contents": [ + { + "parts": [{"text": "Turn the lights down to a romantic level"}], + "role": "user", + }, + { + "parts": [ + { + "functionCall": { + "args": {"color_temp": "warm", "brightness": 20}, + "name": "set_light_values", + } + } + ], + "role": "model", + }, + { + "parts": [ + { + "functionResponse": { + "name": "set_light_values", + "response": { + "result": {"brightness": 20, "colorTemperature": "warm"} + }, + } + } + ], + "role": "user", + }, + ], + "systemInstruction": { + "parts": [{"text": "This the system instruction. Use the function call."}], + "role": "user", + }, + "tools": [ + { + "functionDeclarations": [ + { + "description": "Set the brightness and color temperature of a room light. (mock API).\\n\\n Args:\\n brightness: Light level from 0 to 100. Zero is off and 100 is full brightness\\n color_temp: Color temperature of the light fixture, which can be `daylight`, `cool` or `warm`.\\n\\n Returns:\\n A dictionary containing the set brightness and color temperature.\\n ", + "name": "set_light_values", + "parameters": { + "type": "OBJECT", + "properties": { + "brightness": {"type": "INTEGER"}, + "color_temp": {"type": "STRING"}, + }, + }, + } + ] + } + ], + "generationConfig": {}, + } + + assert convert_request(gemini_request) == [ + { + "role": "system", + "content": "This the system instruction. Use the function call.", + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "Turn the lights down to a romantic level"} + ], + }, + { + "role": "assistant", + "tool_calls": [ + { + "type": "function", + "function": { + "name": "set_light_values", + "arguments": {"color_temp": "warm", "brightness": 20}, + }, + } + ], + }, + { + "role": "tool", + "tool_name": "set_light_values", + "content": {"brightness": 20, "colorTemperature": "warm"}, + }, + ] + + +def test_convert_response_with_tool_call(): + """Test the convert_response function.""" + gemini_response = { + "candidates": [ + { + "content": { + "parts": [ + { + "video_metadata": None, + "thought": None, + "code_execution_result": None, + "executable_code": None, + "file_data": None, + "function_call": None, + "function_response": None, + "inline_data": None, + "text": "OK. I've set the lights to a brightness of 20 and a warm color temperature, creating a romantic ambiance.\n", + } + ], + "role": "model", + }, + "citation_metadata": None, + "finish_message": None, + "token_count": None, + "avg_logprobs": -0.12482070039819788, + "finish_reason": "STOP", + "grounding_metadata": None, + "index": None, + "logprobs_result": None, + "safety_ratings": None, + } + ], + "model_version": "gemini-2.0-flash", + "prompt_feedback": None, + "usage_metadata": { + "cached_content_token_count": None, + "candidates_token_count": 27, + "prompt_token_count": 136, + "total_token_count": 163, + }, + "automatic_function_calling_history": [ + { + "parts": [ + { + "video_metadata": None, + "thought": None, + "code_execution_result": None, + "executable_code": None, + "file_data": None, + "function_call": None, + "function_response": None, + "inline_data": None, + "text": "Turn the lights down to a romantic level", + } + ], + "role": "user", + }, + { + "parts": [ + { + "video_metadata": None, + "thought": None, + "code_execution_result": None, + "executable_code": None, + "file_data": None, + "function_call": { + "id": None, + "args": {"brightness": 20, "color_temp": "warm"}, + "name": "set_light_values", + }, + "function_response": None, + "inline_data": None, + "text": None, + } + ], + "role": "model", + }, + { + "parts": [ + { + "video_metadata": None, + "thought": None, + "code_execution_result": None, + "executable_code": None, + "file_data": None, + "function_call": None, + "function_response": { + "id": None, + "name": "set_light_values", + "response": { + "result": {"brightness": 20, "colorTemperature": "warm"} + }, + }, + "inline_data": None, + "text": None, + } + ], + "role": "user", + }, + ], + "parsed": None, + } + assert convert_response(gemini_response) == [ + { + "role": "assistant", + "content": "OK. I've set the lights to a brightness of 20 and a warm color temperature, creating a romantic ambiance.\n", + } + ]