Add unit tests and move the current integration tests into a separate directory.

.github/workflows/tests_ci.yml

@@ -15,10 +15,25 @@ jobs:
     runs-on: ubuntu-latest
     steps:
         - name: Checkout
-          uses: actions/checkout@v1
-        - name: Run tests
+          uses: actions/checkout@v4
+
+        - name: Set Up Python
+          uses: actions/setup-python@v4
+          with:
+            python-version: "3.10"
+
+        - name: Install Dependencies
+          run: |
+            python -m pip install --upgrade pip
+            pip install pytest
+
+        - name: Run unit tests
+          run: ./run.sh unit-tests -s -vv
+          continue-on-error: true
+
+        - name: Run integration tests
           env:
             OPENAI_API_KEY: ${{ secrets.INVARIANT_TESTING_OPENAI_KEY }}
             ANTHROPIC_API_KEY: ${{ secrets.INVARIANT_TESTING_ANTHROPIC_KEY}}
             GEMINI_API_KEY: ${{ secrets.INVARIANT_TESTING_GEMINI_KEY }}
-          run: ./run.sh tests -s -vv
+          run: ./run.sh integration-tests -s -vv

.gitignore

@@ -4,6 +4,7 @@ __pycache__/
 .py[oc]
 data/
 tests/results/
+tests/integration/results/
 
 # Coverage and build artifacts
 .coverage

gateway/converters/anthropic_to_invariant.py

@@ -1,6 +1,5 @@
 """Converts the request and response formats from Anthropic to Invariant API format."""
 
-
 def convert_anthropic_to_invariant_message_format(
     messages: list[dict], keep_empty_tool_response: bool = False
 ) -> list[dict]:

gateway/converters/gemini_to_invariant.py

@@ -1,8 +1,5 @@
 """Converts the request and response formats from Gemini to Invariant API format."""
 
-import base64
-
-
 def convert_request(request: dict) -> list[dict]:
     """Converts the request from Gemini API to Invariant API format."""
     openai_messages = []

run.sh

@@ -45,11 +45,16 @@ build() {
 down() {
   # Bring down local services
   docker compose -f docker-compose.local.yml down
-  docker compose -f tests/docker-compose.test.yml down
+  docker compose -f tests/integration/docker-compose.test.yml down
 }
 
+unit_tests() {
+  echo "Running unit tests..."
+  
+  pytest tests/unit_tests $@
+}
 
-tests() {
+integration_tests() {
   echo "Setting up test environment to run integration tests..."
 
   # Ensure test network exists
@@ -70,9 +75,9 @@ tests() {
   echo "File successfully downloaded: $FILE"
 
   # Start containers
-  docker compose -f tests/docker-compose.test.yml down
-  docker compose -f tests/docker-compose.test.yml build
-  docker compose -f tests/docker-compose.test.yml up -d
+  GATEWAY_PATH=$(pwd)/gateway docker compose -f tests/integration/docker-compose.test.yml down
+  GATEWAY_PATH=$(pwd)/gateway docker compose -f tests/integration/docker-compose.test.yml build
+  GATEWAY_PATH=$(pwd)/gateway docker compose -f tests/integration/docker-compose.test.yml up -d
 
   until [ "$(docker inspect -f '{{.State.Health.Status}}' invariant-gateway-test-explorer-app-api)" = "healthy" ]; do
     echo "Explorer backend app-api instance container starting..."
@@ -100,15 +105,15 @@ tests() {
   # Make call to signup endpoint
   curl -k -X POST http://127.0.0.1/api/v1/user/signup
 
-  docker build -t 'invariant-gateway-tests' -f ./tests/Dockerfile.test ./tests
+  docker build -t 'invariant-gateway-tests' -f ./tests/integration/Dockerfile.test ./tests
 
   docker run \
-    --mount type=bind,source=./tests,target=/tests \
+    --mount type=bind,source=./tests/integration,target=/tests \
     --network invariant-gateway-web-test \
     -e OPENAI_API_KEY="$OPENAI_API_KEY" \
     -e ANTHROPIC_API_KEY="$ANTHROPIC_API_KEY"\
     -e GEMINI_API_KEY="$GEMINI_API_KEY" \
-    --env-file ./tests/.env.test \
+    --env-file ./tests/integration/.env.test \
     invariant-gateway-tests $@
 }
 
@@ -129,12 +134,16 @@ case "$1" in
   "logs")
     docker compose -f docker-compose.local.yml logs -f
     ;;
-  "tests")
+  "unit-tests")
     shift
-    tests $@
+    unit_tests $@
+    ;;
+  "integration-tests")
+    shift
+    integration_tests $@
     ;;
   *)
-    echo "Usage: $0 [up|build|down|logs|tests]"
+    echo "Usage: $0 [up|build|down|logs|unit-tests|integration-tests]"
     exit 1
     ;;
 esac

tests/integration/Dockerfile.test

@@ -1,7 +1,7 @@
 FROM mcr.microsoft.com/playwright/python:v1.50.0-noble
 
 RUN mkdir -p /tests
-COPY ./requirements.txt /tests/requirements.txt
+COPY ./integration/requirements.txt /tests/requirements.txt
 WORKDIR /tests
 RUN pip install --upgrade pip
 RUN pip install --no-cache-dir -r requirements.txt

tests/integration/anthropic/test_anthropic_header_with_invariant_key.py

@@ -6,7 +6,7 @@ import sys
 import time
 from unittest.mock import patch
 
-# Add tests folder (parent) to sys.path
+# Add integration folder (parent) to sys.path
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import anthropic

tests/integration/anthropic/test_anthropic_with_tool_call.py

@@ -9,7 +9,7 @@ import time
 from pathlib import Path
 from typing import Dict, List
 
-# Add tests folder (parent) to sys.path
+# Add integration folder (parent) to sys.path
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import anthropic
@@ -311,7 +311,7 @@ async def test_response_with_tool_call_with_image(
     """Test the chat completion with image for the weather agent."""
     weather_agent = WeatherAgent(gateway_url, push_to_explorer)
 
-    image_path = Path(__file__).parent.parent / "images" / "new-york.jpeg"
+    image_path = Path(__file__).parent.parent / "resources" / "images" / "new-york.jpeg"
 
     with image_path.open("rb") as image_file:
         base64_image = base64.b64encode(image_file.read()).decode("utf-8")

tests/integration/anthropic/test_anthropic_without_tool_call.py

@@ -5,7 +5,7 @@ import os
 import sys
 import time
 
-# Add tests folder (parent) to sys.path
+# Add integration folder (parent) to sys.path
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import anthropic

tests/integration/docker-compose.test.yml

@@ -24,8 +24,8 @@ services:
   invariant-gateway:
     container_name: invariant-gateway-test
     build:
-      context: ../gateway
-      dockerfile: ../gateway/Dockerfile.gateway
+      context: ${GATEWAY_PATH}
+      dockerfile: ${GATEWAY_PATH}/Dockerfile.gateway
     depends_on:
       app-api:
         condition: service_healthy
@@ -36,7 +36,7 @@ services:
       - DEV_MODE=true
     volumes:
       - type: bind
-        source: ../gateway
+        source: ${GATEWAY_PATH}
         target: /srv/gateway
     networks:
       - invariant-gateway-web-test

tests/integration/gemini/test_generate_content_with_tool_calls.py

@@ -5,7 +5,7 @@ import sys
 import time
 import uuid
 
-# Add tests folder (parent) to sys.path
+# Add integration folder (parent) to sys.path
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import pytest

tests/integration/gemini/test_generate_content_without_tool_calls.py

@@ -7,7 +7,7 @@ import uuid
 from pathlib import Path
 from unittest.mock import patch
 
-# Add tests folder (parent) to sys.path
+# Add integration folder (parent) to sys.path
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import pytest
@@ -128,7 +128,7 @@ async def test_generate_content_with_image(
         },
     )
 
-    image_path = Path(__file__).parent.parent / "images" / "two-cats.png"
+    image_path = Path(__file__).parent.parent / "resources" / "images" / "two-cats.png"
     image = PIL.Image.open(image_path)
 
     chat_response = client.models.generate_content(

tests/integration/open_ai/test_chat_with_tool_call.py

@@ -6,7 +6,7 @@ import sys
 import time
 import uuid
 
-# Add tests folder (parent) to sys.path
+# Add integration folder (parent) to sys.path
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import pytest

tests/integration/open_ai/test_chat_without_tool_calls.py

@@ -8,7 +8,7 @@ import uuid
 from pathlib import Path
 from unittest.mock import patch
 
-# Add tests folder (parent) to sys.path
+# Add integration folder (parent) to sys.path
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import pytest
@@ -113,7 +113,7 @@ async def test_chat_completion_with_image(
         if push_to_explorer
         else f"{gateway_url}/api/v1/gateway/openai",
     )
-    image_path = Path(__file__).parent.parent / "images" / "two-cats.png"
+    image_path = Path(__file__).parent.parent / "resources" / "images" / "two-cats.png"
     with image_path.open("rb") as image_file:
         base64_image = base64.b64encode(image_file.read()).decode("utf-8")
 

tests/unit_tests/converters/test_gemini_to_invariant.py

@@ -0,0 +1,211 @@
+"""Tests for the gemini_to_invariant converters."""
+
+import os
+import sys
+
+# Add root folder (parent) to sys.path
+sys.path.append(
+    os.path.dirname(
+        os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+    )
+)
+
+from gateway.converters.gemini_to_invariant import convert_request, convert_response
+
+
+def test_convert_request_with_tool_call():
+    """Test the convert_request function."""
+    gemini_request = {
+        "contents": [
+            {
+                "parts": [{"text": "Turn the lights down to a romantic level"}],
+                "role": "user",
+            },
+            {
+                "parts": [
+                    {
+                        "functionCall": {
+                            "args": {"color_temp": "warm", "brightness": 20},
+                            "name": "set_light_values",
+                        }
+                    }
+                ],
+                "role": "model",
+            },
+            {
+                "parts": [
+                    {
+                        "functionResponse": {
+                            "name": "set_light_values",
+                            "response": {
+                                "result": {"brightness": 20, "colorTemperature": "warm"}
+                            },
+                        }
+                    }
+                ],
+                "role": "user",
+            },
+        ],
+        "systemInstruction": {
+            "parts": [{"text": "This the system instruction. Use the function call."}],
+            "role": "user",
+        },
+        "tools": [
+            {
+                "functionDeclarations": [
+                    {
+                        "description": "Set the brightness and color temperature of a room light. (mock API).\\n\\n    Args:\\n        brightness: Light level from 0 to 100. Zero is off and 100 is full brightness\\n        color_temp: Color temperature of the light fixture, which can be `daylight`, `cool` or `warm`.\\n\\n    Returns:\\n        A dictionary containing the set brightness and color temperature.\\n    ",
+                        "name": "set_light_values",
+                        "parameters": {
+                            "type": "OBJECT",
+                            "properties": {
+                                "brightness": {"type": "INTEGER"},
+                                "color_temp": {"type": "STRING"},
+                            },
+                        },
+                    }
+                ]
+            }
+        ],
+        "generationConfig": {},
+    }
+
+    assert convert_request(gemini_request) == [
+        {
+            "role": "system",
+            "content": "This the system instruction. Use the function call.",
+        },
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "Turn the lights down to a romantic level"}
+            ],
+        },
+        {
+            "role": "assistant",
+            "tool_calls": [
+                {
+                    "type": "function",
+                    "function": {
+                        "name": "set_light_values",
+                        "arguments": {"color_temp": "warm", "brightness": 20},
+                    },
+                }
+            ],
+        },
+        {
+            "role": "tool",
+            "tool_name": "set_light_values",
+            "content": {"brightness": 20, "colorTemperature": "warm"},
+        },
+    ]
+
+
+def test_convert_response_with_tool_call():
+    """Test the convert_response function."""
+    gemini_response = {
+        "candidates": [
+            {
+                "content": {
+                    "parts": [
+                        {
+                            "video_metadata": None,
+                            "thought": None,
+                            "code_execution_result": None,
+                            "executable_code": None,
+                            "file_data": None,
+                            "function_call": None,
+                            "function_response": None,
+                            "inline_data": None,
+                            "text": "OK. I've set the lights to a brightness of 20 and a warm color temperature, creating a romantic ambiance.\n",
+                        }
+                    ],
+                    "role": "model",
+                },
+                "citation_metadata": None,
+                "finish_message": None,
+                "token_count": None,
+                "avg_logprobs": -0.12482070039819788,
+                "finish_reason": "STOP",
+                "grounding_metadata": None,
+                "index": None,
+                "logprobs_result": None,
+                "safety_ratings": None,
+            }
+        ],
+        "model_version": "gemini-2.0-flash",
+        "prompt_feedback": None,
+        "usage_metadata": {
+            "cached_content_token_count": None,
+            "candidates_token_count": 27,
+            "prompt_token_count": 136,
+            "total_token_count": 163,
+        },
+        "automatic_function_calling_history": [
+            {
+                "parts": [
+                    {
+                        "video_metadata": None,
+                        "thought": None,
+                        "code_execution_result": None,
+                        "executable_code": None,
+                        "file_data": None,
+                        "function_call": None,
+                        "function_response": None,
+                        "inline_data": None,
+                        "text": "Turn the lights down to a romantic level",
+                    }
+                ],
+                "role": "user",
+            },
+            {
+                "parts": [
+                    {
+                        "video_metadata": None,
+                        "thought": None,
+                        "code_execution_result": None,
+                        "executable_code": None,
+                        "file_data": None,
+                        "function_call": {
+                            "id": None,
+                            "args": {"brightness": 20, "color_temp": "warm"},
+                            "name": "set_light_values",
+                        },
+                        "function_response": None,
+                        "inline_data": None,
+                        "text": None,
+                    }
+                ],
+                "role": "model",
+            },
+            {
+                "parts": [
+                    {
+                        "video_metadata": None,
+                        "thought": None,
+                        "code_execution_result": None,
+                        "executable_code": None,
+                        "file_data": None,
+                        "function_call": None,
+                        "function_response": {
+                            "id": None,
+                            "name": "set_light_values",
+                            "response": {
+                                "result": {"brightness": 20, "colorTemperature": "warm"}
+                            },
+                        },
+                        "inline_data": None,
+                        "text": None,
+                    }
+                ],
+                "role": "user",
+            },
+        ],
+        "parsed": None,
+    }
+    assert convert_response(gemini_response) == [
+        {
+            "role": "assistant",
+            "content": "OK. I've set the lights to a brightness of 20 and a warm color temperature, creating a romantic ambiance.\n",
+        }
+    ]