From 35cb889f1a5244500905a0f0dac933304c7be54e Mon Sep 17 00:00:00 2001 From: Hemang Date: Thu, 27 Mar 2025 11:12:38 +0100 Subject: [PATCH] Update guardrails test after annotations ranges filtering. --- .github/workflows/tests_ci.yml | 2 +- run.sh | 13 +++++++++---- tests/integration/docker-compose.test.yml | 1 + .../guardrails/test_guardrails_anthropic.py | 18 +++++++----------- .../guardrails/test_guardrails_open_ai.py | 18 +++++++----------- 5 files changed, 25 insertions(+), 27 deletions(-) diff --git a/.github/workflows/tests_ci.yml b/.github/workflows/tests_ci.yml index 71dfe5e..83bd07a 100644 --- a/.github/workflows/tests_ci.yml +++ b/.github/workflows/tests_ci.yml @@ -36,5 +36,5 @@ jobs: OPENAI_API_KEY: ${{ secrets.INVARIANT_TESTING_OPENAI_KEY }} ANTHROPIC_API_KEY: ${{ secrets.INVARIANT_TESTING_ANTHROPIC_KEY}} GEMINI_API_KEY: ${{ secrets.INVARIANT_TESTING_GEMINI_KEY }} - GUARDRAILS_API_KEY: ${{ secrets.INVARIANT_TESTING_GUARDRAILS_KEY }} + INVARIANT_API_KEY: ${{ secrets.INVARIANT_TESTING_GUARDRAILS_KEY }} run: ./run.sh integration-tests -s -vv diff --git a/run.sh b/run.sh index 6447ede..f666f96 100755 --- a/run.sh +++ b/run.sh @@ -103,11 +103,13 @@ integration_tests() { fi fi + export GATEWAY_PATH=$(pwd)/gateway + export GUARDRAILS_FILE_PATH="$TEST_GUARDRAILS_FILE_PATH" # Start containers - GATEWAY_PATH=$(pwd)/gateway docker compose -f tests/integration/docker-compose.test.yml down - GATEWAY_PATH=$(pwd)/gateway docker compose -f tests/integration/docker-compose.test.yml build - GUARDRAILS_FILE_PATH="$TEST_GUARDRAILS_FILE_PATH" GATEWAY_PATH=$(pwd)/gateway docker compose -f tests/integration/docker-compose.test.yml up -d + docker compose -f tests/integration/docker-compose.test.yml down + docker compose -f tests/integration/docker-compose.test.yml build + docker compose -f tests/integration/docker-compose.test.yml up -d until [ "$(docker inspect -f '{{.State.Health.Status}}' invariant-gateway-test-explorer-app-api)" = "healthy" ]; do echo "Explorer backend app-api instance container starting..." @@ -140,9 +142,12 @@ integration_tests() { -e OPENAI_API_KEY="$OPENAI_API_KEY" \ -e ANTHROPIC_API_KEY="$ANTHROPIC_API_KEY"\ -e GEMINI_API_KEY="$GEMINI_API_KEY" \ - -e GUARDRAILS_API_KEY="$GUARDRAILS_API_KEY" \ + -e INVARIANT_API_KEY="$INVARIANT_API_KEY" \ --env-file ./tests/integration/.env.test \ invariant-gateway-tests $@ + + unset GATEWAY_PATH + unset GUARDRAILS_FILE_PATH } # ----------------------------- diff --git a/tests/integration/docker-compose.test.yml b/tests/integration/docker-compose.test.yml index 26ba1c6..52d481e 100644 --- a/tests/integration/docker-compose.test.yml +++ b/tests/integration/docker-compose.test.yml @@ -35,6 +35,7 @@ services: environment: - DEV_MODE=true - GUARDRAILS_FILE_PATH=${GUARDRAILS_FILE_PATH:+/srv/resources/guardrails.py} + - ${INVARIANT_API_KEY:+INVARIANT_API_KEY=${INVARIANT_API_KEY}} volumes: - type: bind source: ${GATEWAY_PATH} diff --git a/tests/integration/guardrails/test_guardrails_anthropic.py b/tests/integration/guardrails/test_guardrails_anthropic.py index 70bb140..45ba1fd 100644 --- a/tests/integration/guardrails/test_guardrails_anthropic.py +++ b/tests/integration/guardrails/test_guardrails_anthropic.py @@ -28,15 +28,15 @@ async def test_message_content_guardrail_from_file( explorer_api_url, gateway_url, do_stream, push_to_explorer ): """Test the message content guardrail.""" - if not os.getenv("GUARDRAILS_API_KEY"): - pytest.fail("No GUARDRAILS_API_KEY set, failing") + if not os.getenv("INVARIANT_API_KEY"): + pytest.fail("No INVARIANT_API_KEY set, failing") dataset_name = f"test-dataset-anthropic-{uuid.uuid4()}" client = Anthropic( http_client=Client( headers={ - "Invariant-Authorization": f"Bearer {os.getenv('GUARDRAILS_API_KEY')}" + "Invariant-Authorization": f"Bearer {os.getenv('INVARIANT_API_KEY')}" }, ), base_url=f"{gateway_url}/api/v1/gateway/{dataset_name}/anthropic" @@ -112,15 +112,11 @@ async def test_message_content_guardrail_from_file( ) annotations = annotations_response.json() - assert len(annotations) == 2 + assert len(annotations) == 1 assert ( annotations[0]["content"] == "Madrid detected in the response" and annotations[0]["extra_metadata"]["source"] == "guardrails-error" ) - assert ( - annotations[1]["content"] == "Madrid detected in the response" - and annotations[1]["extra_metadata"]["source"] == "guardrails-error" - ) @pytest.mark.skipif( @@ -134,8 +130,8 @@ async def test_tool_call_guardrail_from_file( explorer_api_url, gateway_url, do_stream, push_to_explorer ): """Test the message content guardrail.""" - if not os.getenv("GUARDRAILS_API_KEY"): - pytest.fail("No GUARDRAILS_API_KEY set, failing") + if not os.getenv("INVARIANT_API_KEY"): + pytest.fail("No INVARIANT_API_KEY set, failing") tools = [ { @@ -169,7 +165,7 @@ async def test_tool_call_guardrail_from_file( client = Anthropic( http_client=Client( headers={ - "Invariant-Authorization": f"Bearer {os.getenv('GUARDRAILS_API_KEY')}" + "Invariant-Authorization": f"Bearer {os.getenv('INVARIANT_API_KEY')}" }, ), base_url=f"{gateway_url}/api/v1/gateway/{dataset_name}/anthropic" diff --git a/tests/integration/guardrails/test_guardrails_open_ai.py b/tests/integration/guardrails/test_guardrails_open_ai.py index 2154abb..c013107 100644 --- a/tests/integration/guardrails/test_guardrails_open_ai.py +++ b/tests/integration/guardrails/test_guardrails_open_ai.py @@ -26,15 +26,15 @@ async def test_message_content_guardrail_from_file( explorer_api_url, gateway_url, do_stream, push_to_explorer ): """Test the message content guardrail.""" - if not os.getenv("GUARDRAILS_API_KEY"): - pytest.fail("No GUARDRAILS_API_KEY set, failing") + if not os.getenv("INVARIANT_API_KEY"): + pytest.fail("No INVARIANT_API_KEY set, failing") dataset_name = f"test-dataset-open-ai-{uuid.uuid4()}" client = OpenAI( http_client=Client( headers={ - "Invariant-Authorization": f"Bearer {os.getenv('GUARDRAILS_API_KEY')}" + "Invariant-Authorization": f"Bearer {os.getenv('INVARIANT_API_KEY')}" }, ), base_url=f"{gateway_url}/api/v1/gateway/{dataset_name}/openai" @@ -109,15 +109,11 @@ async def test_message_content_guardrail_from_file( ) annotations = annotations_response.json() - assert len(annotations) == 2 + assert len(annotations) == 1 assert ( annotations[0]["content"] == "Madrid detected in the response" and annotations[0]["extra_metadata"]["source"] == "guardrails-error" ) - assert ( - annotations[1]["content"] == "Madrid detected in the response" - and annotations[1]["extra_metadata"]["source"] == "guardrails-error" - ) @pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="No OPENAI_API_KEY set") @@ -129,8 +125,8 @@ async def test_tool_call_guardrail_from_file( explorer_api_url, gateway_url, do_stream, push_to_explorer ): """Test the message content guardrail.""" - if not os.getenv("GUARDRAILS_API_KEY"): - pytest.fail("No GUARDRAILS_API_KEY set, failing") + if not os.getenv("INVARIANT_API_KEY"): + pytest.fail("No INVARIANT_API_KEY set, failing") tools = [ { @@ -169,7 +165,7 @@ async def test_tool_call_guardrail_from_file( client = OpenAI( http_client=Client( headers={ - "Invariant-Authorization": f"Bearer {os.getenv('GUARDRAILS_API_KEY')}" + "Invariant-Authorization": f"Bearer {os.getenv('INVARIANT_API_KEY')}" }, ), base_url=f"{gateway_url}/api/v1/gateway/{dataset_name}/openai"