CalvinBackup/invariant-gateway
1
0
Fork 0
mirror of https://github.com/invariantlabs-ai/invariant-gateway.git synced 2026-07-08 03:47:48 +02:00
Code Issues Packages Projects Releases Wiki Activity

Update guardrails test after annotations ranges filtering.

Browse Source
This commit is contained in:
Hemang
2025-03-27 11:12:38 +01:00
committed by Hemang Sarkar
parent e55df73487
commit 35cb889f1a
5 changed files with 25 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/tests_ci.yml
+1 -1
View File
@@ -36,5 +36,5 @@ jobs:
OPENAI_API_KEY: ${{ secrets.INVARIANT_TESTING_OPENAI_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.INVARIANT_TESTING_ANTHROPIC_KEY}}
GEMINI_API_KEY: ${{ secrets.INVARIANT_TESTING_GEMINI_KEY }}
GUARDRAILS_API_KEY: ${{ secrets.INVARIANT_TESTING_GUARDRAILS_KEY }}
INVARIANT_API_KEY: ${{ secrets.INVARIANT_TESTING_GUARDRAILS_KEY }}
run: ./run.sh integration-tests -s -vv
run.sh
+9 -4
View File
@@ -103,11 +103,13 @@ integration_tests() {
fi
fi
export GATEWAY_PATH=$(pwd)/gateway
export GUARDRAILS_FILE_PATH="$TEST_GUARDRAILS_FILE_PATH"
# Start containers
GATEWAY_PATH=$(pwd)/gateway docker compose -f tests/integration/docker-compose.test.yml down
GATEWAY_PATH=$(pwd)/gateway docker compose -f tests/integration/docker-compose.test.yml build
GUARDRAILS_FILE_PATH="$TEST_GUARDRAILS_FILE_PATH" GATEWAY_PATH=$(pwd)/gateway docker compose -f tests/integration/docker-compose.test.yml up -d
docker compose -f tests/integration/docker-compose.test.yml down
docker compose -f tests/integration/docker-compose.test.yml build
docker compose -f tests/integration/docker-compose.test.yml up -d
until [ "$(docker inspect -f '{{.State.Health.Status}}' invariant-gateway-test-explorer-app-api)" = "healthy" ]; do
echo "Explorer backend app-api instance container starting..."
@@ -140,9 +142,12 @@ integration_tests() {
-e OPENAI_API_KEY="$OPENAI_API_KEY" \
-e ANTHROPIC_API_KEY="$ANTHROPIC_API_KEY"\
-e GEMINI_API_KEY="$GEMINI_API_KEY" \
-e GUARDRAILS_API_KEY="$GUARDRAILS_API_KEY" \
-e INVARIANT_API_KEY="$INVARIANT_API_KEY" \
--env-file ./tests/integration/.env.test \
invariant-gateway-tests $@
unset GATEWAY_PATH
unset GUARDRAILS_FILE_PATH
}
# -----------------------------
tests/integration/docker-compose.test.yml
+1
View File
@@ -35,6 +35,7 @@ services:
environment:
- DEV_MODE=true
- GUARDRAILS_FILE_PATH=${GUARDRAILS_FILE_PATH:+/srv/resources/guardrails.py}
- ${INVARIANT_API_KEY:+INVARIANT_API_KEY=${INVARIANT_API_KEY}}
volumes:
- type: bind
source: ${GATEWAY_PATH}
tests/integration/guardrails/test_guardrails_anthropic.py
+7 -11
View File
@@ -28,15 +28,15 @@ async def test_message_content_guardrail_from_file(
explorer_api_url, gateway_url, do_stream, push_to_explorer
):
"""Test the message content guardrail."""
if not os.getenv("GUARDRAILS_API_KEY"):
pytest.fail("No GUARDRAILS_API_KEY set, failing")
if not os.getenv("INVARIANT_API_KEY"):
pytest.fail("No INVARIANT_API_KEY set, failing")
dataset_name = f"test-dataset-anthropic-{uuid.uuid4()}"
client = Anthropic(
http_client=Client(
headers={
"Invariant-Authorization": f"Bearer {os.getenv('GUARDRAILS_API_KEY')}"
"Invariant-Authorization": f"Bearer {os.getenv('INVARIANT_API_KEY')}"
},
),
base_url=f"{gateway_url}/api/v1/gateway/{dataset_name}/anthropic"
@@ -112,15 +112,11 @@ async def test_message_content_guardrail_from_file(
)
annotations = annotations_response.json()
assert len(annotations) == 2
assert len(annotations) == 1
assert (
annotations[0]["content"] == "Madrid detected in the response"
and annotations[0]["extra_metadata"]["source"] == "guardrails-error"
)
assert (
annotations[1]["content"] == "Madrid detected in the response"
and annotations[1]["extra_metadata"]["source"] == "guardrails-error"
)
@pytest.mark.skipif(
@@ -134,8 +130,8 @@ async def test_tool_call_guardrail_from_file(
explorer_api_url, gateway_url, do_stream, push_to_explorer
):
"""Test the message content guardrail."""
if not os.getenv("GUARDRAILS_API_KEY"):
pytest.fail("No GUARDRAILS_API_KEY set, failing")
if not os.getenv("INVARIANT_API_KEY"):
pytest.fail("No INVARIANT_API_KEY set, failing")
tools = [
{
@@ -169,7 +165,7 @@ async def test_tool_call_guardrail_from_file(
client = Anthropic(
http_client=Client(
headers={
"Invariant-Authorization": f"Bearer {os.getenv('GUARDRAILS_API_KEY')}"
"Invariant-Authorization": f"Bearer {os.getenv('INVARIANT_API_KEY')}"
},
),
base_url=f"{gateway_url}/api/v1/gateway/{dataset_name}/anthropic"
tests/integration/guardrails/test_guardrails_open_ai.py
+7 -11
View File
@@ -26,15 +26,15 @@ async def test_message_content_guardrail_from_file(
explorer_api_url, gateway_url, do_stream, push_to_explorer
):
"""Test the message content guardrail."""
if not os.getenv("GUARDRAILS_API_KEY"):
pytest.fail("No GUARDRAILS_API_KEY set, failing")
if not os.getenv("INVARIANT_API_KEY"):
pytest.fail("No INVARIANT_API_KEY set, failing")
dataset_name = f"test-dataset-open-ai-{uuid.uuid4()}"
client = OpenAI(
http_client=Client(
headers={
"Invariant-Authorization": f"Bearer {os.getenv('GUARDRAILS_API_KEY')}"
"Invariant-Authorization": f"Bearer {os.getenv('INVARIANT_API_KEY')}"
},
),
base_url=f"{gateway_url}/api/v1/gateway/{dataset_name}/openai"
@@ -109,15 +109,11 @@ async def test_message_content_guardrail_from_file(
)
annotations = annotations_response.json()
assert len(annotations) == 2
assert len(annotations) == 1
assert (
annotations[0]["content"] == "Madrid detected in the response"
and annotations[0]["extra_metadata"]["source"] == "guardrails-error"
)
assert (
annotations[1]["content"] == "Madrid detected in the response"
and annotations[1]["extra_metadata"]["source"] == "guardrails-error"
)
@pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="No OPENAI_API_KEY set")
@@ -129,8 +125,8 @@ async def test_tool_call_guardrail_from_file(
explorer_api_url, gateway_url, do_stream, push_to_explorer
):
"""Test the message content guardrail."""
if not os.getenv("GUARDRAILS_API_KEY"):
pytest.fail("No GUARDRAILS_API_KEY set, failing")
if not os.getenv("INVARIANT_API_KEY"):
pytest.fail("No INVARIANT_API_KEY set, failing")
tools = [
{
@@ -169,7 +165,7 @@ async def test_tool_call_guardrail_from_file(
client = OpenAI(
http_client=Client(
headers={
"Invariant-Authorization": f"Bearer {os.getenv('GUARDRAILS_API_KEY')}"
"Invariant-Authorization": f"Bearer {os.getenv('INVARIANT_API_KEY')}"
},
),
base_url=f"{gateway_url}/api/v1/gateway/{dataset_name}/openai"