Add timeout to httpx.AsyncClient

proxy/routes/anthropic.py

@@ -5,7 +5,7 @@ from typing import Any
 
 import httpx
 from fastapi import APIRouter, Depends, Header, HTTPException, Request
-from utils.constants import IGNORED_HEADERS
+from utils.constants import CLIENT_TIMEOUT, IGNORED_HEADERS
 from utils.explorer import push_trace
 
 proxy = APIRouter()
@@ -50,7 +50,7 @@ async def anthropic_proxy(
     request_body_json = json.loads(request_body)
 
     anthropic_url = f"https://api.anthropic.com/{endpoint}"
-    client = httpx.AsyncClient()
+    client = httpx.AsyncClient(timeout=httpx.Timeout(CLIENT_TIMEOUT))
 
     anthropic_request = client.build_request(
         "POST", anthropic_url, headers=headers, data=request_body

proxy/routes/open_ai.py

@@ -6,7 +6,7 @@ from typing import Any
 import httpx
 from fastapi import APIRouter, Depends, Header, HTTPException, Request, Response
 from starlette.responses import StreamingResponse
-from utils.constants import IGNORED_HEADERS
+from utils.constants import CLIENT_TIMEOUT, IGNORED_HEADERS
 from utils.explorer import push_trace
 
 ALLOWED_OPEN_AI_ENDPOINTS = {"chat/completions"}
@@ -54,7 +54,7 @@ async def openai_proxy(
     is_streaming = request_body_json.get("stream", False)
     invariant_authorization = request.headers.get("invariant-authorization")
 
-    client = httpx.AsyncClient()
+    client = httpx.AsyncClient(timeout=httpx.Timeout(CLIENT_TIMEOUT))
     open_ai_request = client.build_request(
         "POST",
         f"https://api.openai.com/v1/{endpoint}",

proxy/utils/constants.py

@@ -11,3 +11,5 @@ IGNORED_HEADERS = [
     "x-forwarded-server",
     "x-real-ip",
 ]
+
+CLIENT_TIMEOUT = 60.0