bug fix

.env

@@ -6,11 +6,11 @@ LOG_CHAT=true
 CACHE_CHAT_COMPLETION=true
 
 # `CACHE_BACKEND`: Options (MEMORY, LMDB, LevelDB)
-CACHE_BACKEND=LMDB
+CACHE_BACKEND=MEMORY
 
 #LOG_CACHE_DB_INFO=false
 
-BENCHMARK_MODE=true
+#BENCHMARK_MODE=true
 
 # `OPENAI_BASE_URL`: Forward any service address in the style of OpenAI; multiple can be specified, separated by commas.
 # If more than one is specified, neither OPENAI_ROUTE_PREFIX nor EXTRA_ROUTE_PREFIX can be the root route/

openai_forward/app.py

@@ -4,7 +4,6 @@
 from slowapi.errors import RateLimitExceeded
 
 from . import __version__, custom_slowapi
-from .cache.database import db_dict
 from .forward.extra import generic_objs
 from .forward.openai import openai_objs
 from .helper import normalize_route as normalize_route_path
@@ -64,14 +63,16 @@ def healthz(request: Request):
     )
 
 
+@app.on_event("startup")
+async def startup():
+    [await obj.build_client() for obj in openai_objs]
+    [await obj.build_client() for obj in generic_objs]
+
+
 @app.on_event("shutdown")
 async def shutdown():
-    if hasattr(db_dict, "close"):
-        db_dict.close()
-    for obj in openai_objs:
-        await obj.client.close()
-    for obj in generic_objs:
-        await obj.client.close()
+    [await obj.client.close() for obj in openai_objs]
+    [await obj.client.close() for obj in generic_objs]
 
 
 add_route = lambda obj: app.add_route(

openai_forward/forward/base.py

@@ -36,9 +36,8 @@ def __init__(self, base_url: str, route_prefix: str, proxy=None):
         self.PROXY = proxy
         self.ROUTE_PREFIX = route_prefix
         self.client: aiohttp.ClientSession | None = None
-        self.build_client()
 
-    def build_client(self):
+    async def build_client(self):
         connector = TCPConnector(limit=500, limit_per_host=0, force_close=False)
         self.client = aiohttp.ClientSession(connector=connector, timeout=self.timeout)
 
@@ -154,6 +153,7 @@ def prepare_client(self, request: Request, return_origin_header=False) -> dict:
         }
 
     async def reverse_proxy(self, request: Request):
+        assert self.client
         client_config = self.prepare_client(request, return_origin_header=True)
 
         r = await self.send(client_config, data=await request.body())