perf: optimize gateway cold start from ~6.9s to ~460ms (#3918)
Channel lazy load: discover_enabled() only imports enabled channel modules instead of all 18 modules with heavy SDKs (telegram, discord, slack, etc). discover_all() now delegates to discover_enabled(). Lazy OpenAI client: defer AsyncOpenAI() + httpx construction to _ensure_client() with asyncio.Lock double-checked locking. openai and httpx imports moved from module-level into _ensure_client(). Minor: lazy Nanobot/RunResult and CronService exports via __getattr__. Benchmark: 6910ms → 460ms (-93.3%)
This commit is contained in:
chengyongru
Xubin Ren
@@ -16,20 +16,9 @@ from ipaddress import ip_address
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
import json_repair
|
||||
from loguru import logger
|
||||
|
||||
if os.environ.get("LANGFUSE_SECRET_KEY") and importlib.util.find_spec("langfuse"):
|
||||
from langfuse.openai import AsyncOpenAI
|
||||
else:
|
||||
if os.environ.get("LANGFUSE_SECRET_KEY"):
|
||||
logger.warning(
|
||||
"LANGFUSE_SECRET_KEY is set but langfuse is not installed; "
|
||||
"install with `pip install langfuse` to enable tracing"
|
||||
)
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
from nanobot.providers.base import LLMProvider, LLMResponse, ToolCallRequest
|
||||
from nanobot.providers.openai_responses import (
|
||||
consume_sdk_stream,
|
||||
@@ -39,8 +28,15 @@ from nanobot.providers.openai_responses import (
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from openai import AsyncOpenAI as AsyncOpenAIType
|
||||
|
||||
from nanobot.providers.registry import ProviderSpec
|
||||
|
||||
# Module-level placeholder — set lazily by _ensure_client on first real
|
||||
# use, or replaced by tests via ``patch(...)``. Kept as a plain name so
|
||||
# that ``unittest.mock.patch`` can find and replace it.
|
||||
AsyncOpenAI: Any = None
|
||||
|
||||
_ALLOWED_MSG_KEYS = frozenset({
|
||||
"role", "content", "tool_calls", "tool_call_id", "name",
|
||||
"reasoning_content", "extra_content",
|
||||
@@ -302,43 +298,80 @@ class OpenAICompatProvider(LLMProvider):
|
||||
|
||||
effective_base = api_base or (spec.default_api_base if spec else None) or None
|
||||
self._effective_base = effective_base
|
||||
default_headers = {"x-session-affinity": uuid.uuid4().hex}
|
||||
self._default_headers = {"x-session-affinity": uuid.uuid4().hex}
|
||||
if _uses_openrouter_attribution(spec, effective_base):
|
||||
default_headers.update(_DEFAULT_OPENROUTER_HEADERS)
|
||||
self._default_headers.update(_DEFAULT_OPENROUTER_HEADERS)
|
||||
if extra_headers:
|
||||
default_headers.update(extra_headers)
|
||||
self._default_headers.update(extra_headers)
|
||||
self._api_key_for_client = api_key or "no-key"
|
||||
self._is_local = _is_local_endpoint(spec, effective_base)
|
||||
|
||||
# Local model servers (Ollama, llama.cpp, vLLM) often close idle
|
||||
# HTTP connections before the client-side keepalive expires. When
|
||||
# two LLM calls happen seconds apart (e.g. heartbeat _decide then
|
||||
# process_direct), the second call may grab a now-dead pooled
|
||||
# connection, causing a transient APIConnectionError on every first
|
||||
# attempt. Disabling keepalive for local endpoints avoids this by
|
||||
# opening a fresh connection for each request, which is cheap on a
|
||||
# LAN. Cloud providers benefit from keepalive, so we leave the
|
||||
# default pool settings for them.
|
||||
timeout_s = _openai_compat_timeout_s()
|
||||
http_client: httpx.AsyncClient | None = None
|
||||
if _is_local_endpoint(spec, effective_base):
|
||||
http_client = httpx.AsyncClient(
|
||||
limits=httpx.Limits(keepalive_expiry=0),
|
||||
timeout=timeout_s,
|
||||
)
|
||||
# Lazy-init: the OpenAI client and its httpx transport are expensive
|
||||
# to create (~700 ms on Windows). Defer until first use — unless
|
||||
# AsyncOpenAI has been patched (tests), in which case build eagerly.
|
||||
self._client: AsyncOpenAIType | None = None
|
||||
self._client_lock = asyncio.Lock()
|
||||
|
||||
self._client = AsyncOpenAI(
|
||||
api_key=api_key or "no-key",
|
||||
base_url=effective_base,
|
||||
default_headers=default_headers,
|
||||
max_retries=0,
|
||||
timeout=timeout_s,
|
||||
http_client=http_client,
|
||||
)
|
||||
if AsyncOpenAI is not None:
|
||||
self._build_client()
|
||||
|
||||
# Responses API circuit breaker: skip after repeated failures,
|
||||
# probe again after _RESPONSES_PROBE_INTERVAL_S seconds.
|
||||
self._responses_failures: dict[str, int] = {}
|
||||
self._responses_tripped_at: dict[str, float] = {}
|
||||
|
||||
def _build_client(self) -> None:
|
||||
"""Create the OpenAI client using the current module-level AsyncOpenAI."""
|
||||
import httpx
|
||||
|
||||
timeout_s = _openai_compat_timeout_s()
|
||||
http_client: httpx.AsyncClient | None = None
|
||||
if self._is_local:
|
||||
# Local model servers (Ollama, llama.cpp, vLLM) often close idle
|
||||
# HTTP connections before the client-side keepalive expires. When
|
||||
# two LLM calls happen seconds apart (e.g. heartbeat _decide then
|
||||
# process_direct), the second call may grab a now-dead pooled
|
||||
# connection, causing a transient APIConnectionError on every first
|
||||
# attempt. Disabling keepalive for local endpoints avoids this by
|
||||
# opening a fresh connection for each request, which is cheap on a
|
||||
# LAN. Cloud providers benefit from keepalive, so we leave the
|
||||
# default pool settings for them.
|
||||
http_client = httpx.AsyncClient(
|
||||
limits=httpx.Limits(keepalive_expiry=0),
|
||||
timeout=timeout_s,
|
||||
)
|
||||
self._client = AsyncOpenAI(
|
||||
api_key=self._api_key_for_client,
|
||||
base_url=self._effective_base,
|
||||
default_headers=self._default_headers,
|
||||
max_retries=0,
|
||||
timeout=timeout_s,
|
||||
http_client=http_client,
|
||||
)
|
||||
|
||||
async def _ensure_client(self):
|
||||
"""Return the shared OpenAI client, creating it on first call."""
|
||||
if self._client is not None:
|
||||
return self._client
|
||||
async with self._client_lock:
|
||||
if self._client is not None:
|
||||
return self._client
|
||||
global AsyncOpenAI
|
||||
if AsyncOpenAI is None:
|
||||
if os.environ.get("LANGFUSE_SECRET_KEY") and importlib.util.find_spec("langfuse"):
|
||||
from langfuse.openai import AsyncOpenAI as _AsyncOpenAI
|
||||
else:
|
||||
if os.environ.get("LANGFUSE_SECRET_KEY"):
|
||||
logger.warning(
|
||||
"LANGFUSE_SECRET_KEY is set but langfuse is not installed; "
|
||||
"install with `pip install langfuse` to enable tracing"
|
||||
)
|
||||
from openai import AsyncOpenAI as _AsyncOpenAI
|
||||
AsyncOpenAI = _AsyncOpenAI
|
||||
|
||||
self._build_client()
|
||||
return self._client
|
||||
|
||||
def _setup_env(self, api_key: str, api_base: str | None) -> None:
|
||||
"""Set environment variables based on provider spec."""
|
||||
spec = self._spec
|
||||
@@ -1182,6 +1215,7 @@ class OpenAICompatProvider(LLMProvider):
|
||||
reasoning_effort: str | None = None,
|
||||
tool_choice: str | dict[str, Any] | None = None,
|
||||
) -> LLMResponse:
|
||||
await self._ensure_client()
|
||||
try:
|
||||
if self._should_use_responses_api(model, reasoning_effort):
|
||||
try:
|
||||
@@ -1223,6 +1257,7 @@ class OpenAICompatProvider(LLMProvider):
|
||||
on_thinking_delta: Callable[[str], Awaitable[None]] | None = None,
|
||||
on_tool_call_delta: Callable[[dict[str, Any]], Awaitable[None]] | None = None,
|
||||
) -> LLMResponse:
|
||||
await self._ensure_client()
|
||||
idle_timeout_s = int(os.environ.get("NANOBOT_STREAM_IDLE_TIMEOUT_S", "90"))
|
||||
try:
|
||||
if self._should_use_responses_api(model, reasoning_effort):
|
||||
|
||||
Reference in New Issue
Block a user