fix(providers): allow retry and fallback on stream stalled timeout
When a stream stalls mid-response, both the retry layer and FallbackProvider blocked recovery because content had already been emitted via on_content_delta. This left users with truncated replies and no automatic recovery. For error_kind="timeout" specifically: - _run_with_retry now suppresses delta callbacks and retries the same model instead of returning immediately - FallbackProvider now allows failover to a different model with delta callbacks suppressed Non-timeout errors retain the original "skip retry/failover after streamed content" behavior to avoid duplicate output.
This commit is contained in:
committed by
Xubin Ren
parent
dadb35af49
commit
2c5a4e0703
@@ -827,10 +827,22 @@ class LLMProvider(ABC):
|
||||
return response
|
||||
last_response = response
|
||||
if should_retry_guard is not None and not should_retry_guard():
|
||||
logger.warning(
|
||||
"LLM stream failed after content was emitted; skipping retry"
|
||||
)
|
||||
return response
|
||||
is_timeout = (response.error_kind or "").lower() == "timeout"
|
||||
if is_timeout:
|
||||
logger.warning(
|
||||
"LLM stream stalled after content was emitted; "
|
||||
"suppressing delta callbacks and retrying"
|
||||
)
|
||||
kw.setdefault("on_content_delta", None)
|
||||
kw["on_content_delta"] = None
|
||||
kw["on_thinking_delta"] = None
|
||||
kw["on_tool_call_delta"] = None
|
||||
should_retry_guard = None
|
||||
else:
|
||||
logger.warning(
|
||||
"LLM stream failed after content was emitted; skipping retry"
|
||||
)
|
||||
return response
|
||||
error_key = ((response.content or "").strip().lower() or None)
|
||||
if error_key and error_key == last_error_key:
|
||||
identical_error_count += 1
|
||||
|
||||
@@ -149,10 +149,20 @@ class FallbackProvider(LLMProvider):
|
||||
return response
|
||||
|
||||
if has_streamed is not None and has_streamed[0]:
|
||||
logger.warning(
|
||||
"Primary model error but content already streamed; skipping failover"
|
||||
)
|
||||
return response
|
||||
is_timeout = (response.error_kind or "").lower() == "timeout"
|
||||
if is_timeout:
|
||||
logger.warning(
|
||||
"Primary model '{}' stream stalled after content was emitted; "
|
||||
"attempting failover anyway",
|
||||
primary_model,
|
||||
)
|
||||
has_streamed[0] = False
|
||||
kwargs["on_content_delta"] = None
|
||||
else:
|
||||
logger.warning(
|
||||
"Primary model error but content already streamed; skipping failover"
|
||||
)
|
||||
return response
|
||||
|
||||
if not self._should_fallback(response):
|
||||
logger.warning(
|
||||
|
||||
Reference in New Issue
Block a user