fix(providers): allow retry and fallback on stream stalled timeout
When a stream stalls mid-response, both the retry layer and FallbackProvider blocked recovery because content had already been emitted via on_content_delta. This left users with truncated replies and no automatic recovery. For error_kind="timeout" specifically: - _run_with_retry now suppresses delta callbacks and retries the same model instead of returning immediately - FallbackProvider now allows failover to a different model with delta callbacks suppressed Non-timeout errors retain the original "skip retry/failover after streamed content" behavior to avoid duplicate output.
This commit is contained in:
committed by
Xubin Ren
parent
dadb35af49
commit
2c5a4e0703
@@ -163,6 +163,42 @@ async def test_chat_stream_with_retry_does_not_retry_after_emitting_content(monk
|
||||
assert delays == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_stream_with_retry_retries_timeout_after_emitting_content(monkeypatch) -> None:
|
||||
first = LLMResponse(
|
||||
content="Error calling LLM: stream stalled for more than 30 seconds",
|
||||
finish_reason="error",
|
||||
error_kind="timeout",
|
||||
)
|
||||
first._test_stream_delta = "partial" # type: ignore[attr-defined]
|
||||
provider = ScriptedProvider([
|
||||
first,
|
||||
LLMResponse(content="full retry response"),
|
||||
])
|
||||
deltas: list[str] = []
|
||||
delays: list[int] = []
|
||||
|
||||
async def _fake_sleep(delay: int) -> None:
|
||||
delays.append(delay)
|
||||
|
||||
async def _on_delta(delta: str) -> None:
|
||||
deltas.append(delta)
|
||||
|
||||
monkeypatch.setattr("nanobot.providers.base.asyncio.sleep", _fake_sleep)
|
||||
|
||||
response = await provider.chat_stream_with_retry(
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
on_content_delta=_on_delta,
|
||||
)
|
||||
|
||||
assert response.content == "full retry response"
|
||||
assert response.finish_reason == "stop"
|
||||
assert provider.calls == 2
|
||||
assert deltas == ["partial"]
|
||||
assert delays == [1]
|
||||
assert provider.last_kwargs.get("on_content_delta") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_with_retry_uses_provider_generation_defaults() -> None:
|
||||
"""When callers omit generation params, provider.generation defaults are used."""
|
||||
|
||||
Reference in New Issue
Block a user