diff --git a/docs/configuration.md b/docs/configuration.md index 404cb772..b1c6c308 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1486,7 +1486,7 @@ Inline fallback object: Use inline objects only when a fallback is not worth naming as a reusable preset. `fallbackModels` belongs under `agents.defaults`, not inside individual `modelPresets` entries. -Failover normally runs when the primary provider returns a retryable model/provider error before any answer text has been streamed. Stream-stall timeouts are the recovery exception: if the provider already emitted partial answer text and then stalls, nanobot closes the current stream segment and retries/fails over in a new segment. Typical fallback cases include timeouts, connection errors, 5xx server errors, 429 rate limits, overloads, and quota/balance exhaustion. It does not run for malformed requests, authentication/permission errors, content filtering/refusals, or context-length/message-format errors. +Failover normally runs when the primary provider returns a fallbackable model/provider error before any answer text has been streamed. Stream-stall timeouts are the recovery exception: if the provider already emitted partial answer text and then stalls, nanobot closes the current stream segment and retries/fails over in a new segment. Typical fallback cases include timeouts, connection errors, 5xx server errors, 429 rate limits, overloads, authentication/permission failures such as invalid or expired credentials, and quota/balance exhaustion. It does not run for malformed requests, content filtering/refusals, or context-length/message-format errors. If fallback candidates use smaller `contextWindowTokens` values, nanobot builds context using the smallest window in the active chain so every candidate can receive the same prompt. diff --git a/nanobot/providers/fallback_provider.py b/nanobot/providers/fallback_provider.py index f52f53ef..9eb70802 100644 --- a/nanobot/providers/fallback_provider.py +++ b/nanobot/providers/fallback_provider.py @@ -21,10 +21,36 @@ _FALLBACK_ERROR_KINDS = frozenset({ "rate_limit", "overloaded", }) -_NON_FALLBACK_ERROR_KINDS = frozenset({ +_AUTHENTICATION_ERROR_KINDS = frozenset({ "authentication", "auth", "permission", +}) +_AUTHENTICATION_ERROR_TOKENS = ( + "authentication_error", + "authentication error", + "invalid_api_key", + "invalid api key", + "incorrect_api_key", + "incorrect api key", + "expired_api_key", + "expired api key", + "invalid credential", + "expired credential", + "credential has expired", + "credentials have expired", + "invalid_token", + "invalid token", + "expired_token", + "expired token", + "unauthorized", + "permission_denied", + "permission denied", + "access_denied", + "account_deactivated", + "organization_deactivated", +) +_NON_FALLBACK_ERROR_KINDS = frozenset({ "content_filter", "refusal", "context_length", @@ -293,19 +319,36 @@ class FallbackProvider(LLMProvider): def _should_fallback(response: LLMResponse) -> bool: if LLMProvider.is_arrearage_response(response): return True - if response.error_should_retry is False: - return False status = response.error_status_code kind = (response.error_kind or "").lower() error_type = (response.error_type or "").lower() code = (response.error_code or "").lower() text = (response.content or "").lower() + structured_values = (kind, error_type, code) - if status in {400, 401, 403, 404, 422}: - return False + if kind in _AUTHENTICATION_ERROR_KINDS: + return True + if any( + token in value + for value in structured_values + for token in _AUTHENTICATION_ERROR_TOKENS + ): + return True if kind in _NON_FALLBACK_ERROR_KINDS: return False - if any(token in value for value in (kind, error_type, code) for token in _NON_FALLBACK_ERROR_KINDS): + if any( + token in value + for value in structured_values + for token in _NON_FALLBACK_ERROR_KINDS + ): + return False + if status in {401, 403}: + return True + if any(token in text for token in _AUTHENTICATION_ERROR_TOKENS): + return True + if response.error_should_retry is False: + return False + if status in {400, 404, 422}: return False if response.error_should_retry is True: return True diff --git a/tests/agent/test_runner_fallback.py b/tests/agent/test_runner_fallback.py index 9f551e6c..c296709d 100644 --- a/tests/agent/test_runner_fallback.py +++ b/tests/agent/test_runner_fallback.py @@ -483,6 +483,83 @@ class TestFailoverOnArrearageError: factory.assert_not_called() +class TestFailoverOnAuthenticationError: + @pytest.mark.parametrize( + "authentication_error", + [ + pytest.param( + _make_response( + ( + "Error: {'error': {'type': 'authentication_error', " + "'message': 'The API Key appears to be invalid or may have expired.'}}" + ), + finish_reason="error", + error_type="authentication_error", + error_should_retry=False, + ), + id="authentication-error-type", + ), + pytest.param( + _make_response( + "unauthorized", + finish_reason="error", + error_status_code=401, + error_kind="http", + error_should_retry=False, + ), + id="http-401", + ), + pytest.param( + _make_response( + "bad key", + finish_reason="error", + error_type="invalid_request_error", + error_code="invalid_api_key", + error_should_retry=False, + ), + id="invalid-api-key-code", + ), + pytest.param( + _make_response( + "credentials have expired", + finish_reason="error", + error_should_retry=False, + ), + id="expired-credentials-text", + ), + pytest.param( + _make_response( + "permission denied", + finish_reason="error", + error_status_code=403, + error_kind="permission", + error_should_retry=False, + ), + id="permission-error", + ), + ], + ) + @pytest.mark.asyncio + async def test_tries_configured_fallback( + self, + authentication_error: LLMResponse, + ) -> None: + primary = _FakeProvider("primary", authentication_error) + fallback = _FakeProvider("fallback", _make_response("fallback ok")) + fallback_preset = _fallback("fallback-a") + factory = MagicMock(return_value=fallback) + fb = FallbackProvider( + primary=primary, + fallback_presets=[fallback_preset], + provider_factory=factory, + ) + + result = await fb.chat(messages=[{"role": "user", "content": "hi"}]) + + assert result.content == "fallback ok" + factory.assert_called_once_with(fallback_preset) + + class TestNoFallbackOnNonRetryableError: @pytest.mark.asyncio async def test_bad_request(self) -> None: @@ -508,14 +585,15 @@ class TestNoFallbackOnNonRetryableError: factory.assert_not_called() @pytest.mark.asyncio - async def test_auth_error(self) -> None: + async def test_content_filter_takes_precedence_over_403(self) -> None: primary = _FakeProvider( "primary", _make_response( - "unauthorized", + "request blocked by content filter", finish_reason="error", - error_status_code=401, - error_kind="authentication", + error_status_code=403, + error_kind="content_filter", + error_should_retry=False, ), ) factory = MagicMock()