fix(providers): fall back on authentication errors

docs/configuration.md

@@ -1486,7 +1486,7 @@ Inline fallback object:
 
 Use inline objects only when a fallback is not worth naming as a reusable preset. `fallbackModels` belongs under `agents.defaults`, not inside individual `modelPresets` entries.
 
-Failover normally runs when the primary provider returns a retryable model/provider error before any answer text has been streamed. Stream-stall timeouts are the recovery exception: if the provider already emitted partial answer text and then stalls, nanobot closes the current stream segment and retries/fails over in a new segment. Typical fallback cases include timeouts, connection errors, 5xx server errors, 429 rate limits, overloads, and quota/balance exhaustion. It does not run for malformed requests, authentication/permission errors, content filtering/refusals, or context-length/message-format errors.
+Failover normally runs when the primary provider returns a fallbackable model/provider error before any answer text has been streamed. Stream-stall timeouts are the recovery exception: if the provider already emitted partial answer text and then stalls, nanobot closes the current stream segment and retries/fails over in a new segment. Typical fallback cases include timeouts, connection errors, 5xx server errors, 429 rate limits, overloads, authentication/permission failures such as invalid or expired credentials, and quota/balance exhaustion. It does not run for malformed requests, content filtering/refusals, or context-length/message-format errors.
 
 If fallback candidates use smaller `contextWindowTokens` values, nanobot builds context using the smallest window in the active chain so every candidate can receive the same prompt.
 

nanobot/providers/fallback_provider.py

@@ -21,10 +21,36 @@ _FALLBACK_ERROR_KINDS = frozenset({
     "rate_limit",
     "overloaded",
 })
-_NON_FALLBACK_ERROR_KINDS = frozenset({
+_AUTHENTICATION_ERROR_KINDS = frozenset({
     "authentication",
     "auth",
     "permission",
+})
+_AUTHENTICATION_ERROR_TOKENS = (
+    "authentication_error",
+    "authentication error",
+    "invalid_api_key",
+    "invalid api key",
+    "incorrect_api_key",
+    "incorrect api key",
+    "expired_api_key",
+    "expired api key",
+    "invalid credential",
+    "expired credential",
+    "credential has expired",
+    "credentials have expired",
+    "invalid_token",
+    "invalid token",
+    "expired_token",
+    "expired token",
+    "unauthorized",
+    "permission_denied",
+    "permission denied",
+    "access_denied",
+    "account_deactivated",
+    "organization_deactivated",
+)
+_NON_FALLBACK_ERROR_KINDS = frozenset({
     "content_filter",
     "refusal",
     "context_length",
@@ -293,19 +319,36 @@ class FallbackProvider(LLMProvider):
     def _should_fallback(response: LLMResponse) -> bool:
         if LLMProvider.is_arrearage_response(response):
             return True
-        if response.error_should_retry is False:
-            return False
         status = response.error_status_code
         kind = (response.error_kind or "").lower()
         error_type = (response.error_type or "").lower()
         code = (response.error_code or "").lower()
         text = (response.content or "").lower()
+        structured_values = (kind, error_type, code)
 
-        if status in {400, 401, 403, 404, 422}:
+        if kind in _AUTHENTICATION_ERROR_KINDS:
-            return False
+            return True
+        if any(
+            token in value
+            for value in structured_values
+            for token in _AUTHENTICATION_ERROR_TOKENS
+        ):
+            return True
         if kind in _NON_FALLBACK_ERROR_KINDS:
             return False
-        if any(token in value for value in (kind, error_type, code) for token in _NON_FALLBACK_ERROR_KINDS):
+        if any(
+            token in value
+            for value in structured_values
+            for token in _NON_FALLBACK_ERROR_KINDS
+        ):
+            return False
+        if status in {401, 403}:
+            return True
+        if any(token in text for token in _AUTHENTICATION_ERROR_TOKENS):
+            return True
+        if response.error_should_retry is False:
+            return False
+        if status in {400, 404, 422}:
             return False
         if response.error_should_retry is True:
             return True

tests/agent/test_runner_fallback.py

@@ -483,6 +483,83 @@ class TestFailoverOnArrearageError:
         factory.assert_not_called()
 
 
+class TestFailoverOnAuthenticationError:
+    @pytest.mark.parametrize(
+        "authentication_error",
+        [
+            pytest.param(
+                _make_response(
+                    (
+                        "Error: {'error': {'type': 'authentication_error', "
+                        "'message': 'The API Key appears to be invalid or may have expired.'}}"
+                    ),
+                    finish_reason="error",
+                    error_type="authentication_error",
+                    error_should_retry=False,
+                ),
+                id="authentication-error-type",
+            ),
+            pytest.param(
+                _make_response(
+                    "unauthorized",
+                    finish_reason="error",
+                    error_status_code=401,
+                    error_kind="http",
+                    error_should_retry=False,
+                ),
+                id="http-401",
+            ),
+            pytest.param(
+                _make_response(
+                    "bad key",
+                    finish_reason="error",
+                    error_type="invalid_request_error",
+                    error_code="invalid_api_key",
+                    error_should_retry=False,
+                ),
+                id="invalid-api-key-code",
+            ),
+            pytest.param(
+                _make_response(
+                    "credentials have expired",
+                    finish_reason="error",
+                    error_should_retry=False,
+                ),
+                id="expired-credentials-text",
+            ),
+            pytest.param(
+                _make_response(
+                    "permission denied",
+                    finish_reason="error",
+                    error_status_code=403,
+                    error_kind="permission",
+                    error_should_retry=False,
+                ),
+                id="permission-error",
+            ),
+        ],
+    )
+    @pytest.mark.asyncio
+    async def test_tries_configured_fallback(
+        self,
+        authentication_error: LLMResponse,
+    ) -> None:
+        primary = _FakeProvider("primary", authentication_error)
+        fallback = _FakeProvider("fallback", _make_response("fallback ok"))
+        fallback_preset = _fallback("fallback-a")
+        factory = MagicMock(return_value=fallback)
+        fb = FallbackProvider(
+            primary=primary,
+            fallback_presets=[fallback_preset],
+            provider_factory=factory,
+        )
+
+        result = await fb.chat(messages=[{"role": "user", "content": "hi"}])
+
+        assert result.content == "fallback ok"
+        factory.assert_called_once_with(fallback_preset)
+
+
 class TestNoFallbackOnNonRetryableError:
     @pytest.mark.asyncio
     async def test_bad_request(self) -> None:
@@ -508,14 +585,15 @@ class TestNoFallbackOnNonRetryableError:
         factory.assert_not_called()
 
     @pytest.mark.asyncio
-    async def test_auth_error(self) -> None:
+    async def test_content_filter_takes_precedence_over_403(self) -> None:
         primary = _FakeProvider(
             "primary",
             _make_response(
-                "unauthorized",
+                "request blocked by content filter",
                 finish_reason="error",
-                error_status_code=401,
+                error_status_code=403,
-                error_kind="authentication",
+                error_kind="content_filter",
+                error_should_retry=False,
             ),
         )
         factory = MagicMock()