feat(runner): support fallback candidates
Resolve fallbackModels as preset references or explicit inline provider configs so failover uses complete model settings without exposing fallback logic to the agent loop. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -5,7 +5,7 @@ from __future__ import annotations
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from nanobot.config.schema import Config, ModelPresetConfig
|
||||
from nanobot.config.schema import Config, InlineFallbackConfig, ModelPresetConfig
|
||||
from nanobot.providers.base import LLMProvider
|
||||
from nanobot.providers.fallback_provider import FallbackProvider
|
||||
from nanobot.providers.registry import find_by_name
|
||||
@@ -104,6 +104,36 @@ def _make_provider_core(
|
||||
return provider
|
||||
|
||||
|
||||
def _inline_fallback_preset(
|
||||
primary: ModelPresetConfig,
|
||||
fallback: InlineFallbackConfig,
|
||||
) -> ModelPresetConfig:
|
||||
return ModelPresetConfig(
|
||||
model=fallback.model,
|
||||
provider=fallback.provider,
|
||||
max_tokens=fallback.max_tokens if fallback.max_tokens is not None else primary.max_tokens,
|
||||
context_window_tokens=(
|
||||
fallback.context_window_tokens
|
||||
if fallback.context_window_tokens is not None
|
||||
else primary.context_window_tokens
|
||||
),
|
||||
temperature=(
|
||||
fallback.temperature if fallback.temperature is not None else primary.temperature
|
||||
),
|
||||
reasoning_effort=fallback.reasoning_effort,
|
||||
)
|
||||
|
||||
|
||||
def _resolve_fallback_presets(config: Config, primary: ModelPresetConfig) -> list[ModelPresetConfig]:
|
||||
presets: list[ModelPresetConfig] = []
|
||||
for fallback in config.agents.defaults.fallback_models:
|
||||
if isinstance(fallback, str):
|
||||
presets.append(config.model_presets[fallback])
|
||||
else:
|
||||
presets.append(_inline_fallback_preset(primary, fallback))
|
||||
return presets
|
||||
|
||||
|
||||
def make_provider(
|
||||
config: Config,
|
||||
*,
|
||||
@@ -118,14 +148,14 @@ def make_provider(
|
||||
"""
|
||||
resolved = _resolve_model_preset(config, preset_name=preset_name, preset=preset)
|
||||
provider = _make_provider_core(config, preset_name=preset_name, preset=preset, model=model)
|
||||
fallback_presets = _resolve_fallback_presets(config, resolved)
|
||||
|
||||
if resolved.fallback_models:
|
||||
fb_preset = resolved.model_copy(update={"provider": "auto", "fallback_models": []})
|
||||
if fallback_presets:
|
||||
provider = FallbackProvider(
|
||||
primary=provider,
|
||||
fallback_models=resolved.fallback_models,
|
||||
provider_factory=lambda m: _make_provider_core(
|
||||
config, preset_name=preset_name, preset=fb_preset, model=m
|
||||
fallback_presets=fallback_presets,
|
||||
provider_factory=lambda fb: _make_provider_core(
|
||||
config, preset_name=preset_name, preset=fb
|
||||
),
|
||||
)
|
||||
|
||||
@@ -138,9 +168,29 @@ def provider_signature(
|
||||
preset_name: str | None = None,
|
||||
preset: ModelPresetConfig | None = None,
|
||||
) -> tuple[object, ...]:
|
||||
"""Return the config fields that affect the primary LLM provider."""
|
||||
"""Return the config fields that affect the active provider chain."""
|
||||
resolved = _resolve_model_preset(config, preset_name=preset_name, preset=preset)
|
||||
p = config.get_provider(resolved.model, preset=resolved)
|
||||
fallback_presets = _resolve_fallback_presets(config, resolved)
|
||||
|
||||
def _fallback_signature(fallback: ModelPresetConfig) -> tuple[object, ...]:
|
||||
fp = config.get_provider(fallback.model, preset=fallback)
|
||||
return (
|
||||
fallback.model,
|
||||
fallback.provider,
|
||||
config.get_provider_name(fallback.model, preset=fallback),
|
||||
config.get_api_key(fallback.model, preset=fallback),
|
||||
config.get_api_base(fallback.model, preset=fallback),
|
||||
fp.extra_headers if fp else None,
|
||||
fp.extra_body if fp else None,
|
||||
getattr(fp, "region", None) if fp else None,
|
||||
getattr(fp, "profile", None) if fp else None,
|
||||
fallback.max_tokens,
|
||||
fallback.temperature,
|
||||
fallback.reasoning_effort,
|
||||
fallback.context_window_tokens,
|
||||
)
|
||||
|
||||
return (
|
||||
resolved.model,
|
||||
resolved.provider,
|
||||
@@ -155,6 +205,7 @@ def provider_signature(
|
||||
resolved.temperature,
|
||||
resolved.reasoning_effort,
|
||||
resolved.context_window_tokens,
|
||||
tuple(_fallback_signature(fallback) for fallback in fallback_presets),
|
||||
)
|
||||
|
||||
|
||||
@@ -165,10 +216,14 @@ def build_provider_snapshot(
|
||||
preset: ModelPresetConfig | None = None,
|
||||
) -> ProviderSnapshot:
|
||||
resolved = _resolve_model_preset(config, preset_name=preset_name, preset=preset)
|
||||
fallback_windows = [
|
||||
fallback.context_window_tokens
|
||||
for fallback in _resolve_fallback_presets(config, resolved)
|
||||
]
|
||||
return ProviderSnapshot(
|
||||
provider=make_provider(config, preset=resolved),
|
||||
model=resolved.model,
|
||||
context_window_tokens=resolved.context_window_tokens,
|
||||
context_window_tokens=min([resolved.context_window_tokens, *fallback_windows]),
|
||||
signature=provider_signature(config, preset=resolved),
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user