import httpx import pytest from estimator.config.sections import RouteCfg, StepRouteCfg from estimator.domain.errors import LlmAuthError, LlmConfigError, LlmRequestError, LlmTransientError from estimator.domain.llm import LlmResult, Usage from estimator.infra.llm import LlmRouter, build_gateway from tests.unit.llm.conftest import Answer, Recorder, ok_anthropic, ok_openai class FakeClient: def __init__(self, name, result=None, error=None): self.name = name self.result = result self.error = error self.calls = [] async def structured(self, **kw): self.calls.append(kw) if self.error: raise self.error return LlmResult(self.result, Usage(1, 1), self.name, kw["model"]) A = Answer(text="a", score=1) def route(provider="anthropic", model="m", fallback=None, **kw): return StepRouteCfg(provider=provider, model=model, fallback=fallback, **kw) async def run(router, step="critic"): return await router.structured(step, system="s", user="u", schema=Answer) async def test_routes_by_step_and_passes_params(): anth, oai = FakeClient("anthropic", A), FakeClient("openai", A) router = LlmRouter( {"clarify": route("anthropic", "claude-x", temperature=0.2, max_tokens=111), "critic": route("openai", "gpt-x", reasoning_effort="low")}, {"anthropic": anth, "openai": oai}, ) # fmt: skip r1 = await run(router, "clarify") r2 = await run(router, "critic") assert r1.provider == "anthropic" and r2.provider == "openai" assert anth.calls[0]["model"] == "claude-x" assert anth.calls[0]["temperature"] == 0.2 and anth.calls[0]["max_tokens"] == 111 assert oai.calls[0]["reasoning_effort"] == "low" and oai.calls[0]["temperature"] is None async def test_unknown_step(): router = LlmRouter({}, {}) with pytest.raises(LlmConfigError): await run(router, "nope") async def test_fallback_on_transient_error(): primary = FakeClient("openai", error=LlmTransientError("down", provider="openai")) backup = FakeClient("anthropic", A) router = LlmRouter( {"critic": route("openai", "g", fallback=RouteCfg(provider="anthropic", model="c"))}, {"openai": primary, "anthropic": backup}, ) res = await run(router) assert res.provider == "anthropic" and res.via_fallback is True and res.model == "c" assert len(primary.calls) == 1 and len(backup.calls) == 1 async def test_fallback_on_auth_error(): router = LlmRouter( {"critic": route("openai", "g", fallback=RouteCfg(provider="anthropic", model="c"))}, {"openai": FakeClient("openai", error=LlmAuthError("bad", provider="openai")), "anthropic": FakeClient("anthropic", A)}, ) # fmt: skip assert (await run(router)).via_fallback is True async def test_no_fallback_on_bad_request(): backup = FakeClient("anthropic", A) router = LlmRouter( {"critic": route("openai", "g", fallback=RouteCfg(provider="anthropic", model="c"))}, {"openai": FakeClient("openai", error=LlmRequestError("bad", provider="openai")), "anthropic": backup}, ) with pytest.raises(LlmRequestError): await run(router) assert backup.calls == [] async def test_error_propagates_without_fallback_configured(): router = LlmRouter( {"critic": route("openai", "g")}, {"openai": FakeClient("openai", error=LlmTransientError("down"))}, ) with pytest.raises(LlmTransientError): await run(router) async def test_primary_success_does_not_touch_fallback(): backup = FakeClient("anthropic", A) router = LlmRouter( {"critic": route("openai", "g", fallback=RouteCfg(provider="anthropic", model="c"))}, {"openai": FakeClient("openai", A), "anthropic": backup}, ) res = await run(router) assert res.via_fallback is False and backup.calls == [] async def test_build_gateway_end_to_end_with_both_providers(monkeypatch): """Settings -> real clients over mock HTTP; critic on OpenAI falls back to Anthropic.""" from estimator.config.settings import Settings monkeypatch.setenv("EST_DATABASE__DSN", "sqlite:///:memory:") monkeypatch.setenv("EST_LLM__ANTHROPIC__API_KEY", "sk-ant") monkeypatch.setenv("EST_LLM__OPENAI__API_KEY", "sk-oai") monkeypatch.setenv("EST_LLM__STEPS__CRITIC__PROVIDER", "openai") monkeypatch.setenv("EST_LLM__STEPS__CRITIC__MODEL", "gpt-x") monkeypatch.setenv("EST_LLM__STEPS__CRITIC__FALLBACK__PROVIDER", "anthropic") monkeypatch.setenv("EST_LLM__STEPS__CRITIC__FALLBACK__MODEL", "claude-x") settings = Settings() def handler(request: httpx.Request) -> httpx.Response: if "openai" in request.url.host: return httpx.Response(503, json={"error": {"message": "down"}}) return ok_anthropic('{"text": "from fallback", "score": 2}') sleeps = [] async def no_sleep(s): sleeps.append(s) gw = build_gateway(settings, transport=httpx.MockTransport(handler)) for c in gw._clients.values(): # keep retries instant c._sleep = no_sleep try: res = await gw.structured("critic", system="s", user="u", schema=Answer) assert res.parsed.text == "from fallback" and res.via_fallback and res.provider == "anthropic" ok = await gw.structured("clarify", system="s", user="u", schema=Answer) assert ok.provider == "anthropic" and not ok.via_fallback finally: await gw.aclose() async def test_build_gateway_only_creates_used_providers(monkeypatch): from estimator.config.settings import Settings monkeypatch.setenv("EST_DATABASE__DSN", "sqlite:///:memory:") monkeypatch.setenv("EST_LLM__ANTHROPIC__API_KEY", "sk-ant") gw = build_gateway(Settings(), transport=Recorder(ok_openai("{}")).transport) try: assert set(gw._clients) == {"anthropic"} finally: await gw.aclose()