Files

153 lines
5.8 KiB
Python

import httpx
import pytest
from estimator.config.sections import RouteCfg, StepRouteCfg
from estimator.domain.errors import LlmAuthError, LlmConfigError, LlmRequestError, LlmTransientError
from estimator.domain.llm import LlmResult, Usage
from estimator.infra.llm import LlmRouter, build_gateway
from tests.unit.llm.conftest import Answer, Recorder, ok_anthropic, ok_openai
class FakeClient:
def __init__(self, name, result=None, error=None):
self.name = name
self.result = result
self.error = error
self.calls = []
async def structured(self, **kw):
self.calls.append(kw)
if self.error:
raise self.error
return LlmResult(self.result, Usage(1, 1), self.name, kw["model"])
A = Answer(text="a", score=1)
def route(provider="anthropic", model="m", fallback=None, **kw):
return StepRouteCfg(provider=provider, model=model, fallback=fallback, **kw)
async def run(router, step="critic"):
return await router.structured(step, system="s", user="u", schema=Answer)
async def test_routes_by_step_and_passes_params():
anth, oai = FakeClient("anthropic", A), FakeClient("openai", A)
router = LlmRouter(
{"clarify": route("anthropic", "claude-x", temperature=0.2, max_tokens=111),
"critic": route("openai", "gpt-x", reasoning_effort="low")},
{"anthropic": anth, "openai": oai},
) # fmt: skip
r1 = await run(router, "clarify")
r2 = await run(router, "critic")
assert r1.provider == "anthropic" and r2.provider == "openai"
assert anth.calls[0]["model"] == "claude-x"
assert anth.calls[0]["temperature"] == 0.2 and anth.calls[0]["max_tokens"] == 111
assert oai.calls[0]["reasoning_effort"] == "low" and oai.calls[0]["temperature"] is None
async def test_unknown_step():
router = LlmRouter({}, {})
with pytest.raises(LlmConfigError):
await run(router, "nope")
async def test_fallback_on_transient_error():
primary = FakeClient("openai", error=LlmTransientError("down", provider="openai"))
backup = FakeClient("anthropic", A)
router = LlmRouter(
{"critic": route("openai", "g", fallback=RouteCfg(provider="anthropic", model="c"))},
{"openai": primary, "anthropic": backup},
)
res = await run(router)
assert res.provider == "anthropic" and res.via_fallback is True and res.model == "c"
assert len(primary.calls) == 1 and len(backup.calls) == 1
async def test_fallback_on_auth_error():
router = LlmRouter(
{"critic": route("openai", "g", fallback=RouteCfg(provider="anthropic", model="c"))},
{"openai": FakeClient("openai", error=LlmAuthError("bad", provider="openai")),
"anthropic": FakeClient("anthropic", A)},
) # fmt: skip
assert (await run(router)).via_fallback is True
async def test_no_fallback_on_bad_request():
backup = FakeClient("anthropic", A)
router = LlmRouter(
{"critic": route("openai", "g", fallback=RouteCfg(provider="anthropic", model="c"))},
{"openai": FakeClient("openai", error=LlmRequestError("bad", provider="openai")), "anthropic": backup},
)
with pytest.raises(LlmRequestError):
await run(router)
assert backup.calls == []
async def test_error_propagates_without_fallback_configured():
router = LlmRouter(
{"critic": route("openai", "g")},
{"openai": FakeClient("openai", error=LlmTransientError("down"))},
)
with pytest.raises(LlmTransientError):
await run(router)
async def test_primary_success_does_not_touch_fallback():
backup = FakeClient("anthropic", A)
router = LlmRouter(
{"critic": route("openai", "g", fallback=RouteCfg(provider="anthropic", model="c"))},
{"openai": FakeClient("openai", A), "anthropic": backup},
)
res = await run(router)
assert res.via_fallback is False and backup.calls == []
async def test_build_gateway_end_to_end_with_both_providers(monkeypatch):
"""Settings -> real clients over mock HTTP; critic on OpenAI falls back to Anthropic."""
from estimator.config.settings import Settings
monkeypatch.setenv("EST_DATABASE__DSN", "sqlite:///:memory:")
monkeypatch.setenv("EST_LLM__ANTHROPIC__API_KEY", "sk-ant")
monkeypatch.setenv("EST_LLM__OPENAI__API_KEY", "sk-oai")
monkeypatch.setenv("EST_LLM__STEPS__CRITIC__PROVIDER", "openai")
monkeypatch.setenv("EST_LLM__STEPS__CRITIC__MODEL", "gpt-x")
monkeypatch.setenv("EST_LLM__STEPS__CRITIC__FALLBACK__PROVIDER", "anthropic")
monkeypatch.setenv("EST_LLM__STEPS__CRITIC__FALLBACK__MODEL", "claude-x")
settings = Settings()
def handler(request: httpx.Request) -> httpx.Response:
if "openai" in request.url.host:
return httpx.Response(503, json={"error": {"message": "down"}})
return ok_anthropic('{"text": "from fallback", "score": 2}')
sleeps = []
async def no_sleep(s):
sleeps.append(s)
gw = build_gateway(settings, transport=httpx.MockTransport(handler))
for c in gw._clients.values(): # keep retries instant
c._sleep = no_sleep
try:
res = await gw.structured("critic", system="s", user="u", schema=Answer)
assert res.parsed.text == "from fallback" and res.via_fallback and res.provider == "anthropic"
ok = await gw.structured("clarify", system="s", user="u", schema=Answer)
assert ok.provider == "anthropic" and not ok.via_fallback
finally:
await gw.aclose()
async def test_build_gateway_only_creates_used_providers(monkeypatch):
from estimator.config.settings import Settings
monkeypatch.setenv("EST_DATABASE__DSN", "sqlite:///:memory:")
monkeypatch.setenv("EST_LLM__ANTHROPIC__API_KEY", "sk-ant")
gw = build_gateway(Settings(), transport=Recorder(ok_openai("{}")).transport)
try:
assert set(gw._clients) == {"anthropic"}
finally:
await gw.aclose()