Init Project Flexly Estimator
This commit is contained in:
@@ -0,0 +1,152 @@
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from estimator.config.sections import RouteCfg, StepRouteCfg
|
||||
from estimator.domain.errors import LlmAuthError, LlmConfigError, LlmRequestError, LlmTransientError
|
||||
from estimator.domain.llm import LlmResult, Usage
|
||||
from estimator.infra.llm import LlmRouter, build_gateway
|
||||
from tests.unit.llm.conftest import Answer, Recorder, ok_anthropic, ok_openai
|
||||
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, name, result=None, error=None):
|
||||
self.name = name
|
||||
self.result = result
|
||||
self.error = error
|
||||
self.calls = []
|
||||
|
||||
async def structured(self, **kw):
|
||||
self.calls.append(kw)
|
||||
if self.error:
|
||||
raise self.error
|
||||
return LlmResult(self.result, Usage(1, 1), self.name, kw["model"])
|
||||
|
||||
|
||||
A = Answer(text="a", score=1)
|
||||
|
||||
|
||||
def route(provider="anthropic", model="m", fallback=None, **kw):
|
||||
return StepRouteCfg(provider=provider, model=model, fallback=fallback, **kw)
|
||||
|
||||
|
||||
async def run(router, step="critic"):
|
||||
return await router.structured(step, system="s", user="u", schema=Answer)
|
||||
|
||||
|
||||
async def test_routes_by_step_and_passes_params():
|
||||
anth, oai = FakeClient("anthropic", A), FakeClient("openai", A)
|
||||
router = LlmRouter(
|
||||
{"clarify": route("anthropic", "claude-x", temperature=0.2, max_tokens=111),
|
||||
"critic": route("openai", "gpt-x", reasoning_effort="low")},
|
||||
{"anthropic": anth, "openai": oai},
|
||||
) # fmt: skip
|
||||
r1 = await run(router, "clarify")
|
||||
r2 = await run(router, "critic")
|
||||
assert r1.provider == "anthropic" and r2.provider == "openai"
|
||||
assert anth.calls[0]["model"] == "claude-x"
|
||||
assert anth.calls[0]["temperature"] == 0.2 and anth.calls[0]["max_tokens"] == 111
|
||||
assert oai.calls[0]["reasoning_effort"] == "low" and oai.calls[0]["temperature"] is None
|
||||
|
||||
|
||||
async def test_unknown_step():
|
||||
router = LlmRouter({}, {})
|
||||
with pytest.raises(LlmConfigError):
|
||||
await run(router, "nope")
|
||||
|
||||
|
||||
async def test_fallback_on_transient_error():
|
||||
primary = FakeClient("openai", error=LlmTransientError("down", provider="openai"))
|
||||
backup = FakeClient("anthropic", A)
|
||||
router = LlmRouter(
|
||||
{"critic": route("openai", "g", fallback=RouteCfg(provider="anthropic", model="c"))},
|
||||
{"openai": primary, "anthropic": backup},
|
||||
)
|
||||
res = await run(router)
|
||||
assert res.provider == "anthropic" and res.via_fallback is True and res.model == "c"
|
||||
assert len(primary.calls) == 1 and len(backup.calls) == 1
|
||||
|
||||
|
||||
async def test_fallback_on_auth_error():
|
||||
router = LlmRouter(
|
||||
{"critic": route("openai", "g", fallback=RouteCfg(provider="anthropic", model="c"))},
|
||||
{"openai": FakeClient("openai", error=LlmAuthError("bad", provider="openai")),
|
||||
"anthropic": FakeClient("anthropic", A)},
|
||||
) # fmt: skip
|
||||
assert (await run(router)).via_fallback is True
|
||||
|
||||
|
||||
async def test_no_fallback_on_bad_request():
|
||||
backup = FakeClient("anthropic", A)
|
||||
router = LlmRouter(
|
||||
{"critic": route("openai", "g", fallback=RouteCfg(provider="anthropic", model="c"))},
|
||||
{"openai": FakeClient("openai", error=LlmRequestError("bad", provider="openai")), "anthropic": backup},
|
||||
)
|
||||
with pytest.raises(LlmRequestError):
|
||||
await run(router)
|
||||
assert backup.calls == []
|
||||
|
||||
|
||||
async def test_error_propagates_without_fallback_configured():
|
||||
router = LlmRouter(
|
||||
{"critic": route("openai", "g")},
|
||||
{"openai": FakeClient("openai", error=LlmTransientError("down"))},
|
||||
)
|
||||
with pytest.raises(LlmTransientError):
|
||||
await run(router)
|
||||
|
||||
|
||||
async def test_primary_success_does_not_touch_fallback():
|
||||
backup = FakeClient("anthropic", A)
|
||||
router = LlmRouter(
|
||||
{"critic": route("openai", "g", fallback=RouteCfg(provider="anthropic", model="c"))},
|
||||
{"openai": FakeClient("openai", A), "anthropic": backup},
|
||||
)
|
||||
res = await run(router)
|
||||
assert res.via_fallback is False and backup.calls == []
|
||||
|
||||
|
||||
async def test_build_gateway_end_to_end_with_both_providers(monkeypatch):
|
||||
"""Settings -> real clients over mock HTTP; critic on OpenAI falls back to Anthropic."""
|
||||
from estimator.config.settings import Settings
|
||||
|
||||
monkeypatch.setenv("EST_DATABASE__DSN", "sqlite:///:memory:")
|
||||
monkeypatch.setenv("EST_LLM__ANTHROPIC__API_KEY", "sk-ant")
|
||||
monkeypatch.setenv("EST_LLM__OPENAI__API_KEY", "sk-oai")
|
||||
monkeypatch.setenv("EST_LLM__STEPS__CRITIC__PROVIDER", "openai")
|
||||
monkeypatch.setenv("EST_LLM__STEPS__CRITIC__MODEL", "gpt-x")
|
||||
monkeypatch.setenv("EST_LLM__STEPS__CRITIC__FALLBACK__PROVIDER", "anthropic")
|
||||
monkeypatch.setenv("EST_LLM__STEPS__CRITIC__FALLBACK__MODEL", "claude-x")
|
||||
settings = Settings()
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if "openai" in request.url.host:
|
||||
return httpx.Response(503, json={"error": {"message": "down"}})
|
||||
return ok_anthropic('{"text": "from fallback", "score": 2}')
|
||||
|
||||
sleeps = []
|
||||
|
||||
async def no_sleep(s):
|
||||
sleeps.append(s)
|
||||
|
||||
gw = build_gateway(settings, transport=httpx.MockTransport(handler))
|
||||
for c in gw._clients.values(): # keep retries instant
|
||||
c._sleep = no_sleep
|
||||
try:
|
||||
res = await gw.structured("critic", system="s", user="u", schema=Answer)
|
||||
assert res.parsed.text == "from fallback" and res.via_fallback and res.provider == "anthropic"
|
||||
ok = await gw.structured("clarify", system="s", user="u", schema=Answer)
|
||||
assert ok.provider == "anthropic" and not ok.via_fallback
|
||||
finally:
|
||||
await gw.aclose()
|
||||
|
||||
|
||||
async def test_build_gateway_only_creates_used_providers(monkeypatch):
|
||||
from estimator.config.settings import Settings
|
||||
|
||||
monkeypatch.setenv("EST_DATABASE__DSN", "sqlite:///:memory:")
|
||||
monkeypatch.setenv("EST_LLM__ANTHROPIC__API_KEY", "sk-ant")
|
||||
gw = build_gateway(Settings(), transport=Recorder(ok_openai("{}")).transport)
|
||||
try:
|
||||
assert set(gw._clients) == {"anthropic"}
|
||||
finally:
|
||||
await gw.aclose()
|
||||
Reference in New Issue
Block a user