Init Project Flexly Estimator

This commit is contained in:
Alexey Dudin
2026-11-03 09:37:44 +03:00
commit 447464c16c
160 changed files with 12779 additions and 0 deletions
+152
View File
@@ -0,0 +1,152 @@
import httpx
import pytest
from estimator.config.sections import RouteCfg, StepRouteCfg
from estimator.domain.errors import LlmAuthError, LlmConfigError, LlmRequestError, LlmTransientError
from estimator.domain.llm import LlmResult, Usage
from estimator.infra.llm import LlmRouter, build_gateway
from tests.unit.llm.conftest import Answer, Recorder, ok_anthropic, ok_openai
class FakeClient:
def __init__(self, name, result=None, error=None):
self.name = name
self.result = result
self.error = error
self.calls = []
async def structured(self, **kw):
self.calls.append(kw)
if self.error:
raise self.error
return LlmResult(self.result, Usage(1, 1), self.name, kw["model"])
A = Answer(text="a", score=1)
def route(provider="anthropic", model="m", fallback=None, **kw):
return StepRouteCfg(provider=provider, model=model, fallback=fallback, **kw)
async def run(router, step="critic"):
return await router.structured(step, system="s", user="u", schema=Answer)
async def test_routes_by_step_and_passes_params():
anth, oai = FakeClient("anthropic", A), FakeClient("openai", A)
router = LlmRouter(
{"clarify": route("anthropic", "claude-x", temperature=0.2, max_tokens=111),
"critic": route("openai", "gpt-x", reasoning_effort="low")},
{"anthropic": anth, "openai": oai},
) # fmt: skip
r1 = await run(router, "clarify")
r2 = await run(router, "critic")
assert r1.provider == "anthropic" and r2.provider == "openai"
assert anth.calls[0]["model"] == "claude-x"
assert anth.calls[0]["temperature"] == 0.2 and anth.calls[0]["max_tokens"] == 111
assert oai.calls[0]["reasoning_effort"] == "low" and oai.calls[0]["temperature"] is None
async def test_unknown_step():
router = LlmRouter({}, {})
with pytest.raises(LlmConfigError):
await run(router, "nope")
async def test_fallback_on_transient_error():
primary = FakeClient("openai", error=LlmTransientError("down", provider="openai"))
backup = FakeClient("anthropic", A)
router = LlmRouter(
{"critic": route("openai", "g", fallback=RouteCfg(provider="anthropic", model="c"))},
{"openai": primary, "anthropic": backup},
)
res = await run(router)
assert res.provider == "anthropic" and res.via_fallback is True and res.model == "c"
assert len(primary.calls) == 1 and len(backup.calls) == 1
async def test_fallback_on_auth_error():
router = LlmRouter(
{"critic": route("openai", "g", fallback=RouteCfg(provider="anthropic", model="c"))},
{"openai": FakeClient("openai", error=LlmAuthError("bad", provider="openai")),
"anthropic": FakeClient("anthropic", A)},
) # fmt: skip
assert (await run(router)).via_fallback is True
async def test_no_fallback_on_bad_request():
backup = FakeClient("anthropic", A)
router = LlmRouter(
{"critic": route("openai", "g", fallback=RouteCfg(provider="anthropic", model="c"))},
{"openai": FakeClient("openai", error=LlmRequestError("bad", provider="openai")), "anthropic": backup},
)
with pytest.raises(LlmRequestError):
await run(router)
assert backup.calls == []
async def test_error_propagates_without_fallback_configured():
router = LlmRouter(
{"critic": route("openai", "g")},
{"openai": FakeClient("openai", error=LlmTransientError("down"))},
)
with pytest.raises(LlmTransientError):
await run(router)
async def test_primary_success_does_not_touch_fallback():
backup = FakeClient("anthropic", A)
router = LlmRouter(
{"critic": route("openai", "g", fallback=RouteCfg(provider="anthropic", model="c"))},
{"openai": FakeClient("openai", A), "anthropic": backup},
)
res = await run(router)
assert res.via_fallback is False and backup.calls == []
async def test_build_gateway_end_to_end_with_both_providers(monkeypatch):
"""Settings -> real clients over mock HTTP; critic on OpenAI falls back to Anthropic."""
from estimator.config.settings import Settings
monkeypatch.setenv("EST_DATABASE__DSN", "sqlite:///:memory:")
monkeypatch.setenv("EST_LLM__ANTHROPIC__API_KEY", "sk-ant")
monkeypatch.setenv("EST_LLM__OPENAI__API_KEY", "sk-oai")
monkeypatch.setenv("EST_LLM__STEPS__CRITIC__PROVIDER", "openai")
monkeypatch.setenv("EST_LLM__STEPS__CRITIC__MODEL", "gpt-x")
monkeypatch.setenv("EST_LLM__STEPS__CRITIC__FALLBACK__PROVIDER", "anthropic")
monkeypatch.setenv("EST_LLM__STEPS__CRITIC__FALLBACK__MODEL", "claude-x")
settings = Settings()
def handler(request: httpx.Request) -> httpx.Response:
if "openai" in request.url.host:
return httpx.Response(503, json={"error": {"message": "down"}})
return ok_anthropic('{"text": "from fallback", "score": 2}')
sleeps = []
async def no_sleep(s):
sleeps.append(s)
gw = build_gateway(settings, transport=httpx.MockTransport(handler))
for c in gw._clients.values(): # keep retries instant
c._sleep = no_sleep
try:
res = await gw.structured("critic", system="s", user="u", schema=Answer)
assert res.parsed.text == "from fallback" and res.via_fallback and res.provider == "anthropic"
ok = await gw.structured("clarify", system="s", user="u", schema=Answer)
assert ok.provider == "anthropic" and not ok.via_fallback
finally:
await gw.aclose()
async def test_build_gateway_only_creates_used_providers(monkeypatch):
from estimator.config.settings import Settings
monkeypatch.setenv("EST_DATABASE__DSN", "sqlite:///:memory:")
monkeypatch.setenv("EST_LLM__ANTHROPIC__API_KEY", "sk-ant")
gw = build_gateway(Settings(), transport=Recorder(ok_openai("{}")).transport)
try:
assert set(gw._clients) == {"anthropic"}
finally:
await gw.aclose()