Init Project Flexly Estimator

This commit is contained in:
Alexey Dudin
2026-11-03 09:37:44 +03:00
commit 447464c16c
160 changed files with 12779 additions and 0 deletions
View File
+70
View File
@@ -0,0 +1,70 @@
import json
import httpx
import pytest
from pydantic import BaseModel, model_validator
class Answer(BaseModel):
text: str
score: int
@model_validator(mode="after")
def _score_range(self):
if not 0 <= self.score <= 10:
raise ValueError("score must be between 0 and 10")
return self
class Recorder:
"""Scripted httpx transport: each response is consumed in order, requests are recorded."""
def __init__(self, *responses):
self.responses = list(responses)
self.requests: list[httpx.Request] = []
def handler(self, request: httpx.Request) -> httpx.Response:
self.requests.append(request)
item = self.responses.pop(0)
if isinstance(item, Exception):
raise item
return item
@property
def transport(self):
return httpx.MockTransport(self.handler)
def body(self, i: int = 0) -> dict:
return json.loads(self.requests[i].content)
@pytest.fixture
def sleeps():
calls: list[float] = []
async def fake_sleep(s: float) -> None:
calls.append(s)
fake_sleep.calls = calls
return fake_sleep
def ok_openai(content: str, finish="stop", usage=(10, 5)):
return httpx.Response(
200,
json={
"choices": [{"message": {"content": content}, "finish_reason": finish}],
"usage": {"prompt_tokens": usage[0], "completion_tokens": usage[1]},
},
)
def ok_anthropic(text: str, stop="end_turn", usage=(10, 5)):
return httpx.Response(
200,
json={
"content": [{"type": "text", "text": text}],
"stop_reason": stop,
"usage": {"input_tokens": usage[0], "output_tokens": usage[1]},
},
)
+114
View File
@@ -0,0 +1,114 @@
import httpx
import pytest
from estimator.domain.errors import (
LlmAuthError,
LlmOutputError,
LlmRefusalError,
LlmTransientError,
LlmTruncatedError,
)
from estimator.infra.llm import AnthropicClient
from tests.unit.llm.conftest import Answer, Recorder, ok_anthropic
GOOD = '{"text": "hi", "score": 4}'
def client(rec, sleeps, **kw):
return AnthropicClient(
base_url="https://api.anthropic.test",
api_key="sk-ant-secret",
transport=rec.transport,
sleep=sleeps,
rng=lambda: 0.5,
**kw,
)
async def call(c, **kw):
return await c.structured(system="sys", user="usr", schema=Answer, model="claude-x", **kw)
async def test_request_shape_and_parse(sleeps):
rec = Recorder(ok_anthropic(GOOD, usage=(20, 8)))
res = await call(client(rec, sleeps), max_tokens=700)
req = rec.requests[0]
assert str(req.url) == "https://api.anthropic.test/v1/messages"
assert req.headers["x-api-key"] == "sk-ant-secret"
assert req.headers["anthropic-version"] == "2023-06-01"
body = rec.body()
assert body["model"] == "claude-x" and body["max_tokens"] == 700
assert body["system"] == "sys"
assert body["messages"] == [{"role": "user", "content": "usr"}]
fmt = body["output_config"]["format"]
assert fmt["type"] == "json_schema" and fmt["schema"]["additionalProperties"] is False
# forced tool use is rejected by current Claude models, so it must never be sent
assert "tool_choice" not in body and "tools" not in body
assert "temperature" not in body
assert res.parsed == Answer(text="hi", score=4)
assert (res.usage.input_tokens, res.usage.output_tokens) == (20, 8)
assert res.provider == "anthropic"
async def test_custom_api_version_and_temperature(sleeps):
rec = Recorder(ok_anthropic(GOOD))
c = client(rec, sleeps, api_version="2099-01-01")
await call(c, temperature=0.1)
assert rec.requests[0].headers["anthropic-version"] == "2099-01-01"
assert rec.body()["temperature"] == 0.1
async def test_reasoning_effort_is_ignored(sleeps):
rec = Recorder(ok_anthropic(GOOD))
await call(client(rec, sleeps), reasoning_effort="low")
assert "reasoning_effort" not in rec.body()
async def test_multiple_text_blocks_are_joined(sleeps):
r = httpx.Response(
200,
json={
"content": [{"type": "text", "text": '{"text": "a", '}, {"type": "text", "text": '"score": 1}'}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 1, "output_tokens": 1},
},
)
assert (await call(client(Recorder(r), sleeps))).parsed.score == 1
async def test_refusal_and_truncation(sleeps):
with pytest.raises(LlmRefusalError):
await call(client(Recorder(ok_anthropic("", stop="refusal")), sleeps))
with pytest.raises(LlmTruncatedError):
await call(client(Recorder(ok_anthropic('{"te', stop="max_tokens")), sleeps))
async def test_no_text_block(sleeps):
r = httpx.Response(200, json={"content": [], "stop_reason": "end_turn"})
with pytest.raises(LlmOutputError):
await call(client(Recorder(r), sleeps))
async def test_repair_loop(sleeps):
rec = Recorder(ok_anthropic('{"text": "x", "score": -1}'), ok_anthropic(GOOD))
res = await call(client(rec, sleeps, repair_attempts=1))
assert res.parsed.score == 4 and len(rec.requests) == 2
assert "rejected" in rec.body(1)["messages"][0]["content"]
async def test_overloaded_529_is_retried(sleeps):
rec = Recorder(httpx.Response(529, json={"error": {"message": "overloaded"}}), ok_anthropic(GOOD))
assert (await call(client(rec, sleeps))).parsed.score == 4
async def test_persistent_overload_raises_transient(sleeps):
rec = Recorder(*[httpx.Response(529, json={})] * 3)
with pytest.raises(LlmTransientError):
await call(client(rec, sleeps, max_retries=2))
async def test_auth_error(sleeps):
rec = Recorder(httpx.Response(401, json={"error": {"message": "invalid x-api-key"}}))
with pytest.raises(LlmAuthError) as e:
await call(client(rec, sleeps))
assert "sk-ant-secret" not in str(e.value)
+160
View File
@@ -0,0 +1,160 @@
import httpx
import pytest
from estimator.domain.errors import (
LlmAuthError,
LlmOutputError,
LlmRefusalError,
LlmRequestError,
LlmTransientError,
LlmTruncatedError,
)
from estimator.infra.llm import OpenAIChatClient
from tests.unit.llm.conftest import Answer, Recorder, ok_openai
GOOD = '{"text": "hi", "score": 7}'
def client(rec, sleeps, **kw):
return OpenAIChatClient(
base_url="https://api.openai.test/v1",
api_key="sk-secret",
transport=rec.transport,
sleep=sleeps,
rng=lambda: 0.5,
**kw,
)
async def call(c, **kw):
return await c.structured(system="sys", user="usr", schema=Answer, model="m1", **kw)
async def test_request_shape_and_parse(sleeps):
rec = Recorder(ok_openai(GOOD, usage=(12, 3)))
res = await call(client(rec, sleeps), max_tokens=500)
req = rec.requests[0]
assert str(req.url) == "https://api.openai.test/v1/chat/completions"
assert req.headers["authorization"] == "Bearer sk-secret"
body = rec.body()
assert body["model"] == "m1"
assert body["max_completion_tokens"] == 500
assert "max_tokens" not in body and "temperature" not in body and "reasoning_effort" not in body
assert body["messages"][0] == {"role": "system", "content": "sys"}
fmt = body["response_format"]
assert fmt["type"] == "json_schema" and fmt["json_schema"]["strict"] is True
assert fmt["json_schema"]["name"] == "Answer"
assert fmt["json_schema"]["schema"]["additionalProperties"] is False
assert res.parsed == Answer(text="hi", score=7)
assert (res.usage.input_tokens, res.usage.output_tokens) == (12, 3)
assert res.provider == "openai" and res.model == "m1" and not res.via_fallback
async def test_optional_params_sent_only_when_set(sleeps):
rec = Recorder(ok_openai(GOOD))
await call(client(rec, sleeps), temperature=0.3, reasoning_effort="low")
body = rec.body()
assert body["temperature"] == 0.3 and body["reasoning_effort"] == "low"
async def test_code_fence_is_tolerated(sleeps):
rec = Recorder(ok_openai("```json\n" + GOOD + "\n```"))
assert (await call(client(rec, sleeps))).parsed.score == 7
async def test_repair_loop_feeds_back_the_error(sleeps):
rec = Recorder(ok_openai('{"text": "x", "score": 99}'), ok_openai(GOOD))
res = await call(client(rec, sleeps, repair_attempts=1))
assert res.parsed.score == 7
assert len(rec.requests) == 2
second_user = rec.body(1)["messages"][1]["content"]
assert "rejected" in second_user and "score" in second_user
assert res.usage.input_tokens == 20 # usage of both attempts is summed
async def test_repair_exhausted_raises_output_error(sleeps):
rec = Recorder(ok_openai("not json"), ok_openai("still not"))
with pytest.raises(LlmOutputError):
await call(client(rec, sleeps, repair_attempts=1))
assert len(rec.requests) == 2
async def test_no_repair_when_disabled(sleeps):
rec = Recorder(ok_openai("nope"))
with pytest.raises(LlmOutputError):
await call(client(rec, sleeps, repair_attempts=0))
assert len(rec.requests) == 1
async def test_refusal(sleeps):
r = httpx.Response(200, json={"choices": [{"message": {"content": None, "refusal": "no"}, "finish_reason": "stop"}]})
with pytest.raises(LlmRefusalError):
await call(client(Recorder(r), sleeps))
async def test_truncated(sleeps):
with pytest.raises(LlmTruncatedError):
await call(client(Recorder(ok_openai('{"te', finish="length")), sleeps))
async def test_auth_error_not_retried_and_key_not_leaked(sleeps):
rec = Recorder(httpx.Response(401, json={"error": {"message": "bad key"}}))
with pytest.raises(LlmAuthError) as e:
await call(client(rec, sleeps))
assert len(rec.requests) == 1
assert "sk-secret" not in str(e.value)
async def test_429_retries_with_retry_after_then_succeeds(sleeps):
rec = Recorder(
httpx.Response(429, headers={"retry-after": "3"}, json={"error": {"message": "slow down"}}),
ok_openai(GOOD),
)
res = await call(client(rec, sleeps, max_retries=2))
assert res.parsed.score == 7
assert sleeps.calls == [3.0]
async def test_retry_after_is_capped(sleeps):
rec = Recorder(httpx.Response(429, headers={"retry-after": "9999"}, json={}), ok_openai(GOOD))
await call(client(rec, sleeps))
assert sleeps.calls == [30.0]
async def test_5xx_exhausts_retries(sleeps):
rec = Recorder(*[httpx.Response(503, json={"error": {"message": "down"}})] * 3)
with pytest.raises(LlmTransientError):
await call(client(rec, sleeps, max_retries=2))
assert len(rec.requests) == 3 and len(sleeps.calls) == 2
async def test_backoff_grows(sleeps):
rec = Recorder(*[httpx.Response(500, json={})] * 3)
with pytest.raises(LlmTransientError):
await call(client(rec, sleeps, max_retries=2))
assert sleeps.calls[1] > sleeps.calls[0]
async def test_network_error_retried(sleeps):
rec = Recorder(httpx.ConnectError("boom"), ok_openai(GOOD))
assert (await call(client(rec, sleeps))).parsed.score == 7
async def test_network_error_exhausted(sleeps):
rec = Recorder(*[httpx.ReadTimeout("slow")] * 3)
with pytest.raises(LlmTransientError):
await call(client(rec, sleeps, max_retries=2))
async def test_400_is_request_error_without_retry(sleeps):
rec = Recorder(httpx.Response(400, json={"error": {"message": "bad schema"}}))
with pytest.raises(LlmRequestError, match="bad schema"):
await call(client(rec, sleeps))
assert len(rec.requests) == 1
async def test_quota_exhausted_is_not_retried(sleeps):
rec = Recorder(httpx.Response(429, json={"error": {"code": "insufficient_quota", "message": "q"}}))
with pytest.raises(LlmRequestError, match="quota"):
await call(client(rec, sleeps))
assert len(rec.requests) == 1
+152
View File
@@ -0,0 +1,152 @@
import httpx
import pytest
from estimator.config.sections import RouteCfg, StepRouteCfg
from estimator.domain.errors import LlmAuthError, LlmConfigError, LlmRequestError, LlmTransientError
from estimator.domain.llm import LlmResult, Usage
from estimator.infra.llm import LlmRouter, build_gateway
from tests.unit.llm.conftest import Answer, Recorder, ok_anthropic, ok_openai
class FakeClient:
def __init__(self, name, result=None, error=None):
self.name = name
self.result = result
self.error = error
self.calls = []
async def structured(self, **kw):
self.calls.append(kw)
if self.error:
raise self.error
return LlmResult(self.result, Usage(1, 1), self.name, kw["model"])
A = Answer(text="a", score=1)
def route(provider="anthropic", model="m", fallback=None, **kw):
return StepRouteCfg(provider=provider, model=model, fallback=fallback, **kw)
async def run(router, step="critic"):
return await router.structured(step, system="s", user="u", schema=Answer)
async def test_routes_by_step_and_passes_params():
anth, oai = FakeClient("anthropic", A), FakeClient("openai", A)
router = LlmRouter(
{"clarify": route("anthropic", "claude-x", temperature=0.2, max_tokens=111),
"critic": route("openai", "gpt-x", reasoning_effort="low")},
{"anthropic": anth, "openai": oai},
) # fmt: skip
r1 = await run(router, "clarify")
r2 = await run(router, "critic")
assert r1.provider == "anthropic" and r2.provider == "openai"
assert anth.calls[0]["model"] == "claude-x"
assert anth.calls[0]["temperature"] == 0.2 and anth.calls[0]["max_tokens"] == 111
assert oai.calls[0]["reasoning_effort"] == "low" and oai.calls[0]["temperature"] is None
async def test_unknown_step():
router = LlmRouter({}, {})
with pytest.raises(LlmConfigError):
await run(router, "nope")
async def test_fallback_on_transient_error():
primary = FakeClient("openai", error=LlmTransientError("down", provider="openai"))
backup = FakeClient("anthropic", A)
router = LlmRouter(
{"critic": route("openai", "g", fallback=RouteCfg(provider="anthropic", model="c"))},
{"openai": primary, "anthropic": backup},
)
res = await run(router)
assert res.provider == "anthropic" and res.via_fallback is True and res.model == "c"
assert len(primary.calls) == 1 and len(backup.calls) == 1
async def test_fallback_on_auth_error():
router = LlmRouter(
{"critic": route("openai", "g", fallback=RouteCfg(provider="anthropic", model="c"))},
{"openai": FakeClient("openai", error=LlmAuthError("bad", provider="openai")),
"anthropic": FakeClient("anthropic", A)},
) # fmt: skip
assert (await run(router)).via_fallback is True
async def test_no_fallback_on_bad_request():
backup = FakeClient("anthropic", A)
router = LlmRouter(
{"critic": route("openai", "g", fallback=RouteCfg(provider="anthropic", model="c"))},
{"openai": FakeClient("openai", error=LlmRequestError("bad", provider="openai")), "anthropic": backup},
)
with pytest.raises(LlmRequestError):
await run(router)
assert backup.calls == []
async def test_error_propagates_without_fallback_configured():
router = LlmRouter(
{"critic": route("openai", "g")},
{"openai": FakeClient("openai", error=LlmTransientError("down"))},
)
with pytest.raises(LlmTransientError):
await run(router)
async def test_primary_success_does_not_touch_fallback():
backup = FakeClient("anthropic", A)
router = LlmRouter(
{"critic": route("openai", "g", fallback=RouteCfg(provider="anthropic", model="c"))},
{"openai": FakeClient("openai", A), "anthropic": backup},
)
res = await run(router)
assert res.via_fallback is False and backup.calls == []
async def test_build_gateway_end_to_end_with_both_providers(monkeypatch):
"""Settings -> real clients over mock HTTP; critic on OpenAI falls back to Anthropic."""
from estimator.config.settings import Settings
monkeypatch.setenv("EST_DATABASE__DSN", "sqlite:///:memory:")
monkeypatch.setenv("EST_LLM__ANTHROPIC__API_KEY", "sk-ant")
monkeypatch.setenv("EST_LLM__OPENAI__API_KEY", "sk-oai")
monkeypatch.setenv("EST_LLM__STEPS__CRITIC__PROVIDER", "openai")
monkeypatch.setenv("EST_LLM__STEPS__CRITIC__MODEL", "gpt-x")
monkeypatch.setenv("EST_LLM__STEPS__CRITIC__FALLBACK__PROVIDER", "anthropic")
monkeypatch.setenv("EST_LLM__STEPS__CRITIC__FALLBACK__MODEL", "claude-x")
settings = Settings()
def handler(request: httpx.Request) -> httpx.Response:
if "openai" in request.url.host:
return httpx.Response(503, json={"error": {"message": "down"}})
return ok_anthropic('{"text": "from fallback", "score": 2}')
sleeps = []
async def no_sleep(s):
sleeps.append(s)
gw = build_gateway(settings, transport=httpx.MockTransport(handler))
for c in gw._clients.values(): # keep retries instant
c._sleep = no_sleep
try:
res = await gw.structured("critic", system="s", user="u", schema=Answer)
assert res.parsed.text == "from fallback" and res.via_fallback and res.provider == "anthropic"
ok = await gw.structured("clarify", system="s", user="u", schema=Answer)
assert ok.provider == "anthropic" and not ok.via_fallback
finally:
await gw.aclose()
async def test_build_gateway_only_creates_used_providers(monkeypatch):
from estimator.config.settings import Settings
monkeypatch.setenv("EST_DATABASE__DSN", "sqlite:///:memory:")
monkeypatch.setenv("EST_LLM__ANTHROPIC__API_KEY", "sk-ant")
gw = build_gateway(Settings(), transport=Recorder(ok_openai("{}")).transport)
try:
assert set(gw._clients) == {"anthropic"}
finally:
await gw.aclose()
+69
View File
@@ -0,0 +1,69 @@
from pydantic import BaseModel, Field
from estimator.domain.enums import Role
from estimator.infra.llm.schema import to_strict_schema
class Inner(BaseModel):
title: str # a property named like a JSON Schema keyword must survive
role: Role
hours: float = Field(ge=0, le=100, default=1.0)
class Outer(BaseModel):
items: list[Inner]
note: str | None = None
def _objects(node):
if isinstance(node, dict):
if node.get("type") == "object":
yield node
for v in node.values():
yield from _objects(v)
elif isinstance(node, list):
for v in node:
yield from _objects(v)
def test_refs_are_inlined():
schema = to_strict_schema(Outer)
assert "$ref" not in str(schema) and "$defs" not in schema
def test_every_object_is_closed_and_fully_required():
schema = to_strict_schema(Outer)
objs = list(_objects(schema))
assert len(objs) == 2
for o in objs:
assert o["additionalProperties"] is False
assert o["required"] == list(o["properties"].keys())
def test_unsupported_keywords_stripped_but_property_names_kept():
schema = to_strict_schema(Outer)
inner = schema["properties"]["items"]["items"]
assert "title" in inner["properties"] # the field, not the keyword
hours = inner["properties"]["hours"]
assert "minimum" not in hours and "maximum" not in hours and "default" not in hours
def test_enum_is_inlined():
inner = to_strict_schema(Outer)["properties"]["items"]["items"]
assert "analyst" in inner["properties"]["role"]["enum"]
def test_nullable_field_kept_as_anyof():
note = to_strict_schema(Outer)["properties"]["note"]
assert {"type": "null"} in note["anyOf"]
def test_recursive_schema_rejected():
import pytest
class Node(BaseModel):
children: list["Node"]
Node.model_rebuild()
with pytest.raises(ValueError):
to_strict_schema(Node)