Init Project Flexly Estimator

This commit is contained in:
Alexey Dudin
2026-11-03 09:37:44 +03:00
commit 447464c16c
160 changed files with 12779 additions and 0 deletions
@@ -0,0 +1,170 @@
"""Whole stack with only the network faked: Settings -> router -> real OpenAI and Anthropic
clients -> pipeline. Two providers, per-step routing, and a fallback after an outage."""
import json
from pathlib import Path
import httpx
import pytest
from estimator.agent import EstimationPipeline, PipelineInput, PromptStore
from estimator.config.settings import Settings
from estimator.infra.llm import build_gateway
from estimator.pricing import price
from tests.unit.agent.conftest import (
DOC_WBS,
clarify_out,
critic_out,
decompose_out,
estimate_out,
missing,
)
ROOT = Path(__file__).resolve().parents[2]
CANNED = {
"ClarifyOutput": clarify_out(),
"DecomposeOutput": decompose_out(),
"EstimateOutput": estimate_out(),
"CriticOutput": critic_out(missing=[missing("Документация")]),
}
class FakeProviders:
"""Answers like OpenAI and Anthropic, keyed by the schema the request asks for."""
def __init__(self, openai_down=False):
self.openai_down = openai_down
self.seen: list[tuple[str, str, str]] = [] # (provider, schema, model)
def __call__(self, request: httpx.Request) -> httpx.Response:
body = json.loads(request.content)
if request.url.host == "api.openai.com":
schema = body["response_format"]["json_schema"]["name"]
self.seen.append(("openai", schema, body["model"]))
if self.openai_down:
return httpx.Response(503, json={"error": {"message": "down"}})
text = CANNED[schema].model_dump_json()
return httpx.Response(
200,
json={
"choices": [{"message": {"content": text}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 1000, "completion_tokens": 200},
},
)
# Anthropic requests do not name the schema: recognise it by its content
wire = json.dumps(body["output_config"]["format"]["schema"])
if '"missing_tasks"' in wire:
schema = "CriticOutput"
elif '"task_id"' in wire:
schema = "EstimateOutput"
elif '"summary"' in wire:
schema = "ClarifyOutput"
else:
schema = "DecomposeOutput"
self.seen.append(("anthropic", schema, body["model"]))
return httpx.Response(
200,
json={
"content": [{"type": "text", "text": CANNED[schema].model_dump_json()}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 900, "output_tokens": 150},
},
)
def settings(monkeypatch, **env):
base = {
"EST_DATABASE__DSN": "sqlite:///:memory:",
"EST_LLM__ANTHROPIC__API_KEY": "sk-ant",
"EST_LLM__OPENAI__API_KEY": "sk-oai",
# estimate and critic run on OpenAI, critic falls back to Anthropic
"EST_LLM__STEPS__ESTIMATE__PROVIDER": "openai",
"EST_LLM__STEPS__ESTIMATE__MODEL": "gpt-test",
"EST_LLM__STEPS__CRITIC__PROVIDER": "openai",
"EST_LLM__STEPS__CRITIC__MODEL": "gpt-test",
"EST_LLM__STEPS__CRITIC__FALLBACK__PROVIDER": "anthropic",
"EST_LLM__STEPS__CRITIC__FALLBACK__MODEL": "claude-fallback",
}
base.update(env)
for k, v in base.items():
monkeypatch.setenv(k, v)
return Settings()
async def run(monkeypatch, providers, **env):
cfg = settings(monkeypatch, **env)
gw = build_gateway(cfg, transport=httpx.MockTransport(providers))
async def instant(_): # no real waiting between retries
return None
for c in gw._clients.values():
c._sleep = instant
try:
pipe = EstimationPipeline(gw, cfg.agent, PromptStore(ROOT / "prompts"))
return await pipe.run(PipelineInput(requirements="Бот Telegram+MAX+VK с ИИ и amoCRM"))
finally:
await gw.aclose()
async def test_two_providers_in_one_run(monkeypatch):
prov = FakeProviders()
res = await run(monkeypatch, prov)
assert res.status == "done"
assert [(p, s) for p, s, _ in prov.seen] == [
("anthropic", "ClarifyOutput"),
("anthropic", "DecomposeOutput"),
("openai", "EstimateOutput"),
("openai", "CriticOutput"),
]
assert [s.provider for s in res.steps] == ["anthropic", "anthropic", "openai", "openai"]
assert res.steps[2].model == "gpt-test" and not any(s.via_fallback for s in res.steps)
assert len(res.tasks) == len(DOC_WBS) + 1 # critic added documentation
assert res.total_tokens == (900 + 900 + 1000 + 1000, 150 + 150 + 200 + 200)
p = price(res.tasks, {"analyst": 1500, "backend": 1800, "qa": 1200, "devops": 1800})
assert p.hours > 144.7 # the doc example plus the critic's task
async def test_openai_outage_falls_back_for_the_critic_only(monkeypatch):
prov = FakeProviders(openai_down=True)
# estimate stays on Anthropic so only the critic depends on the broken provider
res = await run(
monkeypatch,
prov,
EST_LLM__STEPS__ESTIMATE__PROVIDER="anthropic",
EST_LLM__STEPS__ESTIMATE__MODEL="claude-x",
)
critic_log = res.steps[-1]
assert critic_log.step == "critic" and critic_log.via_fallback
assert critic_log.provider == "anthropic" and critic_log.model == "claude-fallback"
assert any(p == "openai" for p, _, _ in prov.seen) # it did try OpenAI first
assert res.status == "done"
async def test_strict_schema_is_what_goes_over_the_wire(monkeypatch):
seen = []
def handler(request: httpx.Request) -> httpx.Response:
seen.append(json.loads(request.content))
return FakeProviders()(request)
cfg = settings(monkeypatch)
gw = build_gateway(cfg, transport=httpx.MockTransport(handler))
try:
await EstimationPipeline(gw, cfg.agent, PromptStore(ROOT / "prompts")).run(
PipelineInput(requirements="бот")
)
finally:
await gw.aclose()
for body in seen:
schema = body.get("response_format", {}).get("json_schema", {}).get("schema") or body["output_config"]["format"]["schema"]
assert schema["additionalProperties"] is False and "$defs" not in schema
assert pytest.approx(len(seen)) == 4
async def test_outage_without_fallback_is_not_hidden(monkeypatch):
"""Estimate has no fallback: an OpenAI outage must surface, not return a partial estimate."""
from estimator.domain.errors import LlmTransientError
with pytest.raises(LlmTransientError):
await run(monkeypatch, FakeProviders(openai_down=True))