Init Project Flexly Estimator

This commit is contained in:
Alexey Dudin
2026-11-03 09:37:44 +03:00
commit 447464c16c
160 changed files with 12779 additions and 0 deletions
View File
View File
View File
+79
View File
@@ -0,0 +1,79 @@
"""The Alembic migration must produce exactly the schema the ORM models describe."""
import asyncio
import os
import pytest
from alembic import command
from alembic.autogenerate import compare_metadata
from alembic.config import Config
from alembic.migration import MigrationContext
from sqlalchemy.ext.asyncio import create_async_engine
from estimator.infra.db import Base, normalize_dsn
PG = os.getenv("TEST_PG_DSN")
def _alembic(dsn: str, *cmd) -> None:
cfg = Config("alembic.ini")
cfg.cmd_opts = type("O", (), {"x": [f"dsn={dsn}"]})()
getattr(command, cmd[0])(cfg, *cmd[1:])
async def _diff(dsn: str):
engine = create_async_engine(normalize_dsn(dsn))
try:
async with engine.connect() as conn:
def run(sync_conn):
ctx = MigrationContext.configure(sync_conn, opts={"compare_type": True})
return compare_metadata(ctx, Base.metadata)
return await conn.run_sync(run)
finally:
await engine.dispose()
async def _tables(dsn: str) -> set[str]:
engine = create_async_engine(normalize_dsn(dsn))
try:
async with engine.connect() as conn:
from sqlalchemy import inspect
return set(await conn.run_sync(lambda c: inspect(c).get_table_names()))
finally:
await engine.dispose()
def _drop_pg(dsn):
import psycopg
with psycopg.connect(dsn, autocommit=True) as c:
c.execute("drop table if exists estimate_versions, estimates, alembic_version cascade")
@pytest.fixture(params=["sqlite", "postgres"])
def fresh_dsn(request, tmp_path):
if request.param == "postgres":
if not PG:
pytest.skip("set TEST_PG_DSN to run on PostgreSQL")
_drop_pg(PG)
yield PG
_drop_pg(PG)
else:
yield f"sqlite:///{tmp_path / 'mig.db'}"
async def test_upgrade_matches_models_and_downgrade_is_clean(fresh_dsn):
await asyncio.to_thread(_alembic, fresh_dsn, "upgrade", "head")
assert {"estimates", "estimate_versions"} <= await _tables(fresh_dsn)
assert await _diff(fresh_dsn) == [] # no drift between migration and models
await asyncio.to_thread(_alembic, fresh_dsn, "downgrade", "base")
assert await _tables(fresh_dsn) <= {"alembic_version"}
async def test_upgrade_is_idempotent(fresh_dsn):
await asyncio.to_thread(_alembic, fresh_dsn, "upgrade", "head")
await asyncio.to_thread(_alembic, fresh_dsn, "upgrade", "head")
assert await _diff(fresh_dsn) == []
+226
View File
@@ -0,0 +1,226 @@
"""Repository behaviour on SQLite and, when TEST_PG_DSN is set, on PostgreSQL."""
import asyncio
from datetime import UTC
import pytest
from estimator.config.sections import DatabaseCfg
from estimator.domain.enums import Region
from estimator.domain.errors import EstimateNotFound, IdempotencyConflict, InvalidRequestError
from estimator.domain.estimates import EstimateStatus, NewVersion, VersionSource
from estimator.domain.models import AnsweredQuestion
from estimator.infra.db import SqlEstimateRepo, make_engine, make_session_factory
@pytest.fixture
async def repo(backend_dsn):
engine = make_engine(DatabaseCfg(dsn=backend_dsn))
yield SqlEstimateRepo(make_session_factory(engine))
await engine.dispose()
def new(text="бот", **kw) -> NewVersion:
return NewVersion(requirements=text, region=Region.RU, currency="RUB", **kw)
async def test_create_and_get_roundtrip(repo):
data = new(
"бот для Telegram",
answers=[AnsweredQuestion(question="CRM?", answer="amoCRM")],
proceed_with_assumptions=True,
)
eid, version, created = await repo.create(client="a", idempotency_key=None, request_hash="h", data=data)
assert created and version == 1
rec = await repo.get("a", eid)
assert rec.requirements == "бот для Telegram" and rec.status == EstimateStatus.PENDING
assert rec.answers == [AnsweredQuestion(question="CRM?", answer="amoCRM")]
assert rec.proceed_with_assumptions and rec.region == Region.RU
assert rec.created_at.tzinfo is not None and rec.created_at.utcoffset().total_seconds() == 0
assert rec.latest_version == 1 and rec.estimate_created_at == rec.created_at
async def test_other_client_cannot_read_or_extend(repo):
eid, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new())
with pytest.raises(EstimateNotFound):
await repo.get("b", eid)
with pytest.raises(EstimateNotFound):
await repo.add_version(client="b", estimate_id=eid, data=new())
with pytest.raises(EstimateNotFound):
await repo.get("a", "does-not-exist")
async def test_idempotency_same_request_returns_original(repo):
a = await repo.create(client="a", idempotency_key="k1", request_hash="h", data=new())
b = await repo.create(client="a", idempotency_key="k1", request_hash="h", data=new())
assert a[0] == b[0] and a[2] is True and b[2] is False
async def test_idempotency_different_request_conflicts(repo):
await repo.create(client="a", idempotency_key="k1", request_hash="h1", data=new())
with pytest.raises(IdempotencyConflict):
await repo.create(client="a", idempotency_key="k1", request_hash="h2", data=new())
async def test_idempotency_key_is_per_client(repo):
a = await repo.create(client="a", idempotency_key="k1", request_hash="h", data=new())
b = await repo.create(client="b", idempotency_key="k1", request_hash="h", data=new())
assert a[0] != b[0] and a[2] and b[2]
async def test_idempotency_race_creates_one_estimate(repo):
results = await asyncio.gather(
*[repo.create(client="a", idempotency_key="race", request_hash="h", data=new()) for _ in range(6)]
)
assert len({r[0] for r in results}) == 1
assert sum(1 for r in results if r[2]) == 1
async def test_without_key_every_create_is_new(repo):
a = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new())
b = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new())
assert a[0] != b[0]
async def test_add_version_increments_and_keeps_history(repo):
eid, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new("v1 text"))
v2 = await repo.add_version(
client="a", estimate_id=eid, data=new("v2 text", source=VersionSource.RECALC)
)
assert v2 == 2
assert (await repo.get("a", eid)).requirements == "v2 text" # latest by default
assert (await repo.get("a", eid, 1)).requirements == "v1 text" # history is kept
assert (await repo.get("a", eid, 2)).latest_version == 2
with pytest.raises(EstimateNotFound):
await repo.get("a", eid, 9)
async def test_concurrent_add_version_gets_distinct_numbers(repo):
eid, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new())
numbers = await asyncio.gather(
*[repo.add_version(client="a", estimate_id=eid, data=new(f"t{i}")) for i in range(8)]
)
assert sorted(numbers) == list(range(2, 10))
assert (await repo.get("a", eid)).latest_version == 9
async def test_update_version_stores_json_and_times(repo):
from estimator.infra.db.types import utcnow
eid, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new())
now = utcnow()
await repo.update_version(
eid, 1,
status="done", step=None, market_version="2026Q4",
result={"summary": "итог", "tasks": [{"id": "t1", "hours": 1.5}], "ключ": "значение"},
price={"cost_with_reserve": 370000.0}, finished_at=now,
) # fmt: skip
rec = await repo.get_by_id(eid, 1)
assert rec.status == EstimateStatus.DONE
assert rec.result["tasks"][0]["hours"] == 1.5 and rec.result["ключ"] == "значение"
assert rec.price == {"cost_with_reserve": 370000.0}
assert rec.finished_at.tzinfo is not None and abs((rec.finished_at - now).total_seconds()) < 1
assert rec.finished_at.tzinfo == UTC or rec.finished_at.utcoffset().total_seconds() == 0
async def test_update_version_rejects_immutable_fields(repo):
eid, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new())
with pytest.raises(ValueError):
await repo.update_version(eid, 1, requirements="changed")
with pytest.raises(ValueError):
await repo.update_version(eid, 1, client="evil")
async def test_list_pages_newest_first_without_gaps_or_repeats(repo):
ids = []
for i in range(5):
eid, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new(f"n{i}"))
ids.append(eid)
await repo.create(client="other", idempotency_key=None, request_hash="h", data=new("foreign"))
seen, cursor, pages = [], None, 0
while True:
page = await repo.list_page("a", status=None, limit=2, cursor=cursor)
seen += [i.id for i in page.items]
pages += 1
cursor = page.next_cursor
if not cursor:
break
assert pages == 3 and seen == list(reversed(ids))
async def test_list_exact_page_has_no_dangling_cursor(repo):
for i in range(2):
await repo.create(client="a", idempotency_key=None, request_hash="h", data=new(f"n{i}"))
page = await repo.list_page("a", status=None, limit=2, cursor=None)
assert len(page.items) == 2 and page.next_cursor is None
async def test_list_shows_latest_version_and_filters_by_status(repo):
e1, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new("one"))
e2, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new("two"))
await repo.update_version(
e1, 1, status="done", result={"summary": "Готово"}, price={"cost_with_reserve": 123000.0, "cost_expected": 100000.0}
)
await repo.add_version(client="a", estimate_id=e2, data=new("two again", status=EstimateStatus.NEEDS_INPUT))
done = await repo.list_page("a", status=EstimateStatus.DONE, limit=10, cursor=None)
assert [i.id for i in done.items] == [e1]
assert done.items[0].summary == "Готово" and done.items[0].cost == 123000.0
waiting = await repo.list_page("a", status=EstimateStatus.NEEDS_INPUT, limit=10, cursor=None)
assert [(i.id, i.version) for i in waiting.items] == [(e2, 2)] # filter looks at the latest version
async def test_invalid_cursor(repo):
with pytest.raises(InvalidRequestError):
await repo.list_page("a", status=None, limit=5, cursor="not-a-cursor")
async def test_active_versions_for_recovery(repo):
e1, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new())
e2, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new())
e3, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new())
await repo.update_version(e2, 1, status="running")
await repo.update_version(e3, 1, status="done")
active = await repo.active_versions()
assert {(i, v, s.value) for i, v, s in active} == {(e1, 1, "pending"), (e2, 1, "running")}
async def test_engagement_is_stored_per_version_and_defaults_to_fixed_price(repo):
from estimator.domain.enums import Engagement
eid, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new())
assert (await repo.get("a", eid)).engagement == Engagement.FIXED_PRICE
v2 = await repo.add_version(
client="a", estimate_id=eid,
data=new(engagement=Engagement.TIME_AND_MATERIALS, source=VersionSource.REPRICE),
) # fmt: skip
assert (await repo.get("a", eid, v2)).engagement == Engagement.TIME_AND_MATERIALS
assert (await repo.get("a", eid, 1)).engagement == Engagement.FIXED_PRICE
assert (await repo.get("a", eid, v2)).source == VersionSource.REPRICE
async def test_list_amount_follows_the_engagement_model(repo):
from estimator.domain.enums import Engagement
price = {"cost_with_reserve": 370000.0, "cost_expected": 336000.0}
fixed, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new("fp"))
tm, *_ = await repo.create(
client="a", idempotency_key=None, request_hash="h",
data=new("tm", engagement=Engagement.TIME_AND_MATERIALS),
) # fmt: skip
await repo.update_version(fixed, 1, status="done", price=price)
await repo.update_version(tm, 1, status="done", price=price)
items = {i.id: i for i in (await repo.list_page("a", status=None, limit=10, cursor=None)).items}
assert items[fixed].cost == 370000.0 and items[fixed].engagement == Engagement.FIXED_PRICE
assert items[tm].cost == 336000.0 and items[tm].engagement == Engagement.TIME_AND_MATERIALS
async def test_vat_rate_is_stored_per_version(repo):
eid, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new(vat_rate=0.22))
assert (await repo.get("a", eid)).vat_rate == pytest.approx(0.22)
v2 = await repo.add_version(client="a", estimate_id=eid, data=new(vat_rate=0.0))
assert (await repo.get("a", eid, v2)).vat_rate == 0.0
assert (await repo.get("a", eid, 1)).vat_rate == pytest.approx(0.22)
other, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new())
assert (await repo.get("a", other)).vat_rate == 0.0 # the default of a version is no VAT
+50
View File
@@ -0,0 +1,50 @@
"""Real provider calls. Skipped unless EST_LIVE_LLM=1 and the keys are set.
EST_LIVE_LLM=1 EST_LLM__ANTHROPIC__API_KEY=... EST_LLM__OPENAI__API_KEY=... \
pytest tests/integration/test_llm_live.py -m live -s
Set the OpenAI model in EST_LIVE_OPENAI_MODEL and the Anthropic one in EST_LIVE_ANTHROPIC_MODEL.
"""
import os
import pytest
from pydantic import BaseModel
from estimator.infra.llm import AnthropicClient, OpenAIChatClient
pytestmark = [
pytest.mark.live,
pytest.mark.skipif(os.getenv("EST_LIVE_LLM") != "1", reason="set EST_LIVE_LLM=1 to run"),
]
class Capital(BaseModel):
country: str
capital: str
PROMPT = "Name the capital of Finland. Answer with country and capital."
async def test_anthropic_live():
key = os.environ["EST_LLM__ANTHROPIC__API_KEY"]
model = os.getenv("EST_LIVE_ANTHROPIC_MODEL", "claude-sonnet-5-5")
c = AnthropicClient(base_url="https://api.anthropic.com", api_key=key)
try:
res = await c.structured(system="Be brief.", user=PROMPT, schema=Capital, model=model, max_tokens=300)
finally:
await c.aclose()
assert res.parsed.capital.lower().startswith("helsinki")
assert res.usage.input_tokens > 0
async def test_openai_live():
key = os.environ["EST_LLM__OPENAI__API_KEY"]
model = os.environ["EST_LIVE_OPENAI_MODEL"]
c = OpenAIChatClient(base_url="https://api.openai.com/v1", api_key=key)
try:
res = await c.structured(system="Be brief.", user=PROMPT, schema=Capital, model=model, max_tokens=500)
finally:
await c.aclose()
assert res.parsed.capital.lower().startswith("helsinki")
assert res.usage.input_tokens > 0
@@ -0,0 +1,170 @@
"""Whole stack with only the network faked: Settings -> router -> real OpenAI and Anthropic
clients -> pipeline. Two providers, per-step routing, and a fallback after an outage."""
import json
from pathlib import Path
import httpx
import pytest
from estimator.agent import EstimationPipeline, PipelineInput, PromptStore
from estimator.config.settings import Settings
from estimator.infra.llm import build_gateway
from estimator.pricing import price
from tests.unit.agent.conftest import (
DOC_WBS,
clarify_out,
critic_out,
decompose_out,
estimate_out,
missing,
)
ROOT = Path(__file__).resolve().parents[2]
CANNED = {
"ClarifyOutput": clarify_out(),
"DecomposeOutput": decompose_out(),
"EstimateOutput": estimate_out(),
"CriticOutput": critic_out(missing=[missing("Документация")]),
}
class FakeProviders:
"""Answers like OpenAI and Anthropic, keyed by the schema the request asks for."""
def __init__(self, openai_down=False):
self.openai_down = openai_down
self.seen: list[tuple[str, str, str]] = [] # (provider, schema, model)
def __call__(self, request: httpx.Request) -> httpx.Response:
body = json.loads(request.content)
if request.url.host == "api.openai.com":
schema = body["response_format"]["json_schema"]["name"]
self.seen.append(("openai", schema, body["model"]))
if self.openai_down:
return httpx.Response(503, json={"error": {"message": "down"}})
text = CANNED[schema].model_dump_json()
return httpx.Response(
200,
json={
"choices": [{"message": {"content": text}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 1000, "completion_tokens": 200},
},
)
# Anthropic requests do not name the schema: recognise it by its content
wire = json.dumps(body["output_config"]["format"]["schema"])
if '"missing_tasks"' in wire:
schema = "CriticOutput"
elif '"task_id"' in wire:
schema = "EstimateOutput"
elif '"summary"' in wire:
schema = "ClarifyOutput"
else:
schema = "DecomposeOutput"
self.seen.append(("anthropic", schema, body["model"]))
return httpx.Response(
200,
json={
"content": [{"type": "text", "text": CANNED[schema].model_dump_json()}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 900, "output_tokens": 150},
},
)
def settings(monkeypatch, **env):
base = {
"EST_DATABASE__DSN": "sqlite:///:memory:",
"EST_LLM__ANTHROPIC__API_KEY": "sk-ant",
"EST_LLM__OPENAI__API_KEY": "sk-oai",
# estimate and critic run on OpenAI, critic falls back to Anthropic
"EST_LLM__STEPS__ESTIMATE__PROVIDER": "openai",
"EST_LLM__STEPS__ESTIMATE__MODEL": "gpt-test",
"EST_LLM__STEPS__CRITIC__PROVIDER": "openai",
"EST_LLM__STEPS__CRITIC__MODEL": "gpt-test",
"EST_LLM__STEPS__CRITIC__FALLBACK__PROVIDER": "anthropic",
"EST_LLM__STEPS__CRITIC__FALLBACK__MODEL": "claude-fallback",
}
base.update(env)
for k, v in base.items():
monkeypatch.setenv(k, v)
return Settings()
async def run(monkeypatch, providers, **env):
cfg = settings(monkeypatch, **env)
gw = build_gateway(cfg, transport=httpx.MockTransport(providers))
async def instant(_): # no real waiting between retries
return None
for c in gw._clients.values():
c._sleep = instant
try:
pipe = EstimationPipeline(gw, cfg.agent, PromptStore(ROOT / "prompts"))
return await pipe.run(PipelineInput(requirements="Бот Telegram+MAX+VK с ИИ и amoCRM"))
finally:
await gw.aclose()
async def test_two_providers_in_one_run(monkeypatch):
prov = FakeProviders()
res = await run(monkeypatch, prov)
assert res.status == "done"
assert [(p, s) for p, s, _ in prov.seen] == [
("anthropic", "ClarifyOutput"),
("anthropic", "DecomposeOutput"),
("openai", "EstimateOutput"),
("openai", "CriticOutput"),
]
assert [s.provider for s in res.steps] == ["anthropic", "anthropic", "openai", "openai"]
assert res.steps[2].model == "gpt-test" and not any(s.via_fallback for s in res.steps)
assert len(res.tasks) == len(DOC_WBS) + 1 # critic added documentation
assert res.total_tokens == (900 + 900 + 1000 + 1000, 150 + 150 + 200 + 200)
p = price(res.tasks, {"analyst": 1500, "backend": 1800, "qa": 1200, "devops": 1800})
assert p.hours > 144.7 # the doc example plus the critic's task
async def test_openai_outage_falls_back_for_the_critic_only(monkeypatch):
prov = FakeProviders(openai_down=True)
# estimate stays on Anthropic so only the critic depends on the broken provider
res = await run(
monkeypatch,
prov,
EST_LLM__STEPS__ESTIMATE__PROVIDER="anthropic",
EST_LLM__STEPS__ESTIMATE__MODEL="claude-x",
)
critic_log = res.steps[-1]
assert critic_log.step == "critic" and critic_log.via_fallback
assert critic_log.provider == "anthropic" and critic_log.model == "claude-fallback"
assert any(p == "openai" for p, _, _ in prov.seen) # it did try OpenAI first
assert res.status == "done"
async def test_strict_schema_is_what_goes_over_the_wire(monkeypatch):
seen = []
def handler(request: httpx.Request) -> httpx.Response:
seen.append(json.loads(request.content))
return FakeProviders()(request)
cfg = settings(monkeypatch)
gw = build_gateway(cfg, transport=httpx.MockTransport(handler))
try:
await EstimationPipeline(gw, cfg.agent, PromptStore(ROOT / "prompts")).run(
PipelineInput(requirements="бот")
)
finally:
await gw.aclose()
for body in seen:
schema = body.get("response_format", {}).get("json_schema", {}).get("schema") or body["output_config"]["format"]["schema"]
assert schema["additionalProperties"] is False and "$defs" not in schema
assert pytest.approx(len(seen)) == 4
async def test_outage_without_fallback_is_not_hidden(monkeypatch):
"""Estimate has no fallback: an OpenAI outage must surface, not return a partial estimate."""
from estimator.domain.errors import LlmTransientError
with pytest.raises(LlmTransientError):
await run(monkeypatch, FakeProviders(openai_down=True))