Init Project Flexly Estimator
This commit is contained in:
@@ -0,0 +1,79 @@
|
||||
"""The Alembic migration must produce exactly the schema the ORM models describe."""
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
import pytest
|
||||
from alembic import command
|
||||
from alembic.autogenerate import compare_metadata
|
||||
from alembic.config import Config
|
||||
from alembic.migration import MigrationContext
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
|
||||
from estimator.infra.db import Base, normalize_dsn
|
||||
|
||||
PG = os.getenv("TEST_PG_DSN")
|
||||
|
||||
|
||||
def _alembic(dsn: str, *cmd) -> None:
|
||||
cfg = Config("alembic.ini")
|
||||
cfg.cmd_opts = type("O", (), {"x": [f"dsn={dsn}"]})()
|
||||
getattr(command, cmd[0])(cfg, *cmd[1:])
|
||||
|
||||
|
||||
async def _diff(dsn: str):
|
||||
engine = create_async_engine(normalize_dsn(dsn))
|
||||
try:
|
||||
async with engine.connect() as conn:
|
||||
|
||||
def run(sync_conn):
|
||||
ctx = MigrationContext.configure(sync_conn, opts={"compare_type": True})
|
||||
return compare_metadata(ctx, Base.metadata)
|
||||
|
||||
return await conn.run_sync(run)
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def _tables(dsn: str) -> set[str]:
|
||||
engine = create_async_engine(normalize_dsn(dsn))
|
||||
try:
|
||||
async with engine.connect() as conn:
|
||||
from sqlalchemy import inspect
|
||||
|
||||
return set(await conn.run_sync(lambda c: inspect(c).get_table_names()))
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def _drop_pg(dsn):
|
||||
import psycopg
|
||||
|
||||
with psycopg.connect(dsn, autocommit=True) as c:
|
||||
c.execute("drop table if exists estimate_versions, estimates, alembic_version cascade")
|
||||
|
||||
|
||||
@pytest.fixture(params=["sqlite", "postgres"])
|
||||
def fresh_dsn(request, tmp_path):
|
||||
if request.param == "postgres":
|
||||
if not PG:
|
||||
pytest.skip("set TEST_PG_DSN to run on PostgreSQL")
|
||||
_drop_pg(PG)
|
||||
yield PG
|
||||
_drop_pg(PG)
|
||||
else:
|
||||
yield f"sqlite:///{tmp_path / 'mig.db'}"
|
||||
|
||||
|
||||
async def test_upgrade_matches_models_and_downgrade_is_clean(fresh_dsn):
|
||||
await asyncio.to_thread(_alembic, fresh_dsn, "upgrade", "head")
|
||||
assert {"estimates", "estimate_versions"} <= await _tables(fresh_dsn)
|
||||
assert await _diff(fresh_dsn) == [] # no drift between migration and models
|
||||
|
||||
await asyncio.to_thread(_alembic, fresh_dsn, "downgrade", "base")
|
||||
assert await _tables(fresh_dsn) <= {"alembic_version"}
|
||||
|
||||
|
||||
async def test_upgrade_is_idempotent(fresh_dsn):
|
||||
await asyncio.to_thread(_alembic, fresh_dsn, "upgrade", "head")
|
||||
await asyncio.to_thread(_alembic, fresh_dsn, "upgrade", "head")
|
||||
assert await _diff(fresh_dsn) == []
|
||||
@@ -0,0 +1,226 @@
|
||||
"""Repository behaviour on SQLite and, when TEST_PG_DSN is set, on PostgreSQL."""
|
||||
import asyncio
|
||||
from datetime import UTC
|
||||
|
||||
import pytest
|
||||
|
||||
from estimator.config.sections import DatabaseCfg
|
||||
from estimator.domain.enums import Region
|
||||
from estimator.domain.errors import EstimateNotFound, IdempotencyConflict, InvalidRequestError
|
||||
from estimator.domain.estimates import EstimateStatus, NewVersion, VersionSource
|
||||
from estimator.domain.models import AnsweredQuestion
|
||||
from estimator.infra.db import SqlEstimateRepo, make_engine, make_session_factory
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def repo(backend_dsn):
|
||||
engine = make_engine(DatabaseCfg(dsn=backend_dsn))
|
||||
yield SqlEstimateRepo(make_session_factory(engine))
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def new(text="бот", **kw) -> NewVersion:
|
||||
return NewVersion(requirements=text, region=Region.RU, currency="RUB", **kw)
|
||||
|
||||
|
||||
async def test_create_and_get_roundtrip(repo):
|
||||
data = new(
|
||||
"бот для Telegram",
|
||||
answers=[AnsweredQuestion(question="CRM?", answer="amoCRM")],
|
||||
proceed_with_assumptions=True,
|
||||
)
|
||||
eid, version, created = await repo.create(client="a", idempotency_key=None, request_hash="h", data=data)
|
||||
assert created and version == 1
|
||||
rec = await repo.get("a", eid)
|
||||
assert rec.requirements == "бот для Telegram" and rec.status == EstimateStatus.PENDING
|
||||
assert rec.answers == [AnsweredQuestion(question="CRM?", answer="amoCRM")]
|
||||
assert rec.proceed_with_assumptions and rec.region == Region.RU
|
||||
assert rec.created_at.tzinfo is not None and rec.created_at.utcoffset().total_seconds() == 0
|
||||
assert rec.latest_version == 1 and rec.estimate_created_at == rec.created_at
|
||||
|
||||
|
||||
async def test_other_client_cannot_read_or_extend(repo):
|
||||
eid, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new())
|
||||
with pytest.raises(EstimateNotFound):
|
||||
await repo.get("b", eid)
|
||||
with pytest.raises(EstimateNotFound):
|
||||
await repo.add_version(client="b", estimate_id=eid, data=new())
|
||||
with pytest.raises(EstimateNotFound):
|
||||
await repo.get("a", "does-not-exist")
|
||||
|
||||
|
||||
async def test_idempotency_same_request_returns_original(repo):
|
||||
a = await repo.create(client="a", idempotency_key="k1", request_hash="h", data=new())
|
||||
b = await repo.create(client="a", idempotency_key="k1", request_hash="h", data=new())
|
||||
assert a[0] == b[0] and a[2] is True and b[2] is False
|
||||
|
||||
|
||||
async def test_idempotency_different_request_conflicts(repo):
|
||||
await repo.create(client="a", idempotency_key="k1", request_hash="h1", data=new())
|
||||
with pytest.raises(IdempotencyConflict):
|
||||
await repo.create(client="a", idempotency_key="k1", request_hash="h2", data=new())
|
||||
|
||||
|
||||
async def test_idempotency_key_is_per_client(repo):
|
||||
a = await repo.create(client="a", idempotency_key="k1", request_hash="h", data=new())
|
||||
b = await repo.create(client="b", idempotency_key="k1", request_hash="h", data=new())
|
||||
assert a[0] != b[0] and a[2] and b[2]
|
||||
|
||||
|
||||
async def test_idempotency_race_creates_one_estimate(repo):
|
||||
results = await asyncio.gather(
|
||||
*[repo.create(client="a", idempotency_key="race", request_hash="h", data=new()) for _ in range(6)]
|
||||
)
|
||||
assert len({r[0] for r in results}) == 1
|
||||
assert sum(1 for r in results if r[2]) == 1
|
||||
|
||||
|
||||
async def test_without_key_every_create_is_new(repo):
|
||||
a = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new())
|
||||
b = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new())
|
||||
assert a[0] != b[0]
|
||||
|
||||
|
||||
async def test_add_version_increments_and_keeps_history(repo):
|
||||
eid, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new("v1 text"))
|
||||
v2 = await repo.add_version(
|
||||
client="a", estimate_id=eid, data=new("v2 text", source=VersionSource.RECALC)
|
||||
)
|
||||
assert v2 == 2
|
||||
assert (await repo.get("a", eid)).requirements == "v2 text" # latest by default
|
||||
assert (await repo.get("a", eid, 1)).requirements == "v1 text" # history is kept
|
||||
assert (await repo.get("a", eid, 2)).latest_version == 2
|
||||
with pytest.raises(EstimateNotFound):
|
||||
await repo.get("a", eid, 9)
|
||||
|
||||
|
||||
async def test_concurrent_add_version_gets_distinct_numbers(repo):
|
||||
eid, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new())
|
||||
numbers = await asyncio.gather(
|
||||
*[repo.add_version(client="a", estimate_id=eid, data=new(f"t{i}")) for i in range(8)]
|
||||
)
|
||||
assert sorted(numbers) == list(range(2, 10))
|
||||
assert (await repo.get("a", eid)).latest_version == 9
|
||||
|
||||
|
||||
async def test_update_version_stores_json_and_times(repo):
|
||||
from estimator.infra.db.types import utcnow
|
||||
|
||||
eid, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new())
|
||||
now = utcnow()
|
||||
await repo.update_version(
|
||||
eid, 1,
|
||||
status="done", step=None, market_version="2026Q4",
|
||||
result={"summary": "итог", "tasks": [{"id": "t1", "hours": 1.5}], "ключ": "значение"},
|
||||
price={"cost_with_reserve": 370000.0}, finished_at=now,
|
||||
) # fmt: skip
|
||||
rec = await repo.get_by_id(eid, 1)
|
||||
assert rec.status == EstimateStatus.DONE
|
||||
assert rec.result["tasks"][0]["hours"] == 1.5 and rec.result["ключ"] == "значение"
|
||||
assert rec.price == {"cost_with_reserve": 370000.0}
|
||||
assert rec.finished_at.tzinfo is not None and abs((rec.finished_at - now).total_seconds()) < 1
|
||||
assert rec.finished_at.tzinfo == UTC or rec.finished_at.utcoffset().total_seconds() == 0
|
||||
|
||||
|
||||
async def test_update_version_rejects_immutable_fields(repo):
|
||||
eid, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new())
|
||||
with pytest.raises(ValueError):
|
||||
await repo.update_version(eid, 1, requirements="changed")
|
||||
with pytest.raises(ValueError):
|
||||
await repo.update_version(eid, 1, client="evil")
|
||||
|
||||
|
||||
async def test_list_pages_newest_first_without_gaps_or_repeats(repo):
|
||||
ids = []
|
||||
for i in range(5):
|
||||
eid, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new(f"n{i}"))
|
||||
ids.append(eid)
|
||||
await repo.create(client="other", idempotency_key=None, request_hash="h", data=new("foreign"))
|
||||
|
||||
seen, cursor, pages = [], None, 0
|
||||
while True:
|
||||
page = await repo.list_page("a", status=None, limit=2, cursor=cursor)
|
||||
seen += [i.id for i in page.items]
|
||||
pages += 1
|
||||
cursor = page.next_cursor
|
||||
if not cursor:
|
||||
break
|
||||
assert pages == 3 and seen == list(reversed(ids))
|
||||
|
||||
|
||||
async def test_list_exact_page_has_no_dangling_cursor(repo):
|
||||
for i in range(2):
|
||||
await repo.create(client="a", idempotency_key=None, request_hash="h", data=new(f"n{i}"))
|
||||
page = await repo.list_page("a", status=None, limit=2, cursor=None)
|
||||
assert len(page.items) == 2 and page.next_cursor is None
|
||||
|
||||
|
||||
async def test_list_shows_latest_version_and_filters_by_status(repo):
|
||||
e1, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new("one"))
|
||||
e2, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new("two"))
|
||||
await repo.update_version(
|
||||
e1, 1, status="done", result={"summary": "Готово"}, price={"cost_with_reserve": 123000.0, "cost_expected": 100000.0}
|
||||
)
|
||||
await repo.add_version(client="a", estimate_id=e2, data=new("two again", status=EstimateStatus.NEEDS_INPUT))
|
||||
|
||||
done = await repo.list_page("a", status=EstimateStatus.DONE, limit=10, cursor=None)
|
||||
assert [i.id for i in done.items] == [e1]
|
||||
assert done.items[0].summary == "Готово" and done.items[0].cost == 123000.0
|
||||
waiting = await repo.list_page("a", status=EstimateStatus.NEEDS_INPUT, limit=10, cursor=None)
|
||||
assert [(i.id, i.version) for i in waiting.items] == [(e2, 2)] # filter looks at the latest version
|
||||
|
||||
|
||||
async def test_invalid_cursor(repo):
|
||||
with pytest.raises(InvalidRequestError):
|
||||
await repo.list_page("a", status=None, limit=5, cursor="not-a-cursor")
|
||||
|
||||
|
||||
async def test_active_versions_for_recovery(repo):
|
||||
e1, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new())
|
||||
e2, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new())
|
||||
e3, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new())
|
||||
await repo.update_version(e2, 1, status="running")
|
||||
await repo.update_version(e3, 1, status="done")
|
||||
active = await repo.active_versions()
|
||||
assert {(i, v, s.value) for i, v, s in active} == {(e1, 1, "pending"), (e2, 1, "running")}
|
||||
|
||||
|
||||
async def test_engagement_is_stored_per_version_and_defaults_to_fixed_price(repo):
|
||||
from estimator.domain.enums import Engagement
|
||||
|
||||
eid, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new())
|
||||
assert (await repo.get("a", eid)).engagement == Engagement.FIXED_PRICE
|
||||
v2 = await repo.add_version(
|
||||
client="a", estimate_id=eid,
|
||||
data=new(engagement=Engagement.TIME_AND_MATERIALS, source=VersionSource.REPRICE),
|
||||
) # fmt: skip
|
||||
assert (await repo.get("a", eid, v2)).engagement == Engagement.TIME_AND_MATERIALS
|
||||
assert (await repo.get("a", eid, 1)).engagement == Engagement.FIXED_PRICE
|
||||
assert (await repo.get("a", eid, v2)).source == VersionSource.REPRICE
|
||||
|
||||
|
||||
async def test_list_amount_follows_the_engagement_model(repo):
|
||||
from estimator.domain.enums import Engagement
|
||||
|
||||
price = {"cost_with_reserve": 370000.0, "cost_expected": 336000.0}
|
||||
fixed, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new("fp"))
|
||||
tm, *_ = await repo.create(
|
||||
client="a", idempotency_key=None, request_hash="h",
|
||||
data=new("tm", engagement=Engagement.TIME_AND_MATERIALS),
|
||||
) # fmt: skip
|
||||
await repo.update_version(fixed, 1, status="done", price=price)
|
||||
await repo.update_version(tm, 1, status="done", price=price)
|
||||
items = {i.id: i for i in (await repo.list_page("a", status=None, limit=10, cursor=None)).items}
|
||||
assert items[fixed].cost == 370000.0 and items[fixed].engagement == Engagement.FIXED_PRICE
|
||||
assert items[tm].cost == 336000.0 and items[tm].engagement == Engagement.TIME_AND_MATERIALS
|
||||
|
||||
|
||||
async def test_vat_rate_is_stored_per_version(repo):
|
||||
eid, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new(vat_rate=0.22))
|
||||
assert (await repo.get("a", eid)).vat_rate == pytest.approx(0.22)
|
||||
v2 = await repo.add_version(client="a", estimate_id=eid, data=new(vat_rate=0.0))
|
||||
assert (await repo.get("a", eid, v2)).vat_rate == 0.0
|
||||
assert (await repo.get("a", eid, 1)).vat_rate == pytest.approx(0.22)
|
||||
other, *_ = await repo.create(client="a", idempotency_key=None, request_hash="h", data=new())
|
||||
assert (await repo.get("a", other)).vat_rate == 0.0 # the default of a version is no VAT
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Real provider calls. Skipped unless EST_LIVE_LLM=1 and the keys are set.
|
||||
|
||||
EST_LIVE_LLM=1 EST_LLM__ANTHROPIC__API_KEY=... EST_LLM__OPENAI__API_KEY=... \
|
||||
pytest tests/integration/test_llm_live.py -m live -s
|
||||
|
||||
Set the OpenAI model in EST_LIVE_OPENAI_MODEL and the Anthropic one in EST_LIVE_ANTHROPIC_MODEL.
|
||||
"""
|
||||
import os
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel
|
||||
|
||||
from estimator.infra.llm import AnthropicClient, OpenAIChatClient
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.live,
|
||||
pytest.mark.skipif(os.getenv("EST_LIVE_LLM") != "1", reason="set EST_LIVE_LLM=1 to run"),
|
||||
]
|
||||
|
||||
|
||||
class Capital(BaseModel):
|
||||
country: str
|
||||
capital: str
|
||||
|
||||
|
||||
PROMPT = "Name the capital of Finland. Answer with country and capital."
|
||||
|
||||
|
||||
async def test_anthropic_live():
|
||||
key = os.environ["EST_LLM__ANTHROPIC__API_KEY"]
|
||||
model = os.getenv("EST_LIVE_ANTHROPIC_MODEL", "claude-sonnet-5-5")
|
||||
c = AnthropicClient(base_url="https://api.anthropic.com", api_key=key)
|
||||
try:
|
||||
res = await c.structured(system="Be brief.", user=PROMPT, schema=Capital, model=model, max_tokens=300)
|
||||
finally:
|
||||
await c.aclose()
|
||||
assert res.parsed.capital.lower().startswith("helsinki")
|
||||
assert res.usage.input_tokens > 0
|
||||
|
||||
|
||||
async def test_openai_live():
|
||||
key = os.environ["EST_LLM__OPENAI__API_KEY"]
|
||||
model = os.environ["EST_LIVE_OPENAI_MODEL"]
|
||||
c = OpenAIChatClient(base_url="https://api.openai.com/v1", api_key=key)
|
||||
try:
|
||||
res = await c.structured(system="Be brief.", user=PROMPT, schema=Capital, model=model, max_tokens=500)
|
||||
finally:
|
||||
await c.aclose()
|
||||
assert res.parsed.capital.lower().startswith("helsinki")
|
||||
assert res.usage.input_tokens > 0
|
||||
@@ -0,0 +1,170 @@
|
||||
"""Whole stack with only the network faked: Settings -> router -> real OpenAI and Anthropic
|
||||
clients -> pipeline. Two providers, per-step routing, and a fallback after an outage."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from estimator.agent import EstimationPipeline, PipelineInput, PromptStore
|
||||
from estimator.config.settings import Settings
|
||||
from estimator.infra.llm import build_gateway
|
||||
from estimator.pricing import price
|
||||
from tests.unit.agent.conftest import (
|
||||
DOC_WBS,
|
||||
clarify_out,
|
||||
critic_out,
|
||||
decompose_out,
|
||||
estimate_out,
|
||||
missing,
|
||||
)
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
CANNED = {
|
||||
"ClarifyOutput": clarify_out(),
|
||||
"DecomposeOutput": decompose_out(),
|
||||
"EstimateOutput": estimate_out(),
|
||||
"CriticOutput": critic_out(missing=[missing("Документация")]),
|
||||
}
|
||||
|
||||
|
||||
class FakeProviders:
|
||||
"""Answers like OpenAI and Anthropic, keyed by the schema the request asks for."""
|
||||
|
||||
def __init__(self, openai_down=False):
|
||||
self.openai_down = openai_down
|
||||
self.seen: list[tuple[str, str, str]] = [] # (provider, schema, model)
|
||||
|
||||
def __call__(self, request: httpx.Request) -> httpx.Response:
|
||||
body = json.loads(request.content)
|
||||
if request.url.host == "api.openai.com":
|
||||
schema = body["response_format"]["json_schema"]["name"]
|
||||
self.seen.append(("openai", schema, body["model"]))
|
||||
if self.openai_down:
|
||||
return httpx.Response(503, json={"error": {"message": "down"}})
|
||||
text = CANNED[schema].model_dump_json()
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"choices": [{"message": {"content": text}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 1000, "completion_tokens": 200},
|
||||
},
|
||||
)
|
||||
# Anthropic requests do not name the schema: recognise it by its content
|
||||
wire = json.dumps(body["output_config"]["format"]["schema"])
|
||||
if '"missing_tasks"' in wire:
|
||||
schema = "CriticOutput"
|
||||
elif '"task_id"' in wire:
|
||||
schema = "EstimateOutput"
|
||||
elif '"summary"' in wire:
|
||||
schema = "ClarifyOutput"
|
||||
else:
|
||||
schema = "DecomposeOutput"
|
||||
self.seen.append(("anthropic", schema, body["model"]))
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"content": [{"type": "text", "text": CANNED[schema].model_dump_json()}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 900, "output_tokens": 150},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def settings(monkeypatch, **env):
|
||||
base = {
|
||||
"EST_DATABASE__DSN": "sqlite:///:memory:",
|
||||
"EST_LLM__ANTHROPIC__API_KEY": "sk-ant",
|
||||
"EST_LLM__OPENAI__API_KEY": "sk-oai",
|
||||
# estimate and critic run on OpenAI, critic falls back to Anthropic
|
||||
"EST_LLM__STEPS__ESTIMATE__PROVIDER": "openai",
|
||||
"EST_LLM__STEPS__ESTIMATE__MODEL": "gpt-test",
|
||||
"EST_LLM__STEPS__CRITIC__PROVIDER": "openai",
|
||||
"EST_LLM__STEPS__CRITIC__MODEL": "gpt-test",
|
||||
"EST_LLM__STEPS__CRITIC__FALLBACK__PROVIDER": "anthropic",
|
||||
"EST_LLM__STEPS__CRITIC__FALLBACK__MODEL": "claude-fallback",
|
||||
}
|
||||
base.update(env)
|
||||
for k, v in base.items():
|
||||
monkeypatch.setenv(k, v)
|
||||
return Settings()
|
||||
|
||||
|
||||
async def run(monkeypatch, providers, **env):
|
||||
cfg = settings(monkeypatch, **env)
|
||||
gw = build_gateway(cfg, transport=httpx.MockTransport(providers))
|
||||
|
||||
async def instant(_): # no real waiting between retries
|
||||
return None
|
||||
|
||||
for c in gw._clients.values():
|
||||
c._sleep = instant
|
||||
try:
|
||||
pipe = EstimationPipeline(gw, cfg.agent, PromptStore(ROOT / "prompts"))
|
||||
return await pipe.run(PipelineInput(requirements="Бот Telegram+MAX+VK с ИИ и amoCRM"))
|
||||
finally:
|
||||
await gw.aclose()
|
||||
|
||||
|
||||
async def test_two_providers_in_one_run(monkeypatch):
|
||||
prov = FakeProviders()
|
||||
res = await run(monkeypatch, prov)
|
||||
assert res.status == "done"
|
||||
assert [(p, s) for p, s, _ in prov.seen] == [
|
||||
("anthropic", "ClarifyOutput"),
|
||||
("anthropic", "DecomposeOutput"),
|
||||
("openai", "EstimateOutput"),
|
||||
("openai", "CriticOutput"),
|
||||
]
|
||||
assert [s.provider for s in res.steps] == ["anthropic", "anthropic", "openai", "openai"]
|
||||
assert res.steps[2].model == "gpt-test" and not any(s.via_fallback for s in res.steps)
|
||||
assert len(res.tasks) == len(DOC_WBS) + 1 # critic added documentation
|
||||
assert res.total_tokens == (900 + 900 + 1000 + 1000, 150 + 150 + 200 + 200)
|
||||
p = price(res.tasks, {"analyst": 1500, "backend": 1800, "qa": 1200, "devops": 1800})
|
||||
assert p.hours > 144.7 # the doc example plus the critic's task
|
||||
|
||||
|
||||
async def test_openai_outage_falls_back_for_the_critic_only(monkeypatch):
|
||||
prov = FakeProviders(openai_down=True)
|
||||
# estimate stays on Anthropic so only the critic depends on the broken provider
|
||||
res = await run(
|
||||
monkeypatch,
|
||||
prov,
|
||||
EST_LLM__STEPS__ESTIMATE__PROVIDER="anthropic",
|
||||
EST_LLM__STEPS__ESTIMATE__MODEL="claude-x",
|
||||
)
|
||||
critic_log = res.steps[-1]
|
||||
assert critic_log.step == "critic" and critic_log.via_fallback
|
||||
assert critic_log.provider == "anthropic" and critic_log.model == "claude-fallback"
|
||||
assert any(p == "openai" for p, _, _ in prov.seen) # it did try OpenAI first
|
||||
assert res.status == "done"
|
||||
|
||||
|
||||
async def test_strict_schema_is_what_goes_over_the_wire(monkeypatch):
|
||||
seen = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
seen.append(json.loads(request.content))
|
||||
return FakeProviders()(request)
|
||||
|
||||
cfg = settings(monkeypatch)
|
||||
gw = build_gateway(cfg, transport=httpx.MockTransport(handler))
|
||||
try:
|
||||
await EstimationPipeline(gw, cfg.agent, PromptStore(ROOT / "prompts")).run(
|
||||
PipelineInput(requirements="бот")
|
||||
)
|
||||
finally:
|
||||
await gw.aclose()
|
||||
for body in seen:
|
||||
schema = body.get("response_format", {}).get("json_schema", {}).get("schema") or body["output_config"]["format"]["schema"]
|
||||
assert schema["additionalProperties"] is False and "$defs" not in schema
|
||||
assert pytest.approx(len(seen)) == 4
|
||||
|
||||
|
||||
async def test_outage_without_fallback_is_not_hidden(monkeypatch):
|
||||
"""Estimate has no fallback: an OpenAI outage must surface, not return a partial estimate."""
|
||||
from estimator.domain.errors import LlmTransientError
|
||||
|
||||
with pytest.raises(LlmTransientError):
|
||||
await run(monkeypatch, FakeProviders(openai_down=True))
|
||||
Reference in New Issue
Block a user