Add RAG basic implementation

This commit is contained in:
2026-11-09 08:43:30 +03:00
parent 447464c16c
commit 7fa1d774a0
76 changed files with 3030 additions and 124 deletions
@@ -0,0 +1,111 @@
"""Who did what (the audit trail) and the 'preliminary' marker of estimates built on assumptions."""
import pytest
from tests.fakes import AutoGateway
from tests.unit.agent.conftest import clarify_out, q
from tests.unit.api.test_estimates_api import REQ, A, B, create, make_client, wait
EDIT = {"optimistic": 16, "likely": 32, "pessimistic": 64}
def test_every_version_records_who_made_it(monkeypatch, backend_dsn):
gw = AutoGateway(clarify=[clarify_out([q("crm", "Какая CRM?", blocking=True)]), clarify_out([])])
with make_client(monkeypatch, backend_dsn, gw) as c:
v1 = create(c)
assert (v1["created_by"], v1["estimate_created_by"]) == ("client_a", "client_a")
eid = v1["id"]
wait(c, eid)
answered = c.post(f"/api/estimates/{eid}/answers", json={"answers": [{"question_id": "crm", "answer": "amoCRM"}]}, headers=A).json()
wait(c, eid)
edited = c.patch(f"/api/estimates/{eid}/tasks/t4", json=EDIT, headers=A).json()
switched = c.post(f"/api/estimates/{eid}/engagement", json={"engagement": "time_and_materials"}, headers=A).json()
vat = c.post(f"/api/estimates/{eid}/vat", json={"vat_rate": 0}, headers=A).json()
recalculated = c.post(f"/api/estimates/{eid}/recalculate", headers=A).json()
for v in (answered, edited, switched, vat, recalculated):
assert v["created_by"] == "client_a" and v["estimate_created_by"] == "client_a"
assert [v["version"] for v in (answered, edited, switched, vat, recalculated)] == [2, 3, 4, 5, 6]
def test_the_creator_of_the_estimate_stays_while_later_versions_have_their_own_author(monkeypatch, backend_dsn):
"""Two clients cannot share an estimate through the API, so a second author is simulated at the
service level: the estimate keeps its creator and the new version records the person."""
import asyncio
from estimator.config.sections import DatabaseCfg
from estimator.domain.enums import Region
from estimator.domain.estimates import NewVersion
from estimator.infra.db import SqlEstimateRepo, make_engine, make_session_factory
async def go():
engine = make_engine(DatabaseCfg(dsn=backend_dsn))
try:
repo = SqlEstimateRepo(make_session_factory(engine))
data = NewVersion(requirements="x", region=Region.RU, currency="RUB", created_by="alex")
eid, *_ = await repo.create(client="ui", idempotency_key=None, request_hash="h", data=data)
v2 = await repo.add_version(client="ui", estimate_id=eid, data=data.model_copy(update={"created_by": "bob"}))
first, second = await repo.get("ui", eid, 1), await repo.get("ui", eid, v2)
return first, second, (await repo.list_page("ui", status=None, limit=5, cursor=None)).items[0]
finally:
await engine.dispose()
first, second, item = asyncio.run(go())
assert (first.created_by, first.estimate_created_by) == ("alex", "alex")
assert (second.created_by, second.estimate_created_by) == ("bob", "alex")
assert item.created_by == "alex"
def test_the_list_shows_who_started_each_estimate(monkeypatch, backend_dsn):
with make_client(monkeypatch, backend_dsn, AutoGateway()) as c:
wait(c, create(c)["id"])
items = c.get("/api/estimates", headers=A).json()["items"]
assert items[0]["created_by"] == "client_a"
assert c.get("/api/estimates", headers=B).json()["items"] == []
# ------------------------------------------------------------------ preliminary
def test_a_complete_estimate_is_not_preliminary(monkeypatch, backend_dsn):
with make_client(monkeypatch, backend_dsn, AutoGateway()) as c:
body = wait(c, create(c)["id"])
assert body["result"]["preliminary"] is False and body["result"]["preliminary_reasons"] == []
assert c.get("/api/estimates", headers=A).json()["items"][0]["preliminary"] is False
def test_an_estimate_that_rests_on_assumptions_is_preliminary_and_says_why(monkeypatch, backend_dsn):
gw = AutoGateway(clarify=[clarify_out([q("crm", "Какая CRM?", blocking=True), q("tone", "Какой тон?")])])
with make_client(monkeypatch, backend_dsn, gw) as c:
body = wait(c, create(c, {**REQ, "proceed_with_assumptions": True})["id"])
r = body["result"]
assert body["status"] == "done" and r["preliminary"] is True
assert r["preliminary_reasons"] == ["Не уточнено: Какая CRM?", "Не уточнено: Какой тон?"]
assert c.get("/api/estimates", headers=A).json()["items"][0]["preliminary"] is True
def test_answering_the_questions_makes_the_next_version_final(monkeypatch, backend_dsn):
gw = AutoGateway(clarify=[clarify_out([q("crm", "Какая CRM?", blocking=True)]), clarify_out([])])
with make_client(monkeypatch, backend_dsn, gw) as c:
eid = create(c, {**REQ, "proceed_with_assumptions": True})["id"]
wait(c, eid) # the blocking question was skipped by proceeding on assumptions
again = c.post(f"/api/estimates/{eid}/recalculate", json={"proceed_with_assumptions": False}, headers=A).json()
assert again["version"] == 2
final = wait(c, eid)
assert final["status"] == "done" and final["result"]["preliminary"] is False
def test_changing_how_an_estimate_is_shown_keeps_its_preliminary_status(monkeypatch, backend_dsn):
gw = AutoGateway(clarify=[clarify_out([q("crm", "Какая CRM?", blocking=True)])])
with make_client(monkeypatch, backend_dsn, gw) as c:
eid = create(c, {**REQ, "proceed_with_assumptions": True})["id"]
wait(c, eid)
switched = c.post(f"/api/estimates/{eid}/engagement", json={"engagement": "time_and_materials"}, headers=A).json()
assert switched["result"]["preliminary"] is True
edited = c.patch(f"/api/estimates/{eid}/tasks/t4", json=EDIT, headers=A).json()
assert edited["result"]["preliminary"] is True # a manual edit does not answer the open questions
@pytest.mark.parametrize("flag", [True, False])
def test_the_preliminary_flag_depends_on_open_points_not_on_the_proceed_flag(monkeypatch, backend_dsn, flag):
with make_client(monkeypatch, backend_dsn, AutoGateway()) as c: # no questions at all
body = wait(c, create(c, {**REQ, "proceed_with_assumptions": flag})["id"])
assert body["result"]["preliminary"] is False
+22 -4
View File
@@ -13,6 +13,7 @@ from tests.unit.agent.conftest import (
clarify_out,
critic_out,
decompose_out,
estimate_out,
q,
)
@@ -336,12 +337,29 @@ def test_timeout(monkeypatch, backend_dsn):
assert body["error"]["code"] == "timeout"
def test_roles_without_a_rate_are_blocked_up_front(monkeypatch, backend_dsn):
rows = [*DOC_WBS[:-1], ("Админка", Role.FRONTEND, 8, 16, 24)]
def test_roles_without_a_rate_are_blocked_up_front(monkeypatch, backend_dsn, tmp_path):
import re
import shutil
# a market file without a designer rate: the model must not be allowed to plan designer work
shutil.copytree("config", tmp_path / "config")
market = tmp_path / "config" / "market" / "2026Q4.toml"
market.write_text(re.sub(r"^designer = .*\n", "", market.read_text(), flags=re.MULTILINE))
rows = [*DOC_WBS[:-1], ("Макеты", Role.DESIGNER, 8, 16, 24)]
gw = AutoGateway(decompose=[decompose_out(rows)] * 2)
body = fail_case(monkeypatch, backend_dsn, gw)
body = fail_case(monkeypatch, backend_dsn, gw, EST_CONFIG_DIR=str(tmp_path / "config"))
assert body["error"]["code"] == "pipeline_invalid" and "roles not allowed" in body["error"]["message"]
assert "allowed roles: analyst, backend, qa, devops" in gw.calls[1]["user"]
assert "allowed roles: analyst, backend, qa, devops, frontend" in gw.calls[1]["user"]
def test_frontend_and_designer_roles_are_priced(monkeypatch, backend_dsn):
rows = [*DOC_WBS[:-1], ("Админка", Role.FRONTEND, 8, 16, 24), ("Макеты", Role.DESIGNER, 4, 8, 12)]
gw = AutoGateway(decompose=[decompose_out(rows)], estimate=[estimate_out(rows)])
body = fail_case(monkeypatch, backend_dsn, gw)
assert body["status"] == "done"
by_role = {x["role"]: x for x in body["result"]["price"]["by_role"]}
assert by_role["frontend"]["rate"] == 1800 and by_role["designer"]["rate"] == 1500
assert "allowed roles: analyst, backend, qa, devops, frontend, designer" in gw.calls[1]["user"]
def test_critic_outage_still_gives_an_estimate(monkeypatch, backend_dsn):
+118
View File
@@ -0,0 +1,118 @@
"""Retrieval end to end: a loaded corpus, the real store (SQLite and PostgreSQL with pgvector),
the offline embedder, the pipeline and the API."""
import asyncio
import pytest
from fastapi.testclient import TestClient
from sqlalchemy import text
from sqlalchemy.ext.asyncio import create_async_engine
from estimator.api.app import create_app
from estimator.config.settings import Settings
from estimator.domain.errors import LlmTransientError
from estimator.infra.db import normalize_dsn
from estimator.infra.embeddings import HashEmbedder
from tests.fakes import AutoGateway
from tests.rag_helpers import RAG_ENV, load_corpus
from tests.unit.api.test_estimates_api import A, create, make_client, wait
def task(body, task_id):
return next(t for t in body["result"]["tasks"] if t["id"] == task_id)
def test_tasks_get_their_own_similar_past_work(monkeypatch, backend_dsn):
assert load_corpus(backend_dsn).embedded == 4
gw = AutoGateway()
with make_client(monkeypatch, backend_dsn, gw, **RAG_ENV) as c:
body = wait(c, create(c)["id"])
crm = task(body, "t4") # "Интеграция amoCRM"
assert crm["references"][0]["title"] == "Интеграция с amoCRM"
first = crm["references"][0]
assert (first["hours"], first["hours_kind"], first["role"], first["project"]) == (21, "actual", "backend", "P1")
assert 0.3 <= first["score"] <= 1
qa = task(body, "t7") # testing
assert qa["references"][0]["title"] == "Тестирование сценариев бота" and qa["references"][0]["role"] == "qa"
assert task(body, "t8")["references"] == [] # deployment: nothing similar in the corpus
# the same hits were shown to the model, in the prompt of the estimate step
estimate_prompt = next(c["user"] for c in gw.calls if c["step"] == "estimate")
assert "Интеграция с amoCRM: 21 h actual, backend [P1]" in estimate_prompt
assert estimate_prompt.count("<reference_tasks>") >= 2
assert "Разработка логотипа" not in estimate_prompt # a designer task is not a hit for any of them
def test_a_role_never_gets_references_of_another_role(monkeypatch, backend_dsn):
load_corpus(backend_dsn)
with make_client(monkeypatch, backend_dsn, AutoGateway(), **RAG_ENV) as c:
body = wait(c, create(c)["id"])
for t in body["result"]["tasks"]:
assert all(r["role"] in (None, t["role"]) for r in t["references"])
def test_top_k_limits_the_references_per_task(monkeypatch, backend_dsn):
rows = [{"project": f"P{i}", "title": f"Интеграция с amoCRM вариант {i}", "role": "backend", "hours": str(10 + i), "hours_kind": "actual"} for i in range(6)]
load_corpus(backend_dsn, rows)
with make_client(monkeypatch, backend_dsn, AutoGateway(), **{**RAG_ENV, "EST_RAG__TOP_K": "3"}) as c:
body = wait(c, create(c)["id"])
assert len(task(body, "t4")["references"]) == 3
def test_retrieval_is_off_by_default_even_with_a_corpus(monkeypatch, backend_dsn):
load_corpus(backend_dsn)
gw = AutoGateway()
with make_client(monkeypatch, backend_dsn, gw) as c: # rag.enabled is false
body = wait(c, create(c)["id"])
assert all(t["references"] == [] for t in body["result"]["tasks"])
assert all("<reference_tasks>" not in call["user"] for call in gw.calls)
def test_an_empty_corpus_changes_nothing(monkeypatch, backend_dsn):
with make_client(monkeypatch, backend_dsn, AutoGateway(), **RAG_ENV) as c:
body = wait(c, create(c)["id"])
assert body["status"] == "done" and all(t["references"] == [] for t in body["result"]["tasks"])
assert body["result"]["price"]["cost_with_reserve"] == 370_000 # the numbers are the model's, not the corpus's
def test_an_embedding_outage_does_not_stop_the_estimate(monkeypatch, backend_dsn):
load_corpus(backend_dsn)
async def down(self, texts):
raise LlmTransientError("embeddings are down", provider="x")
monkeypatch.setattr(HashEmbedder, "embed", down)
with make_client(monkeypatch, backend_dsn, AutoGateway(), **RAG_ENV) as c:
body = wait(c, create(c)["id"])
assert body["status"] == "done" and any("аналоги" in w for w in body["result"]["warnings"])
assert all(t["references"] == [] for t in body["result"]["tasks"])
def test_references_do_not_change_what_the_model_is_asked_to_return(monkeypatch, backend_dsn):
"""Anchors are a hint in the prompt; hours still come from the model's answer."""
load_corpus(backend_dsn)
with make_client(monkeypatch, backend_dsn, AutoGateway(), **RAG_ENV) as c:
body = wait(c, create(c)["id"])
assert task(body, "t4")["likely"] == 16 and body["result"]["price"]["hours"] == pytest.approx(144.7)
def test_startup_names_the_fix_when_the_reference_table_is_missing(monkeypatch, dsn):
async def drop():
engine = create_async_engine(normalize_dsn(dsn))
async with engine.begin() as conn:
await conn.execute(text("drop table reference_tasks"))
await engine.dispose()
asyncio.run(drop())
for k, v in {"EST_DATABASE__DSN": dsn, "EST_LLM__ANTHROPIC__API_KEY": "sk-test", **RAG_ENV}.items():
monkeypatch.setenv(k, v)
with (
pytest.raises(RuntimeError, match="reference table is missing"),
TestClient(create_app(Settings(), gateway=AutoGateway())),
):
pass
def test_the_corpus_is_not_exposed_through_the_api(monkeypatch, backend_dsn):
load_corpus(backend_dsn)
with make_client(monkeypatch, backend_dsn, AutoGateway(), **RAG_ENV) as c:
for path in ("/api/references", "/api/rag", "/api/reference_tasks"):
assert c.get(path, headers=A).status_code == 404