Add RAG basic implementation
This commit is contained in:
@@ -0,0 +1,111 @@
|
||||
"""Who did what (the audit trail) and the 'preliminary' marker of estimates built on assumptions."""
|
||||
import pytest
|
||||
|
||||
from tests.fakes import AutoGateway
|
||||
from tests.unit.agent.conftest import clarify_out, q
|
||||
from tests.unit.api.test_estimates_api import REQ, A, B, create, make_client, wait
|
||||
|
||||
EDIT = {"optimistic": 16, "likely": 32, "pessimistic": 64}
|
||||
|
||||
|
||||
def test_every_version_records_who_made_it(monkeypatch, backend_dsn):
|
||||
gw = AutoGateway(clarify=[clarify_out([q("crm", "Какая CRM?", blocking=True)]), clarify_out([])])
|
||||
with make_client(monkeypatch, backend_dsn, gw) as c:
|
||||
v1 = create(c)
|
||||
assert (v1["created_by"], v1["estimate_created_by"]) == ("client_a", "client_a")
|
||||
eid = v1["id"]
|
||||
wait(c, eid)
|
||||
answered = c.post(f"/api/estimates/{eid}/answers", json={"answers": [{"question_id": "crm", "answer": "amoCRM"}]}, headers=A).json()
|
||||
wait(c, eid)
|
||||
edited = c.patch(f"/api/estimates/{eid}/tasks/t4", json=EDIT, headers=A).json()
|
||||
switched = c.post(f"/api/estimates/{eid}/engagement", json={"engagement": "time_and_materials"}, headers=A).json()
|
||||
vat = c.post(f"/api/estimates/{eid}/vat", json={"vat_rate": 0}, headers=A).json()
|
||||
recalculated = c.post(f"/api/estimates/{eid}/recalculate", headers=A).json()
|
||||
for v in (answered, edited, switched, vat, recalculated):
|
||||
assert v["created_by"] == "client_a" and v["estimate_created_by"] == "client_a"
|
||||
assert [v["version"] for v in (answered, edited, switched, vat, recalculated)] == [2, 3, 4, 5, 6]
|
||||
|
||||
|
||||
def test_the_creator_of_the_estimate_stays_while_later_versions_have_their_own_author(monkeypatch, backend_dsn):
|
||||
"""Two clients cannot share an estimate through the API, so a second author is simulated at the
|
||||
service level: the estimate keeps its creator and the new version records the person."""
|
||||
import asyncio
|
||||
|
||||
from estimator.config.sections import DatabaseCfg
|
||||
from estimator.domain.enums import Region
|
||||
from estimator.domain.estimates import NewVersion
|
||||
from estimator.infra.db import SqlEstimateRepo, make_engine, make_session_factory
|
||||
|
||||
async def go():
|
||||
engine = make_engine(DatabaseCfg(dsn=backend_dsn))
|
||||
try:
|
||||
repo = SqlEstimateRepo(make_session_factory(engine))
|
||||
data = NewVersion(requirements="x", region=Region.RU, currency="RUB", created_by="alex")
|
||||
eid, *_ = await repo.create(client="ui", idempotency_key=None, request_hash="h", data=data)
|
||||
v2 = await repo.add_version(client="ui", estimate_id=eid, data=data.model_copy(update={"created_by": "bob"}))
|
||||
first, second = await repo.get("ui", eid, 1), await repo.get("ui", eid, v2)
|
||||
return first, second, (await repo.list_page("ui", status=None, limit=5, cursor=None)).items[0]
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
first, second, item = asyncio.run(go())
|
||||
assert (first.created_by, first.estimate_created_by) == ("alex", "alex")
|
||||
assert (second.created_by, second.estimate_created_by) == ("bob", "alex")
|
||||
assert item.created_by == "alex"
|
||||
|
||||
|
||||
def test_the_list_shows_who_started_each_estimate(monkeypatch, backend_dsn):
|
||||
with make_client(monkeypatch, backend_dsn, AutoGateway()) as c:
|
||||
wait(c, create(c)["id"])
|
||||
items = c.get("/api/estimates", headers=A).json()["items"]
|
||||
assert items[0]["created_by"] == "client_a"
|
||||
assert c.get("/api/estimates", headers=B).json()["items"] == []
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ preliminary
|
||||
|
||||
|
||||
def test_a_complete_estimate_is_not_preliminary(monkeypatch, backend_dsn):
|
||||
with make_client(monkeypatch, backend_dsn, AutoGateway()) as c:
|
||||
body = wait(c, create(c)["id"])
|
||||
assert body["result"]["preliminary"] is False and body["result"]["preliminary_reasons"] == []
|
||||
assert c.get("/api/estimates", headers=A).json()["items"][0]["preliminary"] is False
|
||||
|
||||
|
||||
def test_an_estimate_that_rests_on_assumptions_is_preliminary_and_says_why(monkeypatch, backend_dsn):
|
||||
gw = AutoGateway(clarify=[clarify_out([q("crm", "Какая CRM?", blocking=True), q("tone", "Какой тон?")])])
|
||||
with make_client(monkeypatch, backend_dsn, gw) as c:
|
||||
body = wait(c, create(c, {**REQ, "proceed_with_assumptions": True})["id"])
|
||||
r = body["result"]
|
||||
assert body["status"] == "done" and r["preliminary"] is True
|
||||
assert r["preliminary_reasons"] == ["Не уточнено: Какая CRM?", "Не уточнено: Какой тон?"]
|
||||
assert c.get("/api/estimates", headers=A).json()["items"][0]["preliminary"] is True
|
||||
|
||||
|
||||
def test_answering_the_questions_makes_the_next_version_final(monkeypatch, backend_dsn):
|
||||
gw = AutoGateway(clarify=[clarify_out([q("crm", "Какая CRM?", blocking=True)]), clarify_out([])])
|
||||
with make_client(monkeypatch, backend_dsn, gw) as c:
|
||||
eid = create(c, {**REQ, "proceed_with_assumptions": True})["id"]
|
||||
wait(c, eid) # the blocking question was skipped by proceeding on assumptions
|
||||
again = c.post(f"/api/estimates/{eid}/recalculate", json={"proceed_with_assumptions": False}, headers=A).json()
|
||||
assert again["version"] == 2
|
||||
final = wait(c, eid)
|
||||
assert final["status"] == "done" and final["result"]["preliminary"] is False
|
||||
|
||||
|
||||
def test_changing_how_an_estimate_is_shown_keeps_its_preliminary_status(monkeypatch, backend_dsn):
|
||||
gw = AutoGateway(clarify=[clarify_out([q("crm", "Какая CRM?", blocking=True)])])
|
||||
with make_client(monkeypatch, backend_dsn, gw) as c:
|
||||
eid = create(c, {**REQ, "proceed_with_assumptions": True})["id"]
|
||||
wait(c, eid)
|
||||
switched = c.post(f"/api/estimates/{eid}/engagement", json={"engagement": "time_and_materials"}, headers=A).json()
|
||||
assert switched["result"]["preliminary"] is True
|
||||
edited = c.patch(f"/api/estimates/{eid}/tasks/t4", json=EDIT, headers=A).json()
|
||||
assert edited["result"]["preliminary"] is True # a manual edit does not answer the open questions
|
||||
|
||||
|
||||
@pytest.mark.parametrize("flag", [True, False])
|
||||
def test_the_preliminary_flag_depends_on_open_points_not_on_the_proceed_flag(monkeypatch, backend_dsn, flag):
|
||||
with make_client(monkeypatch, backend_dsn, AutoGateway()) as c: # no questions at all
|
||||
body = wait(c, create(c, {**REQ, "proceed_with_assumptions": flag})["id"])
|
||||
assert body["result"]["preliminary"] is False
|
||||
@@ -13,6 +13,7 @@ from tests.unit.agent.conftest import (
|
||||
clarify_out,
|
||||
critic_out,
|
||||
decompose_out,
|
||||
estimate_out,
|
||||
q,
|
||||
)
|
||||
|
||||
@@ -336,12 +337,29 @@ def test_timeout(monkeypatch, backend_dsn):
|
||||
assert body["error"]["code"] == "timeout"
|
||||
|
||||
|
||||
def test_roles_without_a_rate_are_blocked_up_front(monkeypatch, backend_dsn):
|
||||
rows = [*DOC_WBS[:-1], ("Админка", Role.FRONTEND, 8, 16, 24)]
|
||||
def test_roles_without_a_rate_are_blocked_up_front(monkeypatch, backend_dsn, tmp_path):
|
||||
import re
|
||||
import shutil
|
||||
|
||||
# a market file without a designer rate: the model must not be allowed to plan designer work
|
||||
shutil.copytree("config", tmp_path / "config")
|
||||
market = tmp_path / "config" / "market" / "2026Q4.toml"
|
||||
market.write_text(re.sub(r"^designer = .*\n", "", market.read_text(), flags=re.MULTILINE))
|
||||
rows = [*DOC_WBS[:-1], ("Макеты", Role.DESIGNER, 8, 16, 24)]
|
||||
gw = AutoGateway(decompose=[decompose_out(rows)] * 2)
|
||||
body = fail_case(monkeypatch, backend_dsn, gw)
|
||||
body = fail_case(monkeypatch, backend_dsn, gw, EST_CONFIG_DIR=str(tmp_path / "config"))
|
||||
assert body["error"]["code"] == "pipeline_invalid" and "roles not allowed" in body["error"]["message"]
|
||||
assert "allowed roles: analyst, backend, qa, devops" in gw.calls[1]["user"]
|
||||
assert "allowed roles: analyst, backend, qa, devops, frontend" in gw.calls[1]["user"]
|
||||
|
||||
|
||||
def test_frontend_and_designer_roles_are_priced(monkeypatch, backend_dsn):
|
||||
rows = [*DOC_WBS[:-1], ("Админка", Role.FRONTEND, 8, 16, 24), ("Макеты", Role.DESIGNER, 4, 8, 12)]
|
||||
gw = AutoGateway(decompose=[decompose_out(rows)], estimate=[estimate_out(rows)])
|
||||
body = fail_case(monkeypatch, backend_dsn, gw)
|
||||
assert body["status"] == "done"
|
||||
by_role = {x["role"]: x for x in body["result"]["price"]["by_role"]}
|
||||
assert by_role["frontend"]["rate"] == 1800 and by_role["designer"]["rate"] == 1500
|
||||
assert "allowed roles: analyst, backend, qa, devops, frontend, designer" in gw.calls[1]["user"]
|
||||
|
||||
|
||||
def test_critic_outage_still_gives_an_estimate(monkeypatch, backend_dsn):
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Retrieval end to end: a loaded corpus, the real store (SQLite and PostgreSQL with pgvector),
|
||||
the offline embedder, the pipeline and the API."""
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
|
||||
from estimator.api.app import create_app
|
||||
from estimator.config.settings import Settings
|
||||
from estimator.domain.errors import LlmTransientError
|
||||
from estimator.infra.db import normalize_dsn
|
||||
from estimator.infra.embeddings import HashEmbedder
|
||||
from tests.fakes import AutoGateway
|
||||
from tests.rag_helpers import RAG_ENV, load_corpus
|
||||
from tests.unit.api.test_estimates_api import A, create, make_client, wait
|
||||
|
||||
|
||||
def task(body, task_id):
|
||||
return next(t for t in body["result"]["tasks"] if t["id"] == task_id)
|
||||
|
||||
|
||||
def test_tasks_get_their_own_similar_past_work(monkeypatch, backend_dsn):
|
||||
assert load_corpus(backend_dsn).embedded == 4
|
||||
gw = AutoGateway()
|
||||
with make_client(monkeypatch, backend_dsn, gw, **RAG_ENV) as c:
|
||||
body = wait(c, create(c)["id"])
|
||||
crm = task(body, "t4") # "Интеграция amoCRM"
|
||||
assert crm["references"][0]["title"] == "Интеграция с amoCRM"
|
||||
first = crm["references"][0]
|
||||
assert (first["hours"], first["hours_kind"], first["role"], first["project"]) == (21, "actual", "backend", "P1")
|
||||
assert 0.3 <= first["score"] <= 1
|
||||
qa = task(body, "t7") # testing
|
||||
assert qa["references"][0]["title"] == "Тестирование сценариев бота" and qa["references"][0]["role"] == "qa"
|
||||
assert task(body, "t8")["references"] == [] # deployment: nothing similar in the corpus
|
||||
# the same hits were shown to the model, in the prompt of the estimate step
|
||||
estimate_prompt = next(c["user"] for c in gw.calls if c["step"] == "estimate")
|
||||
assert "Интеграция с amoCRM: 21 h actual, backend [P1]" in estimate_prompt
|
||||
assert estimate_prompt.count("<reference_tasks>") >= 2
|
||||
assert "Разработка логотипа" not in estimate_prompt # a designer task is not a hit for any of them
|
||||
|
||||
|
||||
def test_a_role_never_gets_references_of_another_role(monkeypatch, backend_dsn):
|
||||
load_corpus(backend_dsn)
|
||||
with make_client(monkeypatch, backend_dsn, AutoGateway(), **RAG_ENV) as c:
|
||||
body = wait(c, create(c)["id"])
|
||||
for t in body["result"]["tasks"]:
|
||||
assert all(r["role"] in (None, t["role"]) for r in t["references"])
|
||||
|
||||
|
||||
def test_top_k_limits_the_references_per_task(monkeypatch, backend_dsn):
|
||||
rows = [{"project": f"P{i}", "title": f"Интеграция с amoCRM вариант {i}", "role": "backend", "hours": str(10 + i), "hours_kind": "actual"} for i in range(6)]
|
||||
load_corpus(backend_dsn, rows)
|
||||
with make_client(monkeypatch, backend_dsn, AutoGateway(), **{**RAG_ENV, "EST_RAG__TOP_K": "3"}) as c:
|
||||
body = wait(c, create(c)["id"])
|
||||
assert len(task(body, "t4")["references"]) == 3
|
||||
|
||||
|
||||
def test_retrieval_is_off_by_default_even_with_a_corpus(monkeypatch, backend_dsn):
|
||||
load_corpus(backend_dsn)
|
||||
gw = AutoGateway()
|
||||
with make_client(monkeypatch, backend_dsn, gw) as c: # rag.enabled is false
|
||||
body = wait(c, create(c)["id"])
|
||||
assert all(t["references"] == [] for t in body["result"]["tasks"])
|
||||
assert all("<reference_tasks>" not in call["user"] for call in gw.calls)
|
||||
|
||||
|
||||
def test_an_empty_corpus_changes_nothing(monkeypatch, backend_dsn):
|
||||
with make_client(monkeypatch, backend_dsn, AutoGateway(), **RAG_ENV) as c:
|
||||
body = wait(c, create(c)["id"])
|
||||
assert body["status"] == "done" and all(t["references"] == [] for t in body["result"]["tasks"])
|
||||
assert body["result"]["price"]["cost_with_reserve"] == 370_000 # the numbers are the model's, not the corpus's
|
||||
|
||||
|
||||
def test_an_embedding_outage_does_not_stop_the_estimate(monkeypatch, backend_dsn):
|
||||
load_corpus(backend_dsn)
|
||||
|
||||
async def down(self, texts):
|
||||
raise LlmTransientError("embeddings are down", provider="x")
|
||||
|
||||
monkeypatch.setattr(HashEmbedder, "embed", down)
|
||||
with make_client(monkeypatch, backend_dsn, AutoGateway(), **RAG_ENV) as c:
|
||||
body = wait(c, create(c)["id"])
|
||||
assert body["status"] == "done" and any("аналоги" in w for w in body["result"]["warnings"])
|
||||
assert all(t["references"] == [] for t in body["result"]["tasks"])
|
||||
|
||||
|
||||
def test_references_do_not_change_what_the_model_is_asked_to_return(monkeypatch, backend_dsn):
|
||||
"""Anchors are a hint in the prompt; hours still come from the model's answer."""
|
||||
load_corpus(backend_dsn)
|
||||
with make_client(monkeypatch, backend_dsn, AutoGateway(), **RAG_ENV) as c:
|
||||
body = wait(c, create(c)["id"])
|
||||
assert task(body, "t4")["likely"] == 16 and body["result"]["price"]["hours"] == pytest.approx(144.7)
|
||||
|
||||
|
||||
def test_startup_names_the_fix_when_the_reference_table_is_missing(monkeypatch, dsn):
|
||||
async def drop():
|
||||
engine = create_async_engine(normalize_dsn(dsn))
|
||||
async with engine.begin() as conn:
|
||||
await conn.execute(text("drop table reference_tasks"))
|
||||
await engine.dispose()
|
||||
|
||||
asyncio.run(drop())
|
||||
for k, v in {"EST_DATABASE__DSN": dsn, "EST_LLM__ANTHROPIC__API_KEY": "sk-test", **RAG_ENV}.items():
|
||||
monkeypatch.setenv(k, v)
|
||||
with (
|
||||
pytest.raises(RuntimeError, match="reference table is missing"),
|
||||
TestClient(create_app(Settings(), gateway=AutoGateway())),
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
def test_the_corpus_is_not_exposed_through_the_api(monkeypatch, backend_dsn):
|
||||
load_corpus(backend_dsn)
|
||||
with make_client(monkeypatch, backend_dsn, AutoGateway(), **RAG_ENV) as c:
|
||||
for path in ("/api/references", "/api/rag", "/api/reference_tasks"):
|
||||
assert c.get(path, headers=A).status_code == 404
|
||||
Reference in New Issue
Block a user