Files
estimator/tests/unit/api/test_rag_e2e.py
T
2026-11-09 08:43:30 +03:00

119 lines
5.6 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Retrieval end to end: a loaded corpus, the real store (SQLite and PostgreSQL with pgvector),
the offline embedder, the pipeline and the API."""
import asyncio
import pytest
from fastapi.testclient import TestClient
from sqlalchemy import text
from sqlalchemy.ext.asyncio import create_async_engine
from estimator.api.app import create_app
from estimator.config.settings import Settings
from estimator.domain.errors import LlmTransientError
from estimator.infra.db import normalize_dsn
from estimator.infra.embeddings import HashEmbedder
from tests.fakes import AutoGateway
from tests.rag_helpers import RAG_ENV, load_corpus
from tests.unit.api.test_estimates_api import A, create, make_client, wait
def task(body, task_id):
return next(t for t in body["result"]["tasks"] if t["id"] == task_id)
def test_tasks_get_their_own_similar_past_work(monkeypatch, backend_dsn):
assert load_corpus(backend_dsn).embedded == 4
gw = AutoGateway()
with make_client(monkeypatch, backend_dsn, gw, **RAG_ENV) as c:
body = wait(c, create(c)["id"])
crm = task(body, "t4") # "Интеграция amoCRM"
assert crm["references"][0]["title"] == "Интеграция с amoCRM"
first = crm["references"][0]
assert (first["hours"], first["hours_kind"], first["role"], first["project"]) == (21, "actual", "backend", "P1")
assert 0.3 <= first["score"] <= 1
qa = task(body, "t7") # testing
assert qa["references"][0]["title"] == "Тестирование сценариев бота" and qa["references"][0]["role"] == "qa"
assert task(body, "t8")["references"] == [] # deployment: nothing similar in the corpus
# the same hits were shown to the model, in the prompt of the estimate step
estimate_prompt = next(c["user"] for c in gw.calls if c["step"] == "estimate")
assert "Интеграция с amoCRM: 21 h actual, backend [P1]" in estimate_prompt
assert estimate_prompt.count("<reference_tasks>") >= 2
assert "Разработка логотипа" not in estimate_prompt # a designer task is not a hit for any of them
def test_a_role_never_gets_references_of_another_role(monkeypatch, backend_dsn):
load_corpus(backend_dsn)
with make_client(monkeypatch, backend_dsn, AutoGateway(), **RAG_ENV) as c:
body = wait(c, create(c)["id"])
for t in body["result"]["tasks"]:
assert all(r["role"] in (None, t["role"]) for r in t["references"])
def test_top_k_limits_the_references_per_task(monkeypatch, backend_dsn):
rows = [{"project": f"P{i}", "title": f"Интеграция с amoCRM вариант {i}", "role": "backend", "hours": str(10 + i), "hours_kind": "actual"} for i in range(6)]
load_corpus(backend_dsn, rows)
with make_client(monkeypatch, backend_dsn, AutoGateway(), **{**RAG_ENV, "EST_RAG__TOP_K": "3"}) as c:
body = wait(c, create(c)["id"])
assert len(task(body, "t4")["references"]) == 3
def test_retrieval_is_off_by_default_even_with_a_corpus(monkeypatch, backend_dsn):
load_corpus(backend_dsn)
gw = AutoGateway()
with make_client(monkeypatch, backend_dsn, gw) as c: # rag.enabled is false
body = wait(c, create(c)["id"])
assert all(t["references"] == [] for t in body["result"]["tasks"])
assert all("<reference_tasks>" not in call["user"] for call in gw.calls)
def test_an_empty_corpus_changes_nothing(monkeypatch, backend_dsn):
with make_client(monkeypatch, backend_dsn, AutoGateway(), **RAG_ENV) as c:
body = wait(c, create(c)["id"])
assert body["status"] == "done" and all(t["references"] == [] for t in body["result"]["tasks"])
assert body["result"]["price"]["cost_with_reserve"] == 370_000 # the numbers are the model's, not the corpus's
def test_an_embedding_outage_does_not_stop_the_estimate(monkeypatch, backend_dsn):
load_corpus(backend_dsn)
async def down(self, texts):
raise LlmTransientError("embeddings are down", provider="x")
monkeypatch.setattr(HashEmbedder, "embed", down)
with make_client(monkeypatch, backend_dsn, AutoGateway(), **RAG_ENV) as c:
body = wait(c, create(c)["id"])
assert body["status"] == "done" and any("аналоги" in w for w in body["result"]["warnings"])
assert all(t["references"] == [] for t in body["result"]["tasks"])
def test_references_do_not_change_what_the_model_is_asked_to_return(monkeypatch, backend_dsn):
"""Anchors are a hint in the prompt; hours still come from the model's answer."""
load_corpus(backend_dsn)
with make_client(monkeypatch, backend_dsn, AutoGateway(), **RAG_ENV) as c:
body = wait(c, create(c)["id"])
assert task(body, "t4")["likely"] == 16 and body["result"]["price"]["hours"] == pytest.approx(144.7)
def test_startup_names_the_fix_when_the_reference_table_is_missing(monkeypatch, dsn):
async def drop():
engine = create_async_engine(normalize_dsn(dsn))
async with engine.begin() as conn:
await conn.execute(text("drop table reference_tasks"))
await engine.dispose()
asyncio.run(drop())
for k, v in {"EST_DATABASE__DSN": dsn, "EST_LLM__ANTHROPIC__API_KEY": "sk-test", **RAG_ENV}.items():
monkeypatch.setenv(k, v)
with (
pytest.raises(RuntimeError, match="reference table is missing"),
TestClient(create_app(Settings(), gateway=AutoGateway())),
):
pass
def test_the_corpus_is_not_exposed_through_the_api(monkeypatch, backend_dsn):
load_corpus(backend_dsn)
with make_client(monkeypatch, backend_dsn, AutoGateway(), **RAG_ENV) as c:
for path in ("/api/references", "/api/rag", "/api/reference_tasks"):
assert c.get(path, headers=A).status_code == 404