119 lines
5.6 KiB
Python
119 lines
5.6 KiB
Python
"""Retrieval end to end: a loaded corpus, the real store (SQLite and PostgreSQL with pgvector),
|
||
the offline embedder, the pipeline and the API."""
|
||
import asyncio
|
||
|
||
import pytest
|
||
from fastapi.testclient import TestClient
|
||
from sqlalchemy import text
|
||
from sqlalchemy.ext.asyncio import create_async_engine
|
||
|
||
from estimator.api.app import create_app
|
||
from estimator.config.settings import Settings
|
||
from estimator.domain.errors import LlmTransientError
|
||
from estimator.infra.db import normalize_dsn
|
||
from estimator.infra.embeddings import HashEmbedder
|
||
from tests.fakes import AutoGateway
|
||
from tests.rag_helpers import RAG_ENV, load_corpus
|
||
from tests.unit.api.test_estimates_api import A, create, make_client, wait
|
||
|
||
|
||
def task(body, task_id):
|
||
return next(t for t in body["result"]["tasks"] if t["id"] == task_id)
|
||
|
||
|
||
def test_tasks_get_their_own_similar_past_work(monkeypatch, backend_dsn):
|
||
assert load_corpus(backend_dsn).embedded == 4
|
||
gw = AutoGateway()
|
||
with make_client(monkeypatch, backend_dsn, gw, **RAG_ENV) as c:
|
||
body = wait(c, create(c)["id"])
|
||
crm = task(body, "t4") # "Интеграция amoCRM"
|
||
assert crm["references"][0]["title"] == "Интеграция с amoCRM"
|
||
first = crm["references"][0]
|
||
assert (first["hours"], first["hours_kind"], first["role"], first["project"]) == (21, "actual", "backend", "P1")
|
||
assert 0.3 <= first["score"] <= 1
|
||
qa = task(body, "t7") # testing
|
||
assert qa["references"][0]["title"] == "Тестирование сценариев бота" and qa["references"][0]["role"] == "qa"
|
||
assert task(body, "t8")["references"] == [] # deployment: nothing similar in the corpus
|
||
# the same hits were shown to the model, in the prompt of the estimate step
|
||
estimate_prompt = next(c["user"] for c in gw.calls if c["step"] == "estimate")
|
||
assert "Интеграция с amoCRM: 21 h actual, backend [P1]" in estimate_prompt
|
||
assert estimate_prompt.count("<reference_tasks>") >= 2
|
||
assert "Разработка логотипа" not in estimate_prompt # a designer task is not a hit for any of them
|
||
|
||
|
||
def test_a_role_never_gets_references_of_another_role(monkeypatch, backend_dsn):
|
||
load_corpus(backend_dsn)
|
||
with make_client(monkeypatch, backend_dsn, AutoGateway(), **RAG_ENV) as c:
|
||
body = wait(c, create(c)["id"])
|
||
for t in body["result"]["tasks"]:
|
||
assert all(r["role"] in (None, t["role"]) for r in t["references"])
|
||
|
||
|
||
def test_top_k_limits_the_references_per_task(monkeypatch, backend_dsn):
|
||
rows = [{"project": f"P{i}", "title": f"Интеграция с amoCRM вариант {i}", "role": "backend", "hours": str(10 + i), "hours_kind": "actual"} for i in range(6)]
|
||
load_corpus(backend_dsn, rows)
|
||
with make_client(monkeypatch, backend_dsn, AutoGateway(), **{**RAG_ENV, "EST_RAG__TOP_K": "3"}) as c:
|
||
body = wait(c, create(c)["id"])
|
||
assert len(task(body, "t4")["references"]) == 3
|
||
|
||
|
||
def test_retrieval_is_off_by_default_even_with_a_corpus(monkeypatch, backend_dsn):
|
||
load_corpus(backend_dsn)
|
||
gw = AutoGateway()
|
||
with make_client(monkeypatch, backend_dsn, gw) as c: # rag.enabled is false
|
||
body = wait(c, create(c)["id"])
|
||
assert all(t["references"] == [] for t in body["result"]["tasks"])
|
||
assert all("<reference_tasks>" not in call["user"] for call in gw.calls)
|
||
|
||
|
||
def test_an_empty_corpus_changes_nothing(monkeypatch, backend_dsn):
|
||
with make_client(monkeypatch, backend_dsn, AutoGateway(), **RAG_ENV) as c:
|
||
body = wait(c, create(c)["id"])
|
||
assert body["status"] == "done" and all(t["references"] == [] for t in body["result"]["tasks"])
|
||
assert body["result"]["price"]["cost_with_reserve"] == 370_000 # the numbers are the model's, not the corpus's
|
||
|
||
|
||
def test_an_embedding_outage_does_not_stop_the_estimate(monkeypatch, backend_dsn):
|
||
load_corpus(backend_dsn)
|
||
|
||
async def down(self, texts):
|
||
raise LlmTransientError("embeddings are down", provider="x")
|
||
|
||
monkeypatch.setattr(HashEmbedder, "embed", down)
|
||
with make_client(monkeypatch, backend_dsn, AutoGateway(), **RAG_ENV) as c:
|
||
body = wait(c, create(c)["id"])
|
||
assert body["status"] == "done" and any("аналоги" in w for w in body["result"]["warnings"])
|
||
assert all(t["references"] == [] for t in body["result"]["tasks"])
|
||
|
||
|
||
def test_references_do_not_change_what_the_model_is_asked_to_return(monkeypatch, backend_dsn):
|
||
"""Anchors are a hint in the prompt; hours still come from the model's answer."""
|
||
load_corpus(backend_dsn)
|
||
with make_client(monkeypatch, backend_dsn, AutoGateway(), **RAG_ENV) as c:
|
||
body = wait(c, create(c)["id"])
|
||
assert task(body, "t4")["likely"] == 16 and body["result"]["price"]["hours"] == pytest.approx(144.7)
|
||
|
||
|
||
def test_startup_names_the_fix_when_the_reference_table_is_missing(monkeypatch, dsn):
|
||
async def drop():
|
||
engine = create_async_engine(normalize_dsn(dsn))
|
||
async with engine.begin() as conn:
|
||
await conn.execute(text("drop table reference_tasks"))
|
||
await engine.dispose()
|
||
|
||
asyncio.run(drop())
|
||
for k, v in {"EST_DATABASE__DSN": dsn, "EST_LLM__ANTHROPIC__API_KEY": "sk-test", **RAG_ENV}.items():
|
||
monkeypatch.setenv(k, v)
|
||
with (
|
||
pytest.raises(RuntimeError, match="reference table is missing"),
|
||
TestClient(create_app(Settings(), gateway=AutoGateway())),
|
||
):
|
||
pass
|
||
|
||
|
||
def test_the_corpus_is_not_exposed_through_the_api(monkeypatch, backend_dsn):
|
||
load_corpus(backend_dsn)
|
||
with make_client(monkeypatch, backend_dsn, AutoGateway(), **RAG_ENV) as c:
|
||
for path in ("/api/references", "/api/rag", "/api/reference_tasks"):
|
||
assert c.get(path, headers=A).status_code == 404
|