"""Retrieval end to end: a loaded corpus, the real store (SQLite and PostgreSQL with pgvector), the offline embedder, the pipeline and the API.""" import asyncio import pytest from fastapi.testclient import TestClient from sqlalchemy import text from sqlalchemy.ext.asyncio import create_async_engine from estimator.api.app import create_app from estimator.config.settings import Settings from estimator.domain.errors import LlmTransientError from estimator.infra.db import normalize_dsn from estimator.infra.embeddings import HashEmbedder from tests.fakes import AutoGateway from tests.rag_helpers import RAG_ENV, load_corpus from tests.unit.api.test_estimates_api import A, create, make_client, wait def task(body, task_id): return next(t for t in body["result"]["tasks"] if t["id"] == task_id) def test_tasks_get_their_own_similar_past_work(monkeypatch, backend_dsn): assert load_corpus(backend_dsn).embedded == 4 gw = AutoGateway() with make_client(monkeypatch, backend_dsn, gw, **RAG_ENV) as c: body = wait(c, create(c)["id"]) crm = task(body, "t4") # "Интеграция amoCRM" assert crm["references"][0]["title"] == "Интеграция с amoCRM" first = crm["references"][0] assert (first["hours"], first["hours_kind"], first["role"], first["project"]) == (21, "actual", "backend", "P1") assert 0.3 <= first["score"] <= 1 qa = task(body, "t7") # testing assert qa["references"][0]["title"] == "Тестирование сценариев бота" and qa["references"][0]["role"] == "qa" assert task(body, "t8")["references"] == [] # deployment: nothing similar in the corpus # the same hits were shown to the model, in the prompt of the estimate step estimate_prompt = next(c["user"] for c in gw.calls if c["step"] == "estimate") assert "Интеграция с amoCRM: 21 h actual, backend [P1]" in estimate_prompt assert estimate_prompt.count("") >= 2 assert "Разработка логотипа" not in estimate_prompt # a designer task is not a hit for any of them def test_a_role_never_gets_references_of_another_role(monkeypatch, backend_dsn): load_corpus(backend_dsn) with make_client(monkeypatch, backend_dsn, AutoGateway(), **RAG_ENV) as c: body = wait(c, create(c)["id"]) for t in body["result"]["tasks"]: assert all(r["role"] in (None, t["role"]) for r in t["references"]) def test_top_k_limits_the_references_per_task(monkeypatch, backend_dsn): rows = [{"project": f"P{i}", "title": f"Интеграция с amoCRM вариант {i}", "role": "backend", "hours": str(10 + i), "hours_kind": "actual"} for i in range(6)] load_corpus(backend_dsn, rows) with make_client(monkeypatch, backend_dsn, AutoGateway(), **{**RAG_ENV, "EST_RAG__TOP_K": "3"}) as c: body = wait(c, create(c)["id"]) assert len(task(body, "t4")["references"]) == 3 def test_retrieval_is_off_by_default_even_with_a_corpus(monkeypatch, backend_dsn): load_corpus(backend_dsn) gw = AutoGateway() with make_client(monkeypatch, backend_dsn, gw) as c: # rag.enabled is false body = wait(c, create(c)["id"]) assert all(t["references"] == [] for t in body["result"]["tasks"]) assert all("" not in call["user"] for call in gw.calls) def test_an_empty_corpus_changes_nothing(monkeypatch, backend_dsn): with make_client(monkeypatch, backend_dsn, AutoGateway(), **RAG_ENV) as c: body = wait(c, create(c)["id"]) assert body["status"] == "done" and all(t["references"] == [] for t in body["result"]["tasks"]) assert body["result"]["price"]["cost_with_reserve"] == 370_000 # the numbers are the model's, not the corpus's def test_an_embedding_outage_does_not_stop_the_estimate(monkeypatch, backend_dsn): load_corpus(backend_dsn) async def down(self, texts): raise LlmTransientError("embeddings are down", provider="x") monkeypatch.setattr(HashEmbedder, "embed", down) with make_client(monkeypatch, backend_dsn, AutoGateway(), **RAG_ENV) as c: body = wait(c, create(c)["id"]) assert body["status"] == "done" and any("аналоги" in w for w in body["result"]["warnings"]) assert all(t["references"] == [] for t in body["result"]["tasks"]) def test_references_do_not_change_what_the_model_is_asked_to_return(monkeypatch, backend_dsn): """Anchors are a hint in the prompt; hours still come from the model's answer.""" load_corpus(backend_dsn) with make_client(monkeypatch, backend_dsn, AutoGateway(), **RAG_ENV) as c: body = wait(c, create(c)["id"]) assert task(body, "t4")["likely"] == 16 and body["result"]["price"]["hours"] == pytest.approx(144.7) def test_startup_names_the_fix_when_the_reference_table_is_missing(monkeypatch, dsn): async def drop(): engine = create_async_engine(normalize_dsn(dsn)) async with engine.begin() as conn: await conn.execute(text("drop table reference_tasks")) await engine.dispose() asyncio.run(drop()) for k, v in {"EST_DATABASE__DSN": dsn, "EST_LLM__ANTHROPIC__API_KEY": "sk-test", **RAG_ENV}.items(): monkeypatch.setenv(k, v) with ( pytest.raises(RuntimeError, match="reference table is missing"), TestClient(create_app(Settings(), gateway=AutoGateway())), ): pass def test_the_corpus_is_not_exposed_through_the_api(monkeypatch, backend_dsn): load_corpus(backend_dsn) with make_client(monkeypatch, backend_dsn, AutoGateway(), **RAG_ENV) as c: for path in ("/api/references", "/api/rag", "/api/reference_tasks"): assert c.get(path, headers=A).status_code == 404