Files
estimator/tests/unit/rag/test_reference_service.py
2026-11-09 08:43:30 +03:00

164 lines
7.4 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import json
import pytest
from estimator.config.sections import DatabaseCfg
from estimator.domain.enums import Role
from estimator.domain.errors import EstimatorError, LlmTransientError
from estimator.infra.db import SqlReferenceStore, make_engine, make_session_factory
from estimator.infra.embeddings import HashEmbedder
from estimator.services.references import ReferenceService, parse_rows, read_file
def row(**kw):
base = {"project": "P1", "title": "Интеграция amoCRM", "description": "", "role": "backend", "hours": "18", "hours_kind": "actual"}
base.update(kw)
return base
# ------------------------------------------------------------------- validation
def test_a_good_row_is_parsed_with_defaults_and_synonyms():
ok, errors = parse_rows([row(hours="18,5", hours_kind="ФАКТ", role="Backend"), row(title="Без роли", role="", hours_kind="", project="")], default_project="Default")
assert errors == []
assert (ok[0].hours, ok[0].hours_kind, ok[0].role) == (18.5, "actual", Role.BACKEND)
assert (ok[1].role, ok[1].hours_kind, ok[1].project) == (None, "estimate", "Default")
def test_header_names_are_case_insensitive_and_missing_optional_columns_are_fine():
ok, errors = parse_rows([{"Project": "P", "TITLE": "Задача", "Hours": "3"}])
assert errors == [] and ok[0].title == "Задача" and ok[0].description == ""
@pytest.mark.parametrize(
"bad,fragment",
[
(row(title=""), "title"),
(row(title="x" * 301), "title"),
(row(role="wizard"), "unknown role"),
(row(hours="abc"), "hours must be a number"),
(row(hours=""), "hours must be a number"),
(row(hours="-1"), "between 0 and"),
(row(hours="401"), "between 0 and"),
(row(hours="nan"), "between 0 and"),
(row(hours_kind="maybe"), "hours_kind"),
(row(project=""), "project is required"),
],
)
def test_bad_rows_are_reported_with_their_number_and_do_not_stop_the_good_ones(bad, fragment):
ok, errors = parse_rows([row(title="Хорошая"), bad, row(title="Тоже хорошая")])
assert [r.title for r in ok] == ["Хорошая", "Тоже хорошая"]
assert len(errors) == 1 and errors[0].row == 2 and fragment in errors[0].message
def test_the_hours_limit_follows_the_argument():
ok, errors = parse_rows([row(hours="100")], max_hours=50)
assert ok == [] and "between 0 and 50" in errors[0].message
# -------------------------------------------------------------------------- files
def test_csv_with_a_bom_and_json_are_read(tmp_path):
csv_file = tmp_path / "t.csv"
csv_file.write_text("project,title,hours\nP,Задача один,5\nP,Задача два,6\n", encoding="utf-8-sig")
assert [r["title"] for r in read_file(csv_file)] == ["Задача один", "Задача два"]
json_file = tmp_path / "t.json"
json_file.write_text(json.dumps([{"project": "P", "title": "T", "hours": 1}]), encoding="utf-8")
assert read_file(json_file)[0]["title"] == "T"
bad = tmp_path / "bad.json"
bad.write_text('{"not": "a list"}')
with pytest.raises(EstimatorError):
read_file(bad)
# ----------------------------------------------------------------------- import
class Flaky(HashEmbedder):
"""Fails after `ok_calls` successful batches, like a provider that goes down mid-import."""
def __init__(self, ok_calls):
self.ok_calls, self.calls = ok_calls, 0
async def embed(self, texts):
self.calls += 1
if self.calls > self.ok_calls:
raise LlmTransientError("provider down", provider="x")
return await super().embed(texts)
@pytest.fixture
async def parts(dsn):
engine = make_engine(DatabaseCfg(dsn=dsn))
store = SqlReferenceStore(make_session_factory(engine), engine.dialect.name)
yield store
await engine.dispose()
async def test_import_stores_embeds_and_reports(parts):
service = ReferenceService(parts, HashEmbedder(), batch_size=2)
report = await service.import_rows([row(title=f"Задача {i}") for i in range(5)] + [row(title="", hours="1")], actor="alex")
assert (report.rows, report.valid, report.inserted, report.embedded, report.not_embedded) == (6, 5, 5, 5, 0)
assert len(report.errors) == 1 and report.errors[0].row == 6
assert (await service.stats()).embedded == 5
async def test_reimport_is_quiet_and_only_changed_text_is_embedded_again(parts):
service = ReferenceService(parts, HashEmbedder())
rows = [row(title="Интеграция amoCRM", description="сделки и контакты"), row(title="Адаптер MAX")]
await service.import_rows(rows)
again = await service.import_rows(rows)
assert (again.inserted, again.unchanged, again.embedded) == (0, 2, 0)
edited = await service.import_rows([row(title="Интеграция amoCRM", description="сделки, контакты и воронки"), row(title="Адаптер MAX", hours="13")])
assert (edited.updated, edited.embedded) == (2, 1) # two changed, only the text change needs a new vector
async def test_a_dry_run_changes_nothing(parts):
service = ReferenceService(parts, HashEmbedder())
report = await service.import_rows([row(), row(title="")], dry_run=True)
assert (report.valid, len(report.errors), report.dry_run, report.inserted) == (1, 1, True, 0)
assert (await service.stats()).total == 0
async def test_a_provider_failure_leaves_rows_pending_and_the_next_run_finishes(parts):
flaky = Flaky(ok_calls=1)
service = ReferenceService(parts, flaky, batch_size=2)
report = await service.import_rows([row(title=f"Задача {i}") for i in range(5)])
assert report.inserted == 5 and report.embedded == 2 and report.not_embedded == 3
assert "provider down" in report.embed_error
healthy = ReferenceService(parts, HashEmbedder(), batch_size=2)
done, left, error = await healthy.embed_pending()
assert (done, left, error) == (3, 0, "")
assert (await healthy.stats()).stale == 0
async def test_switching_the_model_makes_everything_stale_until_reindexed(parts):
await ReferenceService(parts, HashEmbedder()).import_rows([row(), row(title="Другая")])
class Other(HashEmbedder):
model = "hash-v2"
other = ReferenceService(parts, Other())
assert (await other.stats()).stale == 2
assert (await other.embed_pending())[0] == 2 and (await other.stats()).stale == 0
# ------------------------------------------------------------------------ search
async def test_search_finds_the_relevant_task_and_respects_the_threshold(parts):
service = ReferenceService(parts, HashEmbedder())
await service.import_rows([
row(title="Интеграция с amoCRM", description="сделки контакты"),
row(title="Развёртывание и мониторинг", role="devops"),
row(title="Тестирование сценариев бота", role="qa"),
]) # fmt: skip
hits = await service.search("интеграция с CRM amoCRM", role=None, k=3, min_similarity=0.3)
assert hits[0].title == "Интеграция с amoCRM" and hits[0].score > 0.5
assert all(h.title != "Развёртывание и мониторинг" for h in hits)
assert await service.search("совсем другая тема", role=None, k=3, min_similarity=0.6) == []
devops_only = await service.search("мониторинг", role=Role.DEVOPS, k=3, min_similarity=0.2)
assert [h.role for h in devops_only] == [Role.DEVOPS]