164 lines
7.4 KiB
Python
164 lines
7.4 KiB
Python
import json
|
||
|
||
import pytest
|
||
|
||
from estimator.config.sections import DatabaseCfg
|
||
from estimator.domain.enums import Role
|
||
from estimator.domain.errors import EstimatorError, LlmTransientError
|
||
from estimator.infra.db import SqlReferenceStore, make_engine, make_session_factory
|
||
from estimator.infra.embeddings import HashEmbedder
|
||
from estimator.services.references import ReferenceService, parse_rows, read_file
|
||
|
||
|
||
def row(**kw):
|
||
base = {"project": "P1", "title": "Интеграция amoCRM", "description": "", "role": "backend", "hours": "18", "hours_kind": "actual"}
|
||
base.update(kw)
|
||
return base
|
||
|
||
|
||
# ------------------------------------------------------------------- validation
|
||
|
||
|
||
def test_a_good_row_is_parsed_with_defaults_and_synonyms():
|
||
ok, errors = parse_rows([row(hours="18,5", hours_kind="ФАКТ", role="Backend"), row(title="Без роли", role="", hours_kind="", project="")], default_project="Default")
|
||
assert errors == []
|
||
assert (ok[0].hours, ok[0].hours_kind, ok[0].role) == (18.5, "actual", Role.BACKEND)
|
||
assert (ok[1].role, ok[1].hours_kind, ok[1].project) == (None, "estimate", "Default")
|
||
|
||
|
||
def test_header_names_are_case_insensitive_and_missing_optional_columns_are_fine():
|
||
ok, errors = parse_rows([{"Project": "P", "TITLE": "Задача", "Hours": "3"}])
|
||
assert errors == [] and ok[0].title == "Задача" and ok[0].description == ""
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
"bad,fragment",
|
||
[
|
||
(row(title=""), "title"),
|
||
(row(title="x" * 301), "title"),
|
||
(row(role="wizard"), "unknown role"),
|
||
(row(hours="abc"), "hours must be a number"),
|
||
(row(hours=""), "hours must be a number"),
|
||
(row(hours="-1"), "between 0 and"),
|
||
(row(hours="401"), "between 0 and"),
|
||
(row(hours="nan"), "between 0 and"),
|
||
(row(hours_kind="maybe"), "hours_kind"),
|
||
(row(project=""), "project is required"),
|
||
],
|
||
)
|
||
def test_bad_rows_are_reported_with_their_number_and_do_not_stop_the_good_ones(bad, fragment):
|
||
ok, errors = parse_rows([row(title="Хорошая"), bad, row(title="Тоже хорошая")])
|
||
assert [r.title for r in ok] == ["Хорошая", "Тоже хорошая"]
|
||
assert len(errors) == 1 and errors[0].row == 2 and fragment in errors[0].message
|
||
|
||
|
||
def test_the_hours_limit_follows_the_argument():
|
||
ok, errors = parse_rows([row(hours="100")], max_hours=50)
|
||
assert ok == [] and "between 0 and 50" in errors[0].message
|
||
|
||
|
||
# -------------------------------------------------------------------------- files
|
||
|
||
|
||
def test_csv_with_a_bom_and_json_are_read(tmp_path):
|
||
csv_file = tmp_path / "t.csv"
|
||
csv_file.write_text("project,title,hours\nP,Задача один,5\nP,Задача два,6\n", encoding="utf-8-sig")
|
||
assert [r["title"] for r in read_file(csv_file)] == ["Задача один", "Задача два"]
|
||
json_file = tmp_path / "t.json"
|
||
json_file.write_text(json.dumps([{"project": "P", "title": "T", "hours": 1}]), encoding="utf-8")
|
||
assert read_file(json_file)[0]["title"] == "T"
|
||
bad = tmp_path / "bad.json"
|
||
bad.write_text('{"not": "a list"}')
|
||
with pytest.raises(EstimatorError):
|
||
read_file(bad)
|
||
|
||
|
||
# ----------------------------------------------------------------------- import
|
||
|
||
|
||
class Flaky(HashEmbedder):
|
||
"""Fails after `ok_calls` successful batches, like a provider that goes down mid-import."""
|
||
|
||
def __init__(self, ok_calls):
|
||
self.ok_calls, self.calls = ok_calls, 0
|
||
|
||
async def embed(self, texts):
|
||
self.calls += 1
|
||
if self.calls > self.ok_calls:
|
||
raise LlmTransientError("provider down", provider="x")
|
||
return await super().embed(texts)
|
||
|
||
|
||
@pytest.fixture
|
||
async def parts(dsn):
|
||
engine = make_engine(DatabaseCfg(dsn=dsn))
|
||
store = SqlReferenceStore(make_session_factory(engine), engine.dialect.name)
|
||
yield store
|
||
await engine.dispose()
|
||
|
||
|
||
async def test_import_stores_embeds_and_reports(parts):
|
||
service = ReferenceService(parts, HashEmbedder(), batch_size=2)
|
||
report = await service.import_rows([row(title=f"Задача {i}") for i in range(5)] + [row(title="", hours="1")], actor="alex")
|
||
assert (report.rows, report.valid, report.inserted, report.embedded, report.not_embedded) == (6, 5, 5, 5, 0)
|
||
assert len(report.errors) == 1 and report.errors[0].row == 6
|
||
assert (await service.stats()).embedded == 5
|
||
|
||
|
||
async def test_reimport_is_quiet_and_only_changed_text_is_embedded_again(parts):
|
||
service = ReferenceService(parts, HashEmbedder())
|
||
rows = [row(title="Интеграция amoCRM", description="сделки и контакты"), row(title="Адаптер MAX")]
|
||
await service.import_rows(rows)
|
||
again = await service.import_rows(rows)
|
||
assert (again.inserted, again.unchanged, again.embedded) == (0, 2, 0)
|
||
edited = await service.import_rows([row(title="Интеграция amoCRM", description="сделки, контакты и воронки"), row(title="Адаптер MAX", hours="13")])
|
||
assert (edited.updated, edited.embedded) == (2, 1) # two changed, only the text change needs a new vector
|
||
|
||
|
||
async def test_a_dry_run_changes_nothing(parts):
|
||
service = ReferenceService(parts, HashEmbedder())
|
||
report = await service.import_rows([row(), row(title="")], dry_run=True)
|
||
assert (report.valid, len(report.errors), report.dry_run, report.inserted) == (1, 1, True, 0)
|
||
assert (await service.stats()).total == 0
|
||
|
||
|
||
async def test_a_provider_failure_leaves_rows_pending_and_the_next_run_finishes(parts):
|
||
flaky = Flaky(ok_calls=1)
|
||
service = ReferenceService(parts, flaky, batch_size=2)
|
||
report = await service.import_rows([row(title=f"Задача {i}") for i in range(5)])
|
||
assert report.inserted == 5 and report.embedded == 2 and report.not_embedded == 3
|
||
assert "provider down" in report.embed_error
|
||
healthy = ReferenceService(parts, HashEmbedder(), batch_size=2)
|
||
done, left, error = await healthy.embed_pending()
|
||
assert (done, left, error) == (3, 0, "")
|
||
assert (await healthy.stats()).stale == 0
|
||
|
||
|
||
async def test_switching_the_model_makes_everything_stale_until_reindexed(parts):
|
||
await ReferenceService(parts, HashEmbedder()).import_rows([row(), row(title="Другая")])
|
||
|
||
class Other(HashEmbedder):
|
||
model = "hash-v2"
|
||
|
||
other = ReferenceService(parts, Other())
|
||
assert (await other.stats()).stale == 2
|
||
assert (await other.embed_pending())[0] == 2 and (await other.stats()).stale == 0
|
||
|
||
|
||
# ------------------------------------------------------------------------ search
|
||
|
||
|
||
async def test_search_finds_the_relevant_task_and_respects_the_threshold(parts):
|
||
service = ReferenceService(parts, HashEmbedder())
|
||
await service.import_rows([
|
||
row(title="Интеграция с amoCRM", description="сделки контакты"),
|
||
row(title="Развёртывание и мониторинг", role="devops"),
|
||
row(title="Тестирование сценариев бота", role="qa"),
|
||
]) # fmt: skip
|
||
hits = await service.search("интеграция с CRM amoCRM", role=None, k=3, min_similarity=0.3)
|
||
assert hits[0].title == "Интеграция с amoCRM" and hits[0].score > 0.5
|
||
assert all(h.title != "Развёртывание и мониторинг" for h in hits)
|
||
assert await service.search("совсем другая тема", role=None, k=3, min_similarity=0.6) == []
|
||
devops_only = await service.search("мониторинг", role=Role.DEVOPS, k=3, min_similarity=0.2)
|
||
assert [h.role for h in devops_only] == [Role.DEVOPS]
|