Add RAG basic implementation
This commit is contained in:
@@ -0,0 +1,104 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from estimator.__main__ import main
|
||||
from estimator.config import get_settings
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def env(monkeypatch, dsn):
|
||||
monkeypatch.setenv("EST_DATABASE__DSN", dsn)
|
||||
monkeypatch.setenv("EST_LLM__ANTHROPIC__API_KEY", "sk-test")
|
||||
monkeypatch.setenv("EST_RAG__PROVIDER", "hash")
|
||||
get_settings.cache_clear()
|
||||
yield
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
def write_csv(tmp_path, rows, name="refs.csv"):
|
||||
path = tmp_path / name
|
||||
lines = ["project,title,description,role,hours,hours_kind"] + [",".join(r) for r in rows]
|
||||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
GOOD = [
|
||||
["P1", "Интеграция amoCRM", "сделки и контакты", "backend", "18", "actual"],
|
||||
["P1", "Адаптер MAX", "", "backend", "12", "estimate"],
|
||||
["P2", "Тестирование сценариев", "", "qa", "20", "actual"],
|
||||
]
|
||||
|
||||
|
||||
def test_import_stats_search_delete_cycle(env, tmp_path, capsys):
|
||||
path = write_csv(tmp_path, GOOD)
|
||||
assert main(["rag", "import", str(path)]) == 0
|
||||
out = capsys.readouterr().out
|
||||
assert "rows 3, valid 3, rejected 0" in out and "inserted 3" in out and "embedded 3 with hash-v1" in out
|
||||
|
||||
assert main(["rag", "stats"]) == 0
|
||||
out = capsys.readouterr().out
|
||||
assert "tasks 3 in 2 projects (2 with actual hours, 1 estimates)" in out
|
||||
assert "need (re)indexing: 0" in out and "retrieval is OFF" in out
|
||||
|
||||
assert main(["rag", "search", "интеграция с CRM", "--min-similarity", "0.3"]) == 0
|
||||
out = capsys.readouterr().out
|
||||
assert "Интеграция amoCRM" in out and "18 h actual" in out and "P1" in out
|
||||
assert main(["rag", "search", "ничего общего", "--min-similarity", "0.9"]) == 0
|
||||
assert "nothing above the similarity threshold" in capsys.readouterr().out
|
||||
|
||||
assert main(["rag", "search", "тестирование", "--role", "backend", "--min-similarity", "0.2"]) == 0
|
||||
assert "Тестирование сценариев" not in capsys.readouterr().out # a QA task is not shown for a backend query
|
||||
|
||||
assert main(["rag", "delete", "--project", "P2"]) == 0
|
||||
assert "deleted 1 tasks of 'P2'" in capsys.readouterr().out
|
||||
main(["rag", "stats"])
|
||||
assert "tasks 2 in 1 projects" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_dry_run_validates_without_writing(env, tmp_path, capsys):
|
||||
path = write_csv(tmp_path, GOOD)
|
||||
assert main(["rag", "import", str(path), "--dry-run"]) == 0
|
||||
assert "dry run: nothing was written" in capsys.readouterr().out
|
||||
main(["rag", "stats"])
|
||||
assert "tasks 0 in 0 projects" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_bad_rows_are_listed_and_do_not_block_the_good_ones(env, tmp_path, capsys):
|
||||
path = write_csv(tmp_path, [*GOOD, ["P1", "Плохая", "", "wizard", "5", ""], ["P1", "", "", "", "5", ""]])
|
||||
assert main(["rag", "import", str(path)]) == 0
|
||||
out = capsys.readouterr().out
|
||||
assert "row 4: unknown role 'wizard'" in out and "row 5: title" in out and "valid 3, rejected 2" in out
|
||||
|
||||
|
||||
def test_a_file_with_only_bad_rows_fails(env, tmp_path, capsys):
|
||||
path = write_csv(tmp_path, [["P1", "", "", "", "x", ""]])
|
||||
assert main(["rag", "import", str(path)]) == 1
|
||||
|
||||
|
||||
def test_default_project_and_json_input(env, tmp_path, capsys):
|
||||
path = tmp_path / "refs.json"
|
||||
path.write_text(json.dumps([{"title": "Задача без проекта", "hours": 4}]), encoding="utf-8")
|
||||
assert main(["rag", "import", str(path)]) == 1 # no project in the file and no default: nothing valid
|
||||
assert "project is required" in capsys.readouterr().out
|
||||
assert main(["rag", "import", str(path), "--project", "Старый проект"]) == 0
|
||||
assert "inserted 1" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_without_an_embeddings_key_the_openai_provider_is_refused(monkeypatch, dsn, capsys):
|
||||
monkeypatch.setenv("EST_DATABASE__DSN", dsn)
|
||||
monkeypatch.setenv("EST_LLM__ANTHROPIC__API_KEY", "sk-test")
|
||||
get_settings.cache_clear()
|
||||
try:
|
||||
assert main(["rag", "stats"]) == 2
|
||||
finally:
|
||||
get_settings.cache_clear()
|
||||
assert "No embeddings API key" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_the_commands_work_while_retrieval_is_still_switched_off(env, tmp_path, capsys):
|
||||
"""An administrator loads the corpus first and enables retrieval afterwards."""
|
||||
main(["rag", "import", str(write_csv(tmp_path, GOOD))])
|
||||
capsys.readouterr()
|
||||
main(["rag", "stats"])
|
||||
assert "retrieval is OFF" in capsys.readouterr().out
|
||||
@@ -0,0 +1,139 @@
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from estimator.domain.embeddings import EMBEDDING_DIM
|
||||
from estimator.domain.errors import LlmAuthError, LlmOutputError
|
||||
from estimator.infra.embeddings import HashEmbedder, OpenAIEmbeddings
|
||||
from tests.unit.llm.conftest import Recorder
|
||||
|
||||
E = HashEmbedder()
|
||||
|
||||
|
||||
def dot(a, b):
|
||||
return sum(x * y for x, y in zip(a, b, strict=True))
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- hash embedder
|
||||
|
||||
|
||||
def test_vectors_have_the_schema_dimension_and_unit_length():
|
||||
v = E.embed_one("Интеграция с amoCRM")
|
||||
assert len(v) == EMBEDDING_DIM and dot(v, v) == pytest.approx(1.0)
|
||||
|
||||
|
||||
def test_related_texts_are_closer_than_unrelated_ones():
|
||||
a = E.embed_one("Интеграция с amoCRM")
|
||||
related = E.embed_one("интеграции amoCRM и сделок")
|
||||
unrelated = E.embed_one("Развёртывание и мониторинг")
|
||||
assert dot(a, related) > 0.6
|
||||
assert dot(a, unrelated) < 0.1
|
||||
|
||||
|
||||
def test_case_and_russian_endings_do_not_matter():
|
||||
assert dot(E.embed_one("Тестирование бота"), E.embed_one("ТЕСТИРОВАНИЯ БОТА")) > 0.99
|
||||
|
||||
|
||||
def test_text_without_usable_words_still_gives_a_valid_vector():
|
||||
for text in ("", " ", "и для на", "!!!"):
|
||||
v = E.embed_one(text)
|
||||
assert dot(v, v) == pytest.approx(1.0)
|
||||
|
||||
|
||||
def test_embedding_is_deterministic_across_processes_and_hash_seeds():
|
||||
code = (
|
||||
"from estimator.infra.embeddings import HashEmbedder; import hashlib;"
|
||||
"v = HashEmbedder().embed_one('Интеграция amoCRM'); print(hashlib.sha256(repr(v).encode()).hexdigest())"
|
||||
)
|
||||
out = [
|
||||
subprocess.run([sys.executable, "-c", code], env={"PYTHONHASHSEED": seed, "PATH": ""}, capture_output=True, text=True, check=True).stdout
|
||||
for seed in ("1", "2")
|
||||
]
|
||||
assert out[0] and out[0] == out[1]
|
||||
|
||||
|
||||
async def test_async_interface_matches_the_sync_one():
|
||||
[v] = await E.embed(["тест"])
|
||||
assert v == E.embed_one("тест") and E.model == "hash-v1"
|
||||
|
||||
|
||||
# ------------------------------------------------------------ OpenAI-compatible
|
||||
|
||||
|
||||
def ok(vectors, order=None):
|
||||
items = [{"index": i, "embedding": v} for i, v in enumerate(vectors)]
|
||||
if order:
|
||||
items = [items[i] for i in order]
|
||||
return httpx.Response(200, json={"data": items})
|
||||
|
||||
|
||||
def unit(i):
|
||||
v = [0.0] * EMBEDDING_DIM
|
||||
v[i] = 2.0 # deliberately not normalised
|
||||
return v
|
||||
|
||||
|
||||
def client(rec, sleeps, model="text-embedding-3-small", **kw):
|
||||
return OpenAIEmbeddings(
|
||||
model=model, base_url="https://emb.test/v1", api_key="sk-emb", transport=rec.transport,
|
||||
sleep=sleeps, rng=lambda: 0.5, repair_attempts=0, **kw,
|
||||
) # fmt: skip
|
||||
|
||||
|
||||
async def test_request_shape_for_a_text_embedding_3_model(sleeps):
|
||||
rec = Recorder(ok([unit(0), unit(1)]))
|
||||
await client(rec, sleeps).embed(["один", "два"])
|
||||
req = rec.requests[0]
|
||||
assert str(req.url) == "https://emb.test/v1/embeddings" and req.headers["authorization"] == "Bearer sk-emb"
|
||||
body = rec.body()
|
||||
assert body["model"] == "text-embedding-3-small" and body["input"] == ["один", "два"]
|
||||
assert body["encoding_format"] == "float" and body["dimensions"] == EMBEDDING_DIM
|
||||
|
||||
|
||||
async def test_dimensions_are_not_sent_to_models_that_do_not_support_them(sleeps):
|
||||
rec = Recorder(ok([unit(0)]))
|
||||
await client(rec, sleeps, model="text-embedding-ada-002").embed(["x"])
|
||||
assert "dimensions" not in rec.body()
|
||||
|
||||
|
||||
async def test_vectors_are_normalised_and_returned_in_input_order(sleeps):
|
||||
rec = Recorder(ok([unit(0), unit(1), unit(2)], order=[2, 0, 1])) # the provider answered out of order
|
||||
vectors = await client(rec, sleeps).embed(["a", "b", "c"])
|
||||
assert [v.index(1.0) for v in vectors] == [0, 1, 2]
|
||||
assert all(dot(v, v) == pytest.approx(1.0) for v in vectors)
|
||||
|
||||
|
||||
async def test_batches_follow_the_configured_size(sleeps):
|
||||
rec = Recorder(*[ok([unit(i % 5) for i in range(n)]) for n in (64, 64, 22)])
|
||||
vectors = await client(rec, sleeps, batch_size=64).embed([f"t{i}" for i in range(150)])
|
||||
assert len(vectors) == 150 and len(rec.requests) == 3
|
||||
assert [len(rec.body(i)["input"]) for i in range(3)] == [64, 64, 22]
|
||||
|
||||
|
||||
async def test_no_request_for_no_texts(sleeps):
|
||||
rec = Recorder()
|
||||
assert await client(rec, sleeps).embed([]) == [] and rec.requests == []
|
||||
|
||||
|
||||
async def test_very_long_and_empty_texts_are_made_safe(sleeps):
|
||||
rec = Recorder(ok([unit(0), unit(1)]))
|
||||
await client(rec, sleeps).embed(["я" * 20_000, ""])
|
||||
sent = rec.body()["input"]
|
||||
assert len(sent[0]) == 8000 and sent[1] == " "
|
||||
|
||||
|
||||
async def test_wrong_vector_count_and_size_are_errors(sleeps):
|
||||
with pytest.raises(LlmOutputError, match="asked for 2"):
|
||||
await client(Recorder(ok([unit(0)])), sleeps).embed(["a", "b"])
|
||||
short = httpx.Response(200, json={"data": [{"index": 0, "embedding": [0.1, 0.2]}]})
|
||||
with pytest.raises(LlmOutputError, match=f"needs {EMBEDDING_DIM}"):
|
||||
await client(Recorder(short), sleeps).embed(["a"])
|
||||
|
||||
|
||||
async def test_provider_errors_use_the_shared_handling(sleeps):
|
||||
with pytest.raises(LlmAuthError):
|
||||
await client(Recorder(httpx.Response(401, json={"error": {"message": "bad"}})), sleeps).embed(["a"])
|
||||
rec = Recorder(httpx.Response(429, headers={"retry-after": "2"}, json={}), ok([unit(0)]))
|
||||
assert len(await client(rec, sleeps).embed(["a"])) == 1 and sleeps.calls == [2.0]
|
||||
@@ -0,0 +1,163 @@
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from estimator.config.sections import DatabaseCfg
|
||||
from estimator.domain.enums import Role
|
||||
from estimator.domain.errors import EstimatorError, LlmTransientError
|
||||
from estimator.infra.db import SqlReferenceStore, make_engine, make_session_factory
|
||||
from estimator.infra.embeddings import HashEmbedder
|
||||
from estimator.services.references import ReferenceService, parse_rows, read_file
|
||||
|
||||
|
||||
def row(**kw):
|
||||
base = {"project": "P1", "title": "Интеграция amoCRM", "description": "", "role": "backend", "hours": "18", "hours_kind": "actual"}
|
||||
base.update(kw)
|
||||
return base
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- validation
|
||||
|
||||
|
||||
def test_a_good_row_is_parsed_with_defaults_and_synonyms():
|
||||
ok, errors = parse_rows([row(hours="18,5", hours_kind="ФАКТ", role="Backend"), row(title="Без роли", role="", hours_kind="", project="")], default_project="Default")
|
||||
assert errors == []
|
||||
assert (ok[0].hours, ok[0].hours_kind, ok[0].role) == (18.5, "actual", Role.BACKEND)
|
||||
assert (ok[1].role, ok[1].hours_kind, ok[1].project) == (None, "estimate", "Default")
|
||||
|
||||
|
||||
def test_header_names_are_case_insensitive_and_missing_optional_columns_are_fine():
|
||||
ok, errors = parse_rows([{"Project": "P", "TITLE": "Задача", "Hours": "3"}])
|
||||
assert errors == [] and ok[0].title == "Задача" and ok[0].description == ""
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bad,fragment",
|
||||
[
|
||||
(row(title=""), "title"),
|
||||
(row(title="x" * 301), "title"),
|
||||
(row(role="wizard"), "unknown role"),
|
||||
(row(hours="abc"), "hours must be a number"),
|
||||
(row(hours=""), "hours must be a number"),
|
||||
(row(hours="-1"), "between 0 and"),
|
||||
(row(hours="401"), "between 0 and"),
|
||||
(row(hours="nan"), "between 0 and"),
|
||||
(row(hours_kind="maybe"), "hours_kind"),
|
||||
(row(project=""), "project is required"),
|
||||
],
|
||||
)
|
||||
def test_bad_rows_are_reported_with_their_number_and_do_not_stop_the_good_ones(bad, fragment):
|
||||
ok, errors = parse_rows([row(title="Хорошая"), bad, row(title="Тоже хорошая")])
|
||||
assert [r.title for r in ok] == ["Хорошая", "Тоже хорошая"]
|
||||
assert len(errors) == 1 and errors[0].row == 2 and fragment in errors[0].message
|
||||
|
||||
|
||||
def test_the_hours_limit_follows_the_argument():
|
||||
ok, errors = parse_rows([row(hours="100")], max_hours=50)
|
||||
assert ok == [] and "between 0 and 50" in errors[0].message
|
||||
|
||||
|
||||
# -------------------------------------------------------------------------- files
|
||||
|
||||
|
||||
def test_csv_with_a_bom_and_json_are_read(tmp_path):
|
||||
csv_file = tmp_path / "t.csv"
|
||||
csv_file.write_text("project,title,hours\nP,Задача один,5\nP,Задача два,6\n", encoding="utf-8-sig")
|
||||
assert [r["title"] for r in read_file(csv_file)] == ["Задача один", "Задача два"]
|
||||
json_file = tmp_path / "t.json"
|
||||
json_file.write_text(json.dumps([{"project": "P", "title": "T", "hours": 1}]), encoding="utf-8")
|
||||
assert read_file(json_file)[0]["title"] == "T"
|
||||
bad = tmp_path / "bad.json"
|
||||
bad.write_text('{"not": "a list"}')
|
||||
with pytest.raises(EstimatorError):
|
||||
read_file(bad)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------- import
|
||||
|
||||
|
||||
class Flaky(HashEmbedder):
|
||||
"""Fails after `ok_calls` successful batches, like a provider that goes down mid-import."""
|
||||
|
||||
def __init__(self, ok_calls):
|
||||
self.ok_calls, self.calls = ok_calls, 0
|
||||
|
||||
async def embed(self, texts):
|
||||
self.calls += 1
|
||||
if self.calls > self.ok_calls:
|
||||
raise LlmTransientError("provider down", provider="x")
|
||||
return await super().embed(texts)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def parts(dsn):
|
||||
engine = make_engine(DatabaseCfg(dsn=dsn))
|
||||
store = SqlReferenceStore(make_session_factory(engine), engine.dialect.name)
|
||||
yield store
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_import_stores_embeds_and_reports(parts):
|
||||
service = ReferenceService(parts, HashEmbedder(), batch_size=2)
|
||||
report = await service.import_rows([row(title=f"Задача {i}") for i in range(5)] + [row(title="", hours="1")], actor="alex")
|
||||
assert (report.rows, report.valid, report.inserted, report.embedded, report.not_embedded) == (6, 5, 5, 5, 0)
|
||||
assert len(report.errors) == 1 and report.errors[0].row == 6
|
||||
assert (await service.stats()).embedded == 5
|
||||
|
||||
|
||||
async def test_reimport_is_quiet_and_only_changed_text_is_embedded_again(parts):
|
||||
service = ReferenceService(parts, HashEmbedder())
|
||||
rows = [row(title="Интеграция amoCRM", description="сделки и контакты"), row(title="Адаптер MAX")]
|
||||
await service.import_rows(rows)
|
||||
again = await service.import_rows(rows)
|
||||
assert (again.inserted, again.unchanged, again.embedded) == (0, 2, 0)
|
||||
edited = await service.import_rows([row(title="Интеграция amoCRM", description="сделки, контакты и воронки"), row(title="Адаптер MAX", hours="13")])
|
||||
assert (edited.updated, edited.embedded) == (2, 1) # two changed, only the text change needs a new vector
|
||||
|
||||
|
||||
async def test_a_dry_run_changes_nothing(parts):
|
||||
service = ReferenceService(parts, HashEmbedder())
|
||||
report = await service.import_rows([row(), row(title="")], dry_run=True)
|
||||
assert (report.valid, len(report.errors), report.dry_run, report.inserted) == (1, 1, True, 0)
|
||||
assert (await service.stats()).total == 0
|
||||
|
||||
|
||||
async def test_a_provider_failure_leaves_rows_pending_and_the_next_run_finishes(parts):
|
||||
flaky = Flaky(ok_calls=1)
|
||||
service = ReferenceService(parts, flaky, batch_size=2)
|
||||
report = await service.import_rows([row(title=f"Задача {i}") for i in range(5)])
|
||||
assert report.inserted == 5 and report.embedded == 2 and report.not_embedded == 3
|
||||
assert "provider down" in report.embed_error
|
||||
healthy = ReferenceService(parts, HashEmbedder(), batch_size=2)
|
||||
done, left, error = await healthy.embed_pending()
|
||||
assert (done, left, error) == (3, 0, "")
|
||||
assert (await healthy.stats()).stale == 0
|
||||
|
||||
|
||||
async def test_switching_the_model_makes_everything_stale_until_reindexed(parts):
|
||||
await ReferenceService(parts, HashEmbedder()).import_rows([row(), row(title="Другая")])
|
||||
|
||||
class Other(HashEmbedder):
|
||||
model = "hash-v2"
|
||||
|
||||
other = ReferenceService(parts, Other())
|
||||
assert (await other.stats()).stale == 2
|
||||
assert (await other.embed_pending())[0] == 2 and (await other.stats()).stale == 0
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------ search
|
||||
|
||||
|
||||
async def test_search_finds_the_relevant_task_and_respects_the_threshold(parts):
|
||||
service = ReferenceService(parts, HashEmbedder())
|
||||
await service.import_rows([
|
||||
row(title="Интеграция с amoCRM", description="сделки контакты"),
|
||||
row(title="Развёртывание и мониторинг", role="devops"),
|
||||
row(title="Тестирование сценариев бота", role="qa"),
|
||||
]) # fmt: skip
|
||||
hits = await service.search("интеграция с CRM amoCRM", role=None, k=3, min_similarity=0.3)
|
||||
assert hits[0].title == "Интеграция с amoCRM" and hits[0].score > 0.5
|
||||
assert all(h.title != "Развёртывание и мониторинг" for h in hits)
|
||||
assert await service.search("совсем другая тема", role=None, k=3, min_similarity=0.6) == []
|
||||
devops_only = await service.search("мониторинг", role=Role.DEVOPS, k=3, min_similarity=0.2)
|
||||
assert [h.role for h in devops_only] == [Role.DEVOPS]
|
||||
Reference in New Issue
Block a user