Add RAG basic implementation

This commit is contained in:
2026-11-09 08:43:30 +03:00
parent 447464c16c
commit 7fa1d774a0
76 changed files with 3030 additions and 124 deletions
+9 -2
View File
@@ -10,6 +10,7 @@ from alembic.migration import MigrationContext
from sqlalchemy.ext.asyncio import create_async_engine
from estimator.infra.db import Base, normalize_dsn
from estimator.infra.db.tables import include_object
PG = os.getenv("TEST_PG_DSN")
@@ -26,7 +27,13 @@ async def _diff(dsn: str):
async with engine.connect() as conn:
def run(sync_conn):
ctx = MigrationContext.configure(sync_conn, opts={"compare_type": True})
ctx = MigrationContext.configure(
sync_conn,
opts={
"compare_type": True,
"include_object": lambda *a: include_object(*a, dialect=sync_conn.dialect.name),
},
)
return compare_metadata(ctx, Base.metadata)
return await conn.run_sync(run)
@@ -49,7 +56,7 @@ def _drop_pg(dsn):
import psycopg
with psycopg.connect(dsn, autocommit=True) as c:
c.execute("drop table if exists estimate_versions, estimates, alembic_version cascade")
c.execute("drop table if exists reference_tasks, estimate_versions, estimates, alembic_version cascade")
@pytest.fixture(params=["sqlite", "postgres"])
+212
View File
@@ -0,0 +1,212 @@
"""The reference store on SQLite (dot product in Python) and on PostgreSQL with pgvector."""
import pytest
from estimator.config.sections import DatabaseCfg
from estimator.domain.embeddings import EMBEDDING_DIM
from estimator.domain.enums import Role
from estimator.domain.models import NewReference
from estimator.infra.db import SqlReferenceStore, make_engine, make_session_factory
MODEL = "test-model"
def vec(**axes) -> list[float]:
"""A vector with weight on named axes (a, b, c ...): cosine similarities are easy to read off."""
v = [0.0] * EMBEDDING_DIM
for name, weight in axes.items():
v[ord(name) - ord("a")] = weight
return v
def ref(title, role=Role.BACKEND, project="P1", hours=10, kind="estimate", description="", source=""):
return NewReference(project=project, title=title, role=role, hours=hours, hours_kind=kind, description=description, source=source)
@pytest.fixture
async def store(backend_dsn):
engine = make_engine(DatabaseCfg(dsn=backend_dsn))
yield SqlReferenceStore(make_session_factory(engine), engine.dialect.name)
await engine.dispose()
async def load(store, items, vectors, model=MODEL):
await store.upsert(items)
pending = dict(await store.pending(model, 1000))
by_title = {text.split("\n")[0]: row_id for row_id, text in pending.items() if False}
rows = await store.pending(model, 1000)
ids = {text.split("\n")[0]: row_id for row_id, text in rows}
await store.set_embeddings(model, [(ids[t], v) for t, v in vectors.items()])
return by_title
# ------------------------------------------------------------------------ upsert
async def test_upsert_counts_and_idempotence(store):
items = [ref("Интеграция amoCRM"), ref("Адаптер MAX", hours=12)]
first = await store.upsert(items, actor="alex")
assert (first.inserted, first.updated, first.unchanged) == (2, 0, 0)
again = await store.upsert(items)
assert (again.inserted, again.updated, again.unchanged) == (0, 0, 2)
changed = await store.upsert([ref("Адаптер MAX", hours=15), ref("Новая задача")])
assert (changed.inserted, changed.updated) == (1, 1)
assert (await store.stats(MODEL)).total == 3
async def test_the_same_task_is_recognised_ignoring_case_and_spaces_but_not_role_or_project(store):
await store.upsert([ref("Интеграция CRM")])
r = await store.upsert([ref(" интеграция crm "), ref("Интеграция CRM", role=Role.QA), ref("Интеграция CRM", project="P2")])
# the first is found by its key (so not a new row) and its changed spelling counts as an update
assert (r.inserted, r.updated, r.unchanged) == (2, 1, 0)
assert (await store.stats(MODEL)).total == 3
async def test_duplicates_inside_one_import_keep_the_last(store):
r = await store.upsert([ref("Задача", hours=5), ref("Задача", hours=9)])
assert r.inserted == 1
await load(store, [], {})
[hit] = await _only_hit(store, "Задача", 9)
assert hit.hours == 9
async def _only_hit(store, title, expected_hours):
rows = await store.pending(MODEL, 10)
[(row_id, _)] = rows
await store.set_embeddings(MODEL, [(row_id, vec(a=1))])
return (await store.search([vec(a=1)], [None], model=MODEL, k=5, min_similarity=0.5))[0]
async def test_a_large_import_is_chunked_and_still_idempotent(store):
items = [ref(f"Задача {i}") for i in range(700)]
assert (await store.upsert(items)).inserted == 700
again = await store.upsert(items)
assert (again.inserted, again.unchanged) == (0, 700)
async def test_changing_the_text_drops_the_vector_but_changing_hours_does_not(store):
await store.upsert([ref("Тест", description="первое описание")])
[(row_id, _)] = await store.pending(MODEL, 10)
await store.set_embeddings(MODEL, [(row_id, vec(a=1))])
assert await store.pending(MODEL, 10) == []
await store.upsert([ref("Тест", description="первое описание", hours=99)]) # only the hours
assert await store.pending(MODEL, 10) == []
hit = (await store.search([vec(a=1)], [None], model=MODEL, k=1, min_similarity=0.5))[0][0]
assert hit.hours == 99
await store.upsert([ref("Тест", description="другое описание", hours=99)]) # the text changed
assert [r for r, _ in await store.pending(MODEL, 10)] == [row_id]
# -------------------------------------------------------------------- embeddings
async def test_pending_covers_rows_without_a_vector_or_with_another_models_vector(store):
await store.upsert([ref("A"), ref("B")])
rows = {text: row_id for row_id, text in await store.pending("m1", 10)}
assert set(rows) == {"A", "B"}
await store.set_embeddings("m1", [(rows["A"], vec(a=1))])
assert [t for _, t in await store.pending("m1", 10)] == ["B"]
assert len(await store.pending("m2", 10)) == 2 # a different model sees everything as stale
assert len(await store.pending("m1", 1)) == 1 # the limit applies
async def test_vectors_of_the_wrong_size_are_refused(store):
await store.upsert([ref("A")])
[(row_id, _)] = await store.pending(MODEL, 10)
with pytest.raises(ValueError, match=str(EMBEDDING_DIM)):
await store.set_embeddings(MODEL, [(row_id, [1.0, 2.0])])
# ------------------------------------------------------------------------- search
async def corpus(store):
items = [
ref("Интеграция amoCRM", hours=18, kind="actual", description="CRM"),
ref("Интеграция Bitrix", hours=22, kind="estimate"),
ref("Адаптер MAX", hours=12),
ref("Тестирование", role=Role.QA, hours=20),
ref("Общая задача без роли", role=None, hours=7),
]
await store.upsert(items)
rows = {t.split("\n")[0]: i for i, t in await store.pending(MODEL, 100)}
await store.set_embeddings(MODEL, [
(rows["Интеграция amoCRM"], vec(a=1)), # cosine 1.0 with the query below
(rows["Интеграция Bitrix"], vec(a=1, b=1)), # 0.707
(rows["Адаптер MAX"], vec(b=1)), # 0.0
(rows["Тестирование"], vec(a=1, c=1.2)), # 0.640, and a QA task
(rows["Общая задача без роли"], vec(a=3, b=1)), # 0.949, no role
]) # fmt: skip
async def test_results_are_ordered_by_similarity_with_scores(store):
await corpus(store)
[hits] = await store.search([vec(a=1)], [None], model=MODEL, k=10, min_similarity=0.5)
assert [h.title for h in hits] == ["Интеграция amoCRM", "Общая задача без роли", "Интеграция Bitrix", "Тестирование"]
assert [h.score for h in hits] == pytest.approx([1.0, 0.9487, 0.7071, 0.6402], abs=1e-3)
assert hits[0].hours == 18 and hits[0].hours_kind == "actual" and hits[0].role == Role.BACKEND
assert hits[0].project == "P1" and hits[0].description == "CRM"
async def test_threshold_and_k_cut_the_list(store):
await corpus(store)
[hits] = await store.search([vec(a=1)], [None], model=MODEL, k=10, min_similarity=0.9)
assert [h.title for h in hits] == ["Интеграция amoCRM", "Общая задача без роли"]
[top2] = await store.search([vec(a=1)], [None], model=MODEL, k=2, min_similarity=0.0)
assert len(top2) == 2
[none] = await store.search([vec(c=1, d=1)], [None], model=MODEL, k=5, min_similarity=0.99)
assert none == []
async def test_a_role_filter_keeps_that_role_and_role_less_tasks(store):
await corpus(store)
[hits] = await store.search([vec(a=1)], [Role.BACKEND], model=MODEL, k=10, min_similarity=0.5)
titles = [h.title for h in hits]
assert "Тестирование" not in titles and "Общая задача без роли" in titles and "Интеграция amoCRM" in titles
[qa] = await store.search([vec(a=1)], [Role.QA], model=MODEL, k=10, min_similarity=0.5)
assert [h.title for h in qa] == ["Общая задача без роли", "Тестирование"]
async def test_each_query_in_a_batch_is_answered_separately(store):
await corpus(store)
r = await store.search([vec(a=1), vec(b=1), vec(d=1)], [None, None, None], model=MODEL, k=1, min_similarity=0.5)
assert [x[0].title if x else None for x in r] == ["Интеграция amoCRM", "Адаптер MAX", None]
assert await store.search([], [], model=MODEL, k=3, min_similarity=0) == []
async def test_vectors_from_another_model_are_ignored(store):
await corpus(store)
[hits] = await store.search([vec(a=1)], [None], model="some-other-model", k=5, min_similarity=0.0)
assert hits == []
async def test_stats_and_delete(store):
await corpus(store)
await store.upsert([ref("Из другого проекта", project="P2", kind="actual")])
st = await store.stats(MODEL)
assert (st.total, st.embedded, st.stale, st.projects, st.actual, st.estimates) == (6, 5, 1, 2, 2, 4)
assert (await store.stats("other")).embedded == 0
assert await store.delete_project("P2") == 1 and await store.delete_project("P2") == 0
assert (await store.stats(MODEL)).total == 5
async def test_a_database_without_the_schema_is_reported(store):
await store.check_ready() # the schema exists in these tests
# ----------------------------------------------------------- PostgreSQL specifics
async def test_postgres_uses_an_hnsw_index_on_the_vector_column(store, backend_dsn):
if not backend_dsn.startswith("postgresql"):
pytest.skip("pgvector index: PostgreSQL only")
import psycopg
with psycopg.connect(backend_dsn) as c:
indexes = {r[0]: r[1] for r in c.execute("select indexname, indexdef from pg_indexes where tablename = 'reference_tasks'")}
ext = c.execute("select extversion from pg_extension where extname = 'vector'").fetchone()
column = c.execute("select format_type(atttypid, atttypmod) from pg_attribute where attrelid = 'reference_tasks'::regclass and attname = 'embedding'").fetchone()
assert ext is not None
assert column[0] == f"vector({EMBEDDING_DIM})"
assert "hnsw" in indexes["ix_reference_tasks_embedding_hnsw"] and "vector_cosine_ops" in indexes["ix_reference_tasks_embedding_hnsw"]