Add RAG basic implementation
This commit is contained in:
@@ -0,0 +1,139 @@
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from estimator.domain.embeddings import EMBEDDING_DIM
|
||||
from estimator.domain.errors import LlmAuthError, LlmOutputError
|
||||
from estimator.infra.embeddings import HashEmbedder, OpenAIEmbeddings
|
||||
from tests.unit.llm.conftest import Recorder
|
||||
|
||||
E = HashEmbedder()
|
||||
|
||||
|
||||
def dot(a, b):
|
||||
return sum(x * y for x, y in zip(a, b, strict=True))
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- hash embedder
|
||||
|
||||
|
||||
def test_vectors_have_the_schema_dimension_and_unit_length():
|
||||
v = E.embed_one("Интеграция с amoCRM")
|
||||
assert len(v) == EMBEDDING_DIM and dot(v, v) == pytest.approx(1.0)
|
||||
|
||||
|
||||
def test_related_texts_are_closer_than_unrelated_ones():
|
||||
a = E.embed_one("Интеграция с amoCRM")
|
||||
related = E.embed_one("интеграции amoCRM и сделок")
|
||||
unrelated = E.embed_one("Развёртывание и мониторинг")
|
||||
assert dot(a, related) > 0.6
|
||||
assert dot(a, unrelated) < 0.1
|
||||
|
||||
|
||||
def test_case_and_russian_endings_do_not_matter():
|
||||
assert dot(E.embed_one("Тестирование бота"), E.embed_one("ТЕСТИРОВАНИЯ БОТА")) > 0.99
|
||||
|
||||
|
||||
def test_text_without_usable_words_still_gives_a_valid_vector():
|
||||
for text in ("", " ", "и для на", "!!!"):
|
||||
v = E.embed_one(text)
|
||||
assert dot(v, v) == pytest.approx(1.0)
|
||||
|
||||
|
||||
def test_embedding_is_deterministic_across_processes_and_hash_seeds():
|
||||
code = (
|
||||
"from estimator.infra.embeddings import HashEmbedder; import hashlib;"
|
||||
"v = HashEmbedder().embed_one('Интеграция amoCRM'); print(hashlib.sha256(repr(v).encode()).hexdigest())"
|
||||
)
|
||||
out = [
|
||||
subprocess.run([sys.executable, "-c", code], env={"PYTHONHASHSEED": seed, "PATH": ""}, capture_output=True, text=True, check=True).stdout
|
||||
for seed in ("1", "2")
|
||||
]
|
||||
assert out[0] and out[0] == out[1]
|
||||
|
||||
|
||||
async def test_async_interface_matches_the_sync_one():
|
||||
[v] = await E.embed(["тест"])
|
||||
assert v == E.embed_one("тест") and E.model == "hash-v1"
|
||||
|
||||
|
||||
# ------------------------------------------------------------ OpenAI-compatible
|
||||
|
||||
|
||||
def ok(vectors, order=None):
|
||||
items = [{"index": i, "embedding": v} for i, v in enumerate(vectors)]
|
||||
if order:
|
||||
items = [items[i] for i in order]
|
||||
return httpx.Response(200, json={"data": items})
|
||||
|
||||
|
||||
def unit(i):
|
||||
v = [0.0] * EMBEDDING_DIM
|
||||
v[i] = 2.0 # deliberately not normalised
|
||||
return v
|
||||
|
||||
|
||||
def client(rec, sleeps, model="text-embedding-3-small", **kw):
|
||||
return OpenAIEmbeddings(
|
||||
model=model, base_url="https://emb.test/v1", api_key="sk-emb", transport=rec.transport,
|
||||
sleep=sleeps, rng=lambda: 0.5, repair_attempts=0, **kw,
|
||||
) # fmt: skip
|
||||
|
||||
|
||||
async def test_request_shape_for_a_text_embedding_3_model(sleeps):
|
||||
rec = Recorder(ok([unit(0), unit(1)]))
|
||||
await client(rec, sleeps).embed(["один", "два"])
|
||||
req = rec.requests[0]
|
||||
assert str(req.url) == "https://emb.test/v1/embeddings" and req.headers["authorization"] == "Bearer sk-emb"
|
||||
body = rec.body()
|
||||
assert body["model"] == "text-embedding-3-small" and body["input"] == ["один", "два"]
|
||||
assert body["encoding_format"] == "float" and body["dimensions"] == EMBEDDING_DIM
|
||||
|
||||
|
||||
async def test_dimensions_are_not_sent_to_models_that_do_not_support_them(sleeps):
|
||||
rec = Recorder(ok([unit(0)]))
|
||||
await client(rec, sleeps, model="text-embedding-ada-002").embed(["x"])
|
||||
assert "dimensions" not in rec.body()
|
||||
|
||||
|
||||
async def test_vectors_are_normalised_and_returned_in_input_order(sleeps):
|
||||
rec = Recorder(ok([unit(0), unit(1), unit(2)], order=[2, 0, 1])) # the provider answered out of order
|
||||
vectors = await client(rec, sleeps).embed(["a", "b", "c"])
|
||||
assert [v.index(1.0) for v in vectors] == [0, 1, 2]
|
||||
assert all(dot(v, v) == pytest.approx(1.0) for v in vectors)
|
||||
|
||||
|
||||
async def test_batches_follow_the_configured_size(sleeps):
|
||||
rec = Recorder(*[ok([unit(i % 5) for i in range(n)]) for n in (64, 64, 22)])
|
||||
vectors = await client(rec, sleeps, batch_size=64).embed([f"t{i}" for i in range(150)])
|
||||
assert len(vectors) == 150 and len(rec.requests) == 3
|
||||
assert [len(rec.body(i)["input"]) for i in range(3)] == [64, 64, 22]
|
||||
|
||||
|
||||
async def test_no_request_for_no_texts(sleeps):
|
||||
rec = Recorder()
|
||||
assert await client(rec, sleeps).embed([]) == [] and rec.requests == []
|
||||
|
||||
|
||||
async def test_very_long_and_empty_texts_are_made_safe(sleeps):
|
||||
rec = Recorder(ok([unit(0), unit(1)]))
|
||||
await client(rec, sleeps).embed(["я" * 20_000, ""])
|
||||
sent = rec.body()["input"]
|
||||
assert len(sent[0]) == 8000 and sent[1] == " "
|
||||
|
||||
|
||||
async def test_wrong_vector_count_and_size_are_errors(sleeps):
|
||||
with pytest.raises(LlmOutputError, match="asked for 2"):
|
||||
await client(Recorder(ok([unit(0)])), sleeps).embed(["a", "b"])
|
||||
short = httpx.Response(200, json={"data": [{"index": 0, "embedding": [0.1, 0.2]}]})
|
||||
with pytest.raises(LlmOutputError, match=f"needs {EMBEDDING_DIM}"):
|
||||
await client(Recorder(short), sleeps).embed(["a"])
|
||||
|
||||
|
||||
async def test_provider_errors_use_the_shared_handling(sleeps):
|
||||
with pytest.raises(LlmAuthError):
|
||||
await client(Recorder(httpx.Response(401, json={"error": {"message": "bad"}})), sleeps).embed(["a"])
|
||||
rec = Recorder(httpx.Response(429, headers={"retry-after": "2"}, json={}), ok([unit(0)]))
|
||||
assert len(await client(rec, sleeps).embed(["a"])) == 1 and sleeps.calls == [2.0]
|
||||
Reference in New Issue
Block a user