Files
2026-11-09 08:43:30 +03:00

140 lines
5.3 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import subprocess
import sys
import httpx
import pytest
from estimator.domain.embeddings import EMBEDDING_DIM
from estimator.domain.errors import LlmAuthError, LlmOutputError
from estimator.infra.embeddings import HashEmbedder, OpenAIEmbeddings
from tests.unit.llm.conftest import Recorder
E = HashEmbedder()
def dot(a, b):
return sum(x * y for x, y in zip(a, b, strict=True))
# ----------------------------------------------------------------- hash embedder
def test_vectors_have_the_schema_dimension_and_unit_length():
v = E.embed_one("Интеграция с amoCRM")
assert len(v) == EMBEDDING_DIM and dot(v, v) == pytest.approx(1.0)
def test_related_texts_are_closer_than_unrelated_ones():
a = E.embed_one("Интеграция с amoCRM")
related = E.embed_one("интеграции amoCRM и сделок")
unrelated = E.embed_one("Развёртывание и мониторинг")
assert dot(a, related) > 0.6
assert dot(a, unrelated) < 0.1
def test_case_and_russian_endings_do_not_matter():
assert dot(E.embed_one("Тестирование бота"), E.embed_one("ТЕСТИРОВАНИЯ БОТА")) > 0.99
def test_text_without_usable_words_still_gives_a_valid_vector():
for text in ("", " ", "и для на", "!!!"):
v = E.embed_one(text)
assert dot(v, v) == pytest.approx(1.0)
def test_embedding_is_deterministic_across_processes_and_hash_seeds():
code = (
"from estimator.infra.embeddings import HashEmbedder; import hashlib;"
"v = HashEmbedder().embed_one('Интеграция amoCRM'); print(hashlib.sha256(repr(v).encode()).hexdigest())"
)
out = [
subprocess.run([sys.executable, "-c", code], env={"PYTHONHASHSEED": seed, "PATH": ""}, capture_output=True, text=True, check=True).stdout
for seed in ("1", "2")
]
assert out[0] and out[0] == out[1]
async def test_async_interface_matches_the_sync_one():
[v] = await E.embed(["тест"])
assert v == E.embed_one("тест") and E.model == "hash-v1"
# ------------------------------------------------------------ OpenAI-compatible
def ok(vectors, order=None):
items = [{"index": i, "embedding": v} for i, v in enumerate(vectors)]
if order:
items = [items[i] for i in order]
return httpx.Response(200, json={"data": items})
def unit(i):
v = [0.0] * EMBEDDING_DIM
v[i] = 2.0 # deliberately not normalised
return v
def client(rec, sleeps, model="text-embedding-3-small", **kw):
return OpenAIEmbeddings(
model=model, base_url="https://emb.test/v1", api_key="sk-emb", transport=rec.transport,
sleep=sleeps, rng=lambda: 0.5, repair_attempts=0, **kw,
) # fmt: skip
async def test_request_shape_for_a_text_embedding_3_model(sleeps):
rec = Recorder(ok([unit(0), unit(1)]))
await client(rec, sleeps).embed(["один", "два"])
req = rec.requests[0]
assert str(req.url) == "https://emb.test/v1/embeddings" and req.headers["authorization"] == "Bearer sk-emb"
body = rec.body()
assert body["model"] == "text-embedding-3-small" and body["input"] == ["один", "два"]
assert body["encoding_format"] == "float" and body["dimensions"] == EMBEDDING_DIM
async def test_dimensions_are_not_sent_to_models_that_do_not_support_them(sleeps):
rec = Recorder(ok([unit(0)]))
await client(rec, sleeps, model="text-embedding-ada-002").embed(["x"])
assert "dimensions" not in rec.body()
async def test_vectors_are_normalised_and_returned_in_input_order(sleeps):
rec = Recorder(ok([unit(0), unit(1), unit(2)], order=[2, 0, 1])) # the provider answered out of order
vectors = await client(rec, sleeps).embed(["a", "b", "c"])
assert [v.index(1.0) for v in vectors] == [0, 1, 2]
assert all(dot(v, v) == pytest.approx(1.0) for v in vectors)
async def test_batches_follow_the_configured_size(sleeps):
rec = Recorder(*[ok([unit(i % 5) for i in range(n)]) for n in (64, 64, 22)])
vectors = await client(rec, sleeps, batch_size=64).embed([f"t{i}" for i in range(150)])
assert len(vectors) == 150 and len(rec.requests) == 3
assert [len(rec.body(i)["input"]) for i in range(3)] == [64, 64, 22]
async def test_no_request_for_no_texts(sleeps):
rec = Recorder()
assert await client(rec, sleeps).embed([]) == [] and rec.requests == []
async def test_very_long_and_empty_texts_are_made_safe(sleeps):
rec = Recorder(ok([unit(0), unit(1)]))
await client(rec, sleeps).embed(["я" * 20_000, ""])
sent = rec.body()["input"]
assert len(sent[0]) == 8000 and sent[1] == " "
async def test_wrong_vector_count_and_size_are_errors(sleeps):
with pytest.raises(LlmOutputError, match="asked for 2"):
await client(Recorder(ok([unit(0)])), sleeps).embed(["a", "b"])
short = httpx.Response(200, json={"data": [{"index": 0, "embedding": [0.1, 0.2]}]})
with pytest.raises(LlmOutputError, match=f"needs {EMBEDDING_DIM}"):
await client(Recorder(short), sleeps).embed(["a"])
async def test_provider_errors_use_the_shared_handling(sleeps):
with pytest.raises(LlmAuthError):
await client(Recorder(httpx.Response(401, json={"error": {"message": "bad"}})), sleeps).embed(["a"])
rec = Recorder(httpx.Response(429, headers={"retry-after": "2"}, json={}), ok([unit(0)]))
assert len(await client(rec, sleeps).embed(["a"])) == 1 and sleeps.calls == [2.0]