161 lines
5.7 KiB
Python
161 lines
5.7 KiB
Python
import httpx
|
|
import pytest
|
|
|
|
from estimator.domain.errors import (
|
|
LlmAuthError,
|
|
LlmOutputError,
|
|
LlmRefusalError,
|
|
LlmRequestError,
|
|
LlmTransientError,
|
|
LlmTruncatedError,
|
|
)
|
|
from estimator.infra.llm import OpenAIChatClient
|
|
from tests.unit.llm.conftest import Answer, Recorder, ok_openai
|
|
|
|
GOOD = '{"text": "hi", "score": 7}'
|
|
|
|
|
|
def client(rec, sleeps, **kw):
|
|
return OpenAIChatClient(
|
|
base_url="https://api.openai.test/v1",
|
|
api_key="sk-secret",
|
|
transport=rec.transport,
|
|
sleep=sleeps,
|
|
rng=lambda: 0.5,
|
|
**kw,
|
|
)
|
|
|
|
|
|
async def call(c, **kw):
|
|
return await c.structured(system="sys", user="usr", schema=Answer, model="m1", **kw)
|
|
|
|
|
|
async def test_request_shape_and_parse(sleeps):
|
|
rec = Recorder(ok_openai(GOOD, usage=(12, 3)))
|
|
res = await call(client(rec, sleeps), max_tokens=500)
|
|
req = rec.requests[0]
|
|
assert str(req.url) == "https://api.openai.test/v1/chat/completions"
|
|
assert req.headers["authorization"] == "Bearer sk-secret"
|
|
body = rec.body()
|
|
assert body["model"] == "m1"
|
|
assert body["max_completion_tokens"] == 500
|
|
assert "max_tokens" not in body and "temperature" not in body and "reasoning_effort" not in body
|
|
assert body["messages"][0] == {"role": "system", "content": "sys"}
|
|
fmt = body["response_format"]
|
|
assert fmt["type"] == "json_schema" and fmt["json_schema"]["strict"] is True
|
|
assert fmt["json_schema"]["name"] == "Answer"
|
|
assert fmt["json_schema"]["schema"]["additionalProperties"] is False
|
|
assert res.parsed == Answer(text="hi", score=7)
|
|
assert (res.usage.input_tokens, res.usage.output_tokens) == (12, 3)
|
|
assert res.provider == "openai" and res.model == "m1" and not res.via_fallback
|
|
|
|
|
|
async def test_optional_params_sent_only_when_set(sleeps):
|
|
rec = Recorder(ok_openai(GOOD))
|
|
await call(client(rec, sleeps), temperature=0.3, reasoning_effort="low")
|
|
body = rec.body()
|
|
assert body["temperature"] == 0.3 and body["reasoning_effort"] == "low"
|
|
|
|
|
|
async def test_code_fence_is_tolerated(sleeps):
|
|
rec = Recorder(ok_openai("```json\n" + GOOD + "\n```"))
|
|
assert (await call(client(rec, sleeps))).parsed.score == 7
|
|
|
|
|
|
async def test_repair_loop_feeds_back_the_error(sleeps):
|
|
rec = Recorder(ok_openai('{"text": "x", "score": 99}'), ok_openai(GOOD))
|
|
res = await call(client(rec, sleeps, repair_attempts=1))
|
|
assert res.parsed.score == 7
|
|
assert len(rec.requests) == 2
|
|
second_user = rec.body(1)["messages"][1]["content"]
|
|
assert "rejected" in second_user and "score" in second_user
|
|
assert res.usage.input_tokens == 20 # usage of both attempts is summed
|
|
|
|
|
|
async def test_repair_exhausted_raises_output_error(sleeps):
|
|
rec = Recorder(ok_openai("not json"), ok_openai("still not"))
|
|
with pytest.raises(LlmOutputError):
|
|
await call(client(rec, sleeps, repair_attempts=1))
|
|
assert len(rec.requests) == 2
|
|
|
|
|
|
async def test_no_repair_when_disabled(sleeps):
|
|
rec = Recorder(ok_openai("nope"))
|
|
with pytest.raises(LlmOutputError):
|
|
await call(client(rec, sleeps, repair_attempts=0))
|
|
assert len(rec.requests) == 1
|
|
|
|
|
|
async def test_refusal(sleeps):
|
|
r = httpx.Response(200, json={"choices": [{"message": {"content": None, "refusal": "no"}, "finish_reason": "stop"}]})
|
|
with pytest.raises(LlmRefusalError):
|
|
await call(client(Recorder(r), sleeps))
|
|
|
|
|
|
async def test_truncated(sleeps):
|
|
with pytest.raises(LlmTruncatedError):
|
|
await call(client(Recorder(ok_openai('{"te', finish="length")), sleeps))
|
|
|
|
|
|
async def test_auth_error_not_retried_and_key_not_leaked(sleeps):
|
|
rec = Recorder(httpx.Response(401, json={"error": {"message": "bad key"}}))
|
|
with pytest.raises(LlmAuthError) as e:
|
|
await call(client(rec, sleeps))
|
|
assert len(rec.requests) == 1
|
|
assert "sk-secret" not in str(e.value)
|
|
|
|
|
|
async def test_429_retries_with_retry_after_then_succeeds(sleeps):
|
|
rec = Recorder(
|
|
httpx.Response(429, headers={"retry-after": "3"}, json={"error": {"message": "slow down"}}),
|
|
ok_openai(GOOD),
|
|
)
|
|
res = await call(client(rec, sleeps, max_retries=2))
|
|
assert res.parsed.score == 7
|
|
assert sleeps.calls == [3.0]
|
|
|
|
|
|
async def test_retry_after_is_capped(sleeps):
|
|
rec = Recorder(httpx.Response(429, headers={"retry-after": "9999"}, json={}), ok_openai(GOOD))
|
|
await call(client(rec, sleeps))
|
|
assert sleeps.calls == [30.0]
|
|
|
|
|
|
async def test_5xx_exhausts_retries(sleeps):
|
|
rec = Recorder(*[httpx.Response(503, json={"error": {"message": "down"}})] * 3)
|
|
with pytest.raises(LlmTransientError):
|
|
await call(client(rec, sleeps, max_retries=2))
|
|
assert len(rec.requests) == 3 and len(sleeps.calls) == 2
|
|
|
|
|
|
async def test_backoff_grows(sleeps):
|
|
rec = Recorder(*[httpx.Response(500, json={})] * 3)
|
|
with pytest.raises(LlmTransientError):
|
|
await call(client(rec, sleeps, max_retries=2))
|
|
assert sleeps.calls[1] > sleeps.calls[0]
|
|
|
|
|
|
async def test_network_error_retried(sleeps):
|
|
rec = Recorder(httpx.ConnectError("boom"), ok_openai(GOOD))
|
|
assert (await call(client(rec, sleeps))).parsed.score == 7
|
|
|
|
|
|
async def test_network_error_exhausted(sleeps):
|
|
rec = Recorder(*[httpx.ReadTimeout("slow")] * 3)
|
|
with pytest.raises(LlmTransientError):
|
|
await call(client(rec, sleeps, max_retries=2))
|
|
|
|
|
|
async def test_400_is_request_error_without_retry(sleeps):
|
|
rec = Recorder(httpx.Response(400, json={"error": {"message": "bad schema"}}))
|
|
with pytest.raises(LlmRequestError, match="bad schema"):
|
|
await call(client(rec, sleeps))
|
|
assert len(rec.requests) == 1
|
|
|
|
|
|
async def test_quota_exhausted_is_not_retried(sleeps):
|
|
rec = Recorder(httpx.Response(429, json={"error": {"code": "insufficient_quota", "message": "q"}}))
|
|
with pytest.raises(LlmRequestError, match="quota"):
|
|
await call(client(rec, sleeps))
|
|
assert len(rec.requests) == 1
|