Files

161 lines
5.7 KiB
Python

import httpx
import pytest
from estimator.domain.errors import (
LlmAuthError,
LlmOutputError,
LlmRefusalError,
LlmRequestError,
LlmTransientError,
LlmTruncatedError,
)
from estimator.infra.llm import OpenAIChatClient
from tests.unit.llm.conftest import Answer, Recorder, ok_openai
GOOD = '{"text": "hi", "score": 7}'
def client(rec, sleeps, **kw):
return OpenAIChatClient(
base_url="https://api.openai.test/v1",
api_key="sk-secret",
transport=rec.transport,
sleep=sleeps,
rng=lambda: 0.5,
**kw,
)
async def call(c, **kw):
return await c.structured(system="sys", user="usr", schema=Answer, model="m1", **kw)
async def test_request_shape_and_parse(sleeps):
rec = Recorder(ok_openai(GOOD, usage=(12, 3)))
res = await call(client(rec, sleeps), max_tokens=500)
req = rec.requests[0]
assert str(req.url) == "https://api.openai.test/v1/chat/completions"
assert req.headers["authorization"] == "Bearer sk-secret"
body = rec.body()
assert body["model"] == "m1"
assert body["max_completion_tokens"] == 500
assert "max_tokens" not in body and "temperature" not in body and "reasoning_effort" not in body
assert body["messages"][0] == {"role": "system", "content": "sys"}
fmt = body["response_format"]
assert fmt["type"] == "json_schema" and fmt["json_schema"]["strict"] is True
assert fmt["json_schema"]["name"] == "Answer"
assert fmt["json_schema"]["schema"]["additionalProperties"] is False
assert res.parsed == Answer(text="hi", score=7)
assert (res.usage.input_tokens, res.usage.output_tokens) == (12, 3)
assert res.provider == "openai" and res.model == "m1" and not res.via_fallback
async def test_optional_params_sent_only_when_set(sleeps):
rec = Recorder(ok_openai(GOOD))
await call(client(rec, sleeps), temperature=0.3, reasoning_effort="low")
body = rec.body()
assert body["temperature"] == 0.3 and body["reasoning_effort"] == "low"
async def test_code_fence_is_tolerated(sleeps):
rec = Recorder(ok_openai("```json\n" + GOOD + "\n```"))
assert (await call(client(rec, sleeps))).parsed.score == 7
async def test_repair_loop_feeds_back_the_error(sleeps):
rec = Recorder(ok_openai('{"text": "x", "score": 99}'), ok_openai(GOOD))
res = await call(client(rec, sleeps, repair_attempts=1))
assert res.parsed.score == 7
assert len(rec.requests) == 2
second_user = rec.body(1)["messages"][1]["content"]
assert "rejected" in second_user and "score" in second_user
assert res.usage.input_tokens == 20 # usage of both attempts is summed
async def test_repair_exhausted_raises_output_error(sleeps):
rec = Recorder(ok_openai("not json"), ok_openai("still not"))
with pytest.raises(LlmOutputError):
await call(client(rec, sleeps, repair_attempts=1))
assert len(rec.requests) == 2
async def test_no_repair_when_disabled(sleeps):
rec = Recorder(ok_openai("nope"))
with pytest.raises(LlmOutputError):
await call(client(rec, sleeps, repair_attempts=0))
assert len(rec.requests) == 1
async def test_refusal(sleeps):
r = httpx.Response(200, json={"choices": [{"message": {"content": None, "refusal": "no"}, "finish_reason": "stop"}]})
with pytest.raises(LlmRefusalError):
await call(client(Recorder(r), sleeps))
async def test_truncated(sleeps):
with pytest.raises(LlmTruncatedError):
await call(client(Recorder(ok_openai('{"te', finish="length")), sleeps))
async def test_auth_error_not_retried_and_key_not_leaked(sleeps):
rec = Recorder(httpx.Response(401, json={"error": {"message": "bad key"}}))
with pytest.raises(LlmAuthError) as e:
await call(client(rec, sleeps))
assert len(rec.requests) == 1
assert "sk-secret" not in str(e.value)
async def test_429_retries_with_retry_after_then_succeeds(sleeps):
rec = Recorder(
httpx.Response(429, headers={"retry-after": "3"}, json={"error": {"message": "slow down"}}),
ok_openai(GOOD),
)
res = await call(client(rec, sleeps, max_retries=2))
assert res.parsed.score == 7
assert sleeps.calls == [3.0]
async def test_retry_after_is_capped(sleeps):
rec = Recorder(httpx.Response(429, headers={"retry-after": "9999"}, json={}), ok_openai(GOOD))
await call(client(rec, sleeps))
assert sleeps.calls == [30.0]
async def test_5xx_exhausts_retries(sleeps):
rec = Recorder(*[httpx.Response(503, json={"error": {"message": "down"}})] * 3)
with pytest.raises(LlmTransientError):
await call(client(rec, sleeps, max_retries=2))
assert len(rec.requests) == 3 and len(sleeps.calls) == 2
async def test_backoff_grows(sleeps):
rec = Recorder(*[httpx.Response(500, json={})] * 3)
with pytest.raises(LlmTransientError):
await call(client(rec, sleeps, max_retries=2))
assert sleeps.calls[1] > sleeps.calls[0]
async def test_network_error_retried(sleeps):
rec = Recorder(httpx.ConnectError("boom"), ok_openai(GOOD))
assert (await call(client(rec, sleeps))).parsed.score == 7
async def test_network_error_exhausted(sleeps):
rec = Recorder(*[httpx.ReadTimeout("slow")] * 3)
with pytest.raises(LlmTransientError):
await call(client(rec, sleeps, max_retries=2))
async def test_400_is_request_error_without_retry(sleeps):
rec = Recorder(httpx.Response(400, json={"error": {"message": "bad schema"}}))
with pytest.raises(LlmRequestError, match="bad schema"):
await call(client(rec, sleeps))
assert len(rec.requests) == 1
async def test_quota_exhausted_is_not_retried(sleeps):
rec = Recorder(httpx.Response(429, json={"error": {"code": "insufficient_quota", "message": "q"}}))
with pytest.raises(LlmRequestError, match="quota"):
await call(client(rec, sleeps))
assert len(rec.requests) == 1