import httpx import pytest from estimator.domain.errors import ( LlmAuthError, LlmOutputError, LlmRefusalError, LlmRequestError, LlmTransientError, LlmTruncatedError, ) from estimator.infra.llm import OpenAIChatClient from tests.unit.llm.conftest import Answer, Recorder, ok_openai GOOD = '{"text": "hi", "score": 7}' def client(rec, sleeps, **kw): return OpenAIChatClient( base_url="https://api.openai.test/v1", api_key="sk-secret", transport=rec.transport, sleep=sleeps, rng=lambda: 0.5, **kw, ) async def call(c, **kw): return await c.structured(system="sys", user="usr", schema=Answer, model="m1", **kw) async def test_request_shape_and_parse(sleeps): rec = Recorder(ok_openai(GOOD, usage=(12, 3))) res = await call(client(rec, sleeps), max_tokens=500) req = rec.requests[0] assert str(req.url) == "https://api.openai.test/v1/chat/completions" assert req.headers["authorization"] == "Bearer sk-secret" body = rec.body() assert body["model"] == "m1" assert body["max_completion_tokens"] == 500 assert "max_tokens" not in body and "temperature" not in body and "reasoning_effort" not in body assert body["messages"][0] == {"role": "system", "content": "sys"} fmt = body["response_format"] assert fmt["type"] == "json_schema" and fmt["json_schema"]["strict"] is True assert fmt["json_schema"]["name"] == "Answer" assert fmt["json_schema"]["schema"]["additionalProperties"] is False assert res.parsed == Answer(text="hi", score=7) assert (res.usage.input_tokens, res.usage.output_tokens) == (12, 3) assert res.provider == "openai" and res.model == "m1" and not res.via_fallback async def test_optional_params_sent_only_when_set(sleeps): rec = Recorder(ok_openai(GOOD)) await call(client(rec, sleeps), temperature=0.3, reasoning_effort="low") body = rec.body() assert body["temperature"] == 0.3 and body["reasoning_effort"] == "low" async def test_code_fence_is_tolerated(sleeps): rec = Recorder(ok_openai("```json\n" + GOOD + "\n```")) assert (await call(client(rec, sleeps))).parsed.score == 7 async def test_repair_loop_feeds_back_the_error(sleeps): rec = Recorder(ok_openai('{"text": "x", "score": 99}'), ok_openai(GOOD)) res = await call(client(rec, sleeps, repair_attempts=1)) assert res.parsed.score == 7 assert len(rec.requests) == 2 second_user = rec.body(1)["messages"][1]["content"] assert "rejected" in second_user and "score" in second_user assert res.usage.input_tokens == 20 # usage of both attempts is summed async def test_repair_exhausted_raises_output_error(sleeps): rec = Recorder(ok_openai("not json"), ok_openai("still not")) with pytest.raises(LlmOutputError): await call(client(rec, sleeps, repair_attempts=1)) assert len(rec.requests) == 2 async def test_no_repair_when_disabled(sleeps): rec = Recorder(ok_openai("nope")) with pytest.raises(LlmOutputError): await call(client(rec, sleeps, repair_attempts=0)) assert len(rec.requests) == 1 async def test_refusal(sleeps): r = httpx.Response(200, json={"choices": [{"message": {"content": None, "refusal": "no"}, "finish_reason": "stop"}]}) with pytest.raises(LlmRefusalError): await call(client(Recorder(r), sleeps)) async def test_truncated(sleeps): with pytest.raises(LlmTruncatedError): await call(client(Recorder(ok_openai('{"te', finish="length")), sleeps)) async def test_auth_error_not_retried_and_key_not_leaked(sleeps): rec = Recorder(httpx.Response(401, json={"error": {"message": "bad key"}})) with pytest.raises(LlmAuthError) as e: await call(client(rec, sleeps)) assert len(rec.requests) == 1 assert "sk-secret" not in str(e.value) async def test_429_retries_with_retry_after_then_succeeds(sleeps): rec = Recorder( httpx.Response(429, headers={"retry-after": "3"}, json={"error": {"message": "slow down"}}), ok_openai(GOOD), ) res = await call(client(rec, sleeps, max_retries=2)) assert res.parsed.score == 7 assert sleeps.calls == [3.0] async def test_retry_after_is_capped(sleeps): rec = Recorder(httpx.Response(429, headers={"retry-after": "9999"}, json={}), ok_openai(GOOD)) await call(client(rec, sleeps)) assert sleeps.calls == [30.0] async def test_5xx_exhausts_retries(sleeps): rec = Recorder(*[httpx.Response(503, json={"error": {"message": "down"}})] * 3) with pytest.raises(LlmTransientError): await call(client(rec, sleeps, max_retries=2)) assert len(rec.requests) == 3 and len(sleeps.calls) == 2 async def test_backoff_grows(sleeps): rec = Recorder(*[httpx.Response(500, json={})] * 3) with pytest.raises(LlmTransientError): await call(client(rec, sleeps, max_retries=2)) assert sleeps.calls[1] > sleeps.calls[0] async def test_network_error_retried(sleeps): rec = Recorder(httpx.ConnectError("boom"), ok_openai(GOOD)) assert (await call(client(rec, sleeps))).parsed.score == 7 async def test_network_error_exhausted(sleeps): rec = Recorder(*[httpx.ReadTimeout("slow")] * 3) with pytest.raises(LlmTransientError): await call(client(rec, sleeps, max_retries=2)) async def test_400_is_request_error_without_retry(sleeps): rec = Recorder(httpx.Response(400, json={"error": {"message": "bad schema"}})) with pytest.raises(LlmRequestError, match="bad schema"): await call(client(rec, sleeps)) assert len(rec.requests) == 1 async def test_quota_exhausted_is_not_retried(sleeps): rec = Recorder(httpx.Response(429, json={"error": {"code": "insufficient_quota", "message": "q"}})) with pytest.raises(LlmRequestError, match="quota"): await call(client(rec, sleeps)) assert len(rec.requests) == 1