Files
estimator/tests/unit/llm/test_anthropic.py
T

115 lines
4.0 KiB
Python

import httpx
import pytest
from estimator.domain.errors import (
LlmAuthError,
LlmOutputError,
LlmRefusalError,
LlmTransientError,
LlmTruncatedError,
)
from estimator.infra.llm import AnthropicClient
from tests.unit.llm.conftest import Answer, Recorder, ok_anthropic
GOOD = '{"text": "hi", "score": 4}'
def client(rec, sleeps, **kw):
return AnthropicClient(
base_url="https://api.anthropic.test",
api_key="sk-ant-secret",
transport=rec.transport,
sleep=sleeps,
rng=lambda: 0.5,
**kw,
)
async def call(c, **kw):
return await c.structured(system="sys", user="usr", schema=Answer, model="claude-x", **kw)
async def test_request_shape_and_parse(sleeps):
rec = Recorder(ok_anthropic(GOOD, usage=(20, 8)))
res = await call(client(rec, sleeps), max_tokens=700)
req = rec.requests[0]
assert str(req.url) == "https://api.anthropic.test/v1/messages"
assert req.headers["x-api-key"] == "sk-ant-secret"
assert req.headers["anthropic-version"] == "2023-06-01"
body = rec.body()
assert body["model"] == "claude-x" and body["max_tokens"] == 700
assert body["system"] == "sys"
assert body["messages"] == [{"role": "user", "content": "usr"}]
fmt = body["output_config"]["format"]
assert fmt["type"] == "json_schema" and fmt["schema"]["additionalProperties"] is False
# forced tool use is rejected by current Claude models, so it must never be sent
assert "tool_choice" not in body and "tools" not in body
assert "temperature" not in body
assert res.parsed == Answer(text="hi", score=4)
assert (res.usage.input_tokens, res.usage.output_tokens) == (20, 8)
assert res.provider == "anthropic"
async def test_custom_api_version_and_temperature(sleeps):
rec = Recorder(ok_anthropic(GOOD))
c = client(rec, sleeps, api_version="2099-01-01")
await call(c, temperature=0.1)
assert rec.requests[0].headers["anthropic-version"] == "2099-01-01"
assert rec.body()["temperature"] == 0.1
async def test_reasoning_effort_is_ignored(sleeps):
rec = Recorder(ok_anthropic(GOOD))
await call(client(rec, sleeps), reasoning_effort="low")
assert "reasoning_effort" not in rec.body()
async def test_multiple_text_blocks_are_joined(sleeps):
r = httpx.Response(
200,
json={
"content": [{"type": "text", "text": '{"text": "a", '}, {"type": "text", "text": '"score": 1}'}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 1, "output_tokens": 1},
},
)
assert (await call(client(Recorder(r), sleeps))).parsed.score == 1
async def test_refusal_and_truncation(sleeps):
with pytest.raises(LlmRefusalError):
await call(client(Recorder(ok_anthropic("", stop="refusal")), sleeps))
with pytest.raises(LlmTruncatedError):
await call(client(Recorder(ok_anthropic('{"te', stop="max_tokens")), sleeps))
async def test_no_text_block(sleeps):
r = httpx.Response(200, json={"content": [], "stop_reason": "end_turn"})
with pytest.raises(LlmOutputError):
await call(client(Recorder(r), sleeps))
async def test_repair_loop(sleeps):
rec = Recorder(ok_anthropic('{"text": "x", "score": -1}'), ok_anthropic(GOOD))
res = await call(client(rec, sleeps, repair_attempts=1))
assert res.parsed.score == 4 and len(rec.requests) == 2
assert "rejected" in rec.body(1)["messages"][0]["content"]
async def test_overloaded_529_is_retried(sleeps):
rec = Recorder(httpx.Response(529, json={"error": {"message": "overloaded"}}), ok_anthropic(GOOD))
assert (await call(client(rec, sleeps))).parsed.score == 4
async def test_persistent_overload_raises_transient(sleeps):
rec = Recorder(*[httpx.Response(529, json={})] * 3)
with pytest.raises(LlmTransientError):
await call(client(rec, sleeps, max_retries=2))
async def test_auth_error(sleeps):
rec = Recorder(httpx.Response(401, json={"error": {"message": "invalid x-api-key"}}))
with pytest.raises(LlmAuthError) as e:
await call(client(rec, sleeps))
assert "sk-ant-secret" not in str(e.value)