115 lines
4.0 KiB
Python
115 lines
4.0 KiB
Python
import httpx
|
|
import pytest
|
|
|
|
from estimator.domain.errors import (
|
|
LlmAuthError,
|
|
LlmOutputError,
|
|
LlmRefusalError,
|
|
LlmTransientError,
|
|
LlmTruncatedError,
|
|
)
|
|
from estimator.infra.llm import AnthropicClient
|
|
from tests.unit.llm.conftest import Answer, Recorder, ok_anthropic
|
|
|
|
GOOD = '{"text": "hi", "score": 4}'
|
|
|
|
|
|
def client(rec, sleeps, **kw):
|
|
return AnthropicClient(
|
|
base_url="https://api.anthropic.test",
|
|
api_key="sk-ant-secret",
|
|
transport=rec.transport,
|
|
sleep=sleeps,
|
|
rng=lambda: 0.5,
|
|
**kw,
|
|
)
|
|
|
|
|
|
async def call(c, **kw):
|
|
return await c.structured(system="sys", user="usr", schema=Answer, model="claude-x", **kw)
|
|
|
|
|
|
async def test_request_shape_and_parse(sleeps):
|
|
rec = Recorder(ok_anthropic(GOOD, usage=(20, 8)))
|
|
res = await call(client(rec, sleeps), max_tokens=700)
|
|
req = rec.requests[0]
|
|
assert str(req.url) == "https://api.anthropic.test/v1/messages"
|
|
assert req.headers["x-api-key"] == "sk-ant-secret"
|
|
assert req.headers["anthropic-version"] == "2023-06-01"
|
|
body = rec.body()
|
|
assert body["model"] == "claude-x" and body["max_tokens"] == 700
|
|
assert body["system"] == "sys"
|
|
assert body["messages"] == [{"role": "user", "content": "usr"}]
|
|
fmt = body["output_config"]["format"]
|
|
assert fmt["type"] == "json_schema" and fmt["schema"]["additionalProperties"] is False
|
|
# forced tool use is rejected by current Claude models, so it must never be sent
|
|
assert "tool_choice" not in body and "tools" not in body
|
|
assert "temperature" not in body
|
|
assert res.parsed == Answer(text="hi", score=4)
|
|
assert (res.usage.input_tokens, res.usage.output_tokens) == (20, 8)
|
|
assert res.provider == "anthropic"
|
|
|
|
|
|
async def test_custom_api_version_and_temperature(sleeps):
|
|
rec = Recorder(ok_anthropic(GOOD))
|
|
c = client(rec, sleeps, api_version="2099-01-01")
|
|
await call(c, temperature=0.1)
|
|
assert rec.requests[0].headers["anthropic-version"] == "2099-01-01"
|
|
assert rec.body()["temperature"] == 0.1
|
|
|
|
|
|
async def test_reasoning_effort_is_ignored(sleeps):
|
|
rec = Recorder(ok_anthropic(GOOD))
|
|
await call(client(rec, sleeps), reasoning_effort="low")
|
|
assert "reasoning_effort" not in rec.body()
|
|
|
|
|
|
async def test_multiple_text_blocks_are_joined(sleeps):
|
|
r = httpx.Response(
|
|
200,
|
|
json={
|
|
"content": [{"type": "text", "text": '{"text": "a", '}, {"type": "text", "text": '"score": 1}'}],
|
|
"stop_reason": "end_turn",
|
|
"usage": {"input_tokens": 1, "output_tokens": 1},
|
|
},
|
|
)
|
|
assert (await call(client(Recorder(r), sleeps))).parsed.score == 1
|
|
|
|
|
|
async def test_refusal_and_truncation(sleeps):
|
|
with pytest.raises(LlmRefusalError):
|
|
await call(client(Recorder(ok_anthropic("", stop="refusal")), sleeps))
|
|
with pytest.raises(LlmTruncatedError):
|
|
await call(client(Recorder(ok_anthropic('{"te', stop="max_tokens")), sleeps))
|
|
|
|
|
|
async def test_no_text_block(sleeps):
|
|
r = httpx.Response(200, json={"content": [], "stop_reason": "end_turn"})
|
|
with pytest.raises(LlmOutputError):
|
|
await call(client(Recorder(r), sleeps))
|
|
|
|
|
|
async def test_repair_loop(sleeps):
|
|
rec = Recorder(ok_anthropic('{"text": "x", "score": -1}'), ok_anthropic(GOOD))
|
|
res = await call(client(rec, sleeps, repair_attempts=1))
|
|
assert res.parsed.score == 4 and len(rec.requests) == 2
|
|
assert "rejected" in rec.body(1)["messages"][0]["content"]
|
|
|
|
|
|
async def test_overloaded_529_is_retried(sleeps):
|
|
rec = Recorder(httpx.Response(529, json={"error": {"message": "overloaded"}}), ok_anthropic(GOOD))
|
|
assert (await call(client(rec, sleeps))).parsed.score == 4
|
|
|
|
|
|
async def test_persistent_overload_raises_transient(sleeps):
|
|
rec = Recorder(*[httpx.Response(529, json={})] * 3)
|
|
with pytest.raises(LlmTransientError):
|
|
await call(client(rec, sleeps, max_retries=2))
|
|
|
|
|
|
async def test_auth_error(sleeps):
|
|
rec = Recorder(httpx.Response(401, json={"error": {"message": "invalid x-api-key"}}))
|
|
with pytest.raises(LlmAuthError) as e:
|
|
await call(client(rec, sleeps))
|
|
assert "sk-ant-secret" not in str(e.value)
|