Init Project Flexly Estimator
This commit is contained in:
@@ -0,0 +1,114 @@
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from estimator.domain.errors import (
|
||||
LlmAuthError,
|
||||
LlmOutputError,
|
||||
LlmRefusalError,
|
||||
LlmTransientError,
|
||||
LlmTruncatedError,
|
||||
)
|
||||
from estimator.infra.llm import AnthropicClient
|
||||
from tests.unit.llm.conftest import Answer, Recorder, ok_anthropic
|
||||
|
||||
GOOD = '{"text": "hi", "score": 4}'
|
||||
|
||||
|
||||
def client(rec, sleeps, **kw):
|
||||
return AnthropicClient(
|
||||
base_url="https://api.anthropic.test",
|
||||
api_key="sk-ant-secret",
|
||||
transport=rec.transport,
|
||||
sleep=sleeps,
|
||||
rng=lambda: 0.5,
|
||||
**kw,
|
||||
)
|
||||
|
||||
|
||||
async def call(c, **kw):
|
||||
return await c.structured(system="sys", user="usr", schema=Answer, model="claude-x", **kw)
|
||||
|
||||
|
||||
async def test_request_shape_and_parse(sleeps):
|
||||
rec = Recorder(ok_anthropic(GOOD, usage=(20, 8)))
|
||||
res = await call(client(rec, sleeps), max_tokens=700)
|
||||
req = rec.requests[0]
|
||||
assert str(req.url) == "https://api.anthropic.test/v1/messages"
|
||||
assert req.headers["x-api-key"] == "sk-ant-secret"
|
||||
assert req.headers["anthropic-version"] == "2023-06-01"
|
||||
body = rec.body()
|
||||
assert body["model"] == "claude-x" and body["max_tokens"] == 700
|
||||
assert body["system"] == "sys"
|
||||
assert body["messages"] == [{"role": "user", "content": "usr"}]
|
||||
fmt = body["output_config"]["format"]
|
||||
assert fmt["type"] == "json_schema" and fmt["schema"]["additionalProperties"] is False
|
||||
# forced tool use is rejected by current Claude models, so it must never be sent
|
||||
assert "tool_choice" not in body and "tools" not in body
|
||||
assert "temperature" not in body
|
||||
assert res.parsed == Answer(text="hi", score=4)
|
||||
assert (res.usage.input_tokens, res.usage.output_tokens) == (20, 8)
|
||||
assert res.provider == "anthropic"
|
||||
|
||||
|
||||
async def test_custom_api_version_and_temperature(sleeps):
|
||||
rec = Recorder(ok_anthropic(GOOD))
|
||||
c = client(rec, sleeps, api_version="2099-01-01")
|
||||
await call(c, temperature=0.1)
|
||||
assert rec.requests[0].headers["anthropic-version"] == "2099-01-01"
|
||||
assert rec.body()["temperature"] == 0.1
|
||||
|
||||
|
||||
async def test_reasoning_effort_is_ignored(sleeps):
|
||||
rec = Recorder(ok_anthropic(GOOD))
|
||||
await call(client(rec, sleeps), reasoning_effort="low")
|
||||
assert "reasoning_effort" not in rec.body()
|
||||
|
||||
|
||||
async def test_multiple_text_blocks_are_joined(sleeps):
|
||||
r = httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"content": [{"type": "text", "text": '{"text": "a", '}, {"type": "text", "text": '"score": 1}'}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1},
|
||||
},
|
||||
)
|
||||
assert (await call(client(Recorder(r), sleeps))).parsed.score == 1
|
||||
|
||||
|
||||
async def test_refusal_and_truncation(sleeps):
|
||||
with pytest.raises(LlmRefusalError):
|
||||
await call(client(Recorder(ok_anthropic("", stop="refusal")), sleeps))
|
||||
with pytest.raises(LlmTruncatedError):
|
||||
await call(client(Recorder(ok_anthropic('{"te', stop="max_tokens")), sleeps))
|
||||
|
||||
|
||||
async def test_no_text_block(sleeps):
|
||||
r = httpx.Response(200, json={"content": [], "stop_reason": "end_turn"})
|
||||
with pytest.raises(LlmOutputError):
|
||||
await call(client(Recorder(r), sleeps))
|
||||
|
||||
|
||||
async def test_repair_loop(sleeps):
|
||||
rec = Recorder(ok_anthropic('{"text": "x", "score": -1}'), ok_anthropic(GOOD))
|
||||
res = await call(client(rec, sleeps, repair_attempts=1))
|
||||
assert res.parsed.score == 4 and len(rec.requests) == 2
|
||||
assert "rejected" in rec.body(1)["messages"][0]["content"]
|
||||
|
||||
|
||||
async def test_overloaded_529_is_retried(sleeps):
|
||||
rec = Recorder(httpx.Response(529, json={"error": {"message": "overloaded"}}), ok_anthropic(GOOD))
|
||||
assert (await call(client(rec, sleeps))).parsed.score == 4
|
||||
|
||||
|
||||
async def test_persistent_overload_raises_transient(sleeps):
|
||||
rec = Recorder(*[httpx.Response(529, json={})] * 3)
|
||||
with pytest.raises(LlmTransientError):
|
||||
await call(client(rec, sleeps, max_retries=2))
|
||||
|
||||
|
||||
async def test_auth_error(sleeps):
|
||||
rec = Recorder(httpx.Response(401, json={"error": {"message": "invalid x-api-key"}}))
|
||||
with pytest.raises(LlmAuthError) as e:
|
||||
await call(client(rec, sleeps))
|
||||
assert "sk-ant-secret" not in str(e.value)
|
||||
Reference in New Issue
Block a user