171 lines
6.4 KiB
Python
171 lines
6.4 KiB
Python
"""Whole stack with only the network faked: Settings -> router -> real OpenAI and Anthropic
|
||
clients -> pipeline. Two providers, per-step routing, and a fallback after an outage."""
|
||
import json
|
||
from pathlib import Path
|
||
|
||
import httpx
|
||
import pytest
|
||
|
||
from estimator.agent import EstimationPipeline, PipelineInput, PromptStore
|
||
from estimator.config.settings import Settings
|
||
from estimator.infra.llm import build_gateway
|
||
from estimator.pricing import price
|
||
from tests.unit.agent.conftest import (
|
||
DOC_WBS,
|
||
clarify_out,
|
||
critic_out,
|
||
decompose_out,
|
||
estimate_out,
|
||
missing,
|
||
)
|
||
|
||
ROOT = Path(__file__).resolve().parents[2]
|
||
|
||
CANNED = {
|
||
"ClarifyOutput": clarify_out(),
|
||
"DecomposeOutput": decompose_out(),
|
||
"EstimateOutput": estimate_out(),
|
||
"CriticOutput": critic_out(missing=[missing("Документация")]),
|
||
}
|
||
|
||
|
||
class FakeProviders:
|
||
"""Answers like OpenAI and Anthropic, keyed by the schema the request asks for."""
|
||
|
||
def __init__(self, openai_down=False):
|
||
self.openai_down = openai_down
|
||
self.seen: list[tuple[str, str, str]] = [] # (provider, schema, model)
|
||
|
||
def __call__(self, request: httpx.Request) -> httpx.Response:
|
||
body = json.loads(request.content)
|
||
if request.url.host == "api.openai.com":
|
||
schema = body["response_format"]["json_schema"]["name"]
|
||
self.seen.append(("openai", schema, body["model"]))
|
||
if self.openai_down:
|
||
return httpx.Response(503, json={"error": {"message": "down"}})
|
||
text = CANNED[schema].model_dump_json()
|
||
return httpx.Response(
|
||
200,
|
||
json={
|
||
"choices": [{"message": {"content": text}, "finish_reason": "stop"}],
|
||
"usage": {"prompt_tokens": 1000, "completion_tokens": 200},
|
||
},
|
||
)
|
||
# Anthropic requests do not name the schema: recognise it by its content
|
||
wire = json.dumps(body["output_config"]["format"]["schema"])
|
||
if '"missing_tasks"' in wire:
|
||
schema = "CriticOutput"
|
||
elif '"task_id"' in wire:
|
||
schema = "EstimateOutput"
|
||
elif '"summary"' in wire:
|
||
schema = "ClarifyOutput"
|
||
else:
|
||
schema = "DecomposeOutput"
|
||
self.seen.append(("anthropic", schema, body["model"]))
|
||
return httpx.Response(
|
||
200,
|
||
json={
|
||
"content": [{"type": "text", "text": CANNED[schema].model_dump_json()}],
|
||
"stop_reason": "end_turn",
|
||
"usage": {"input_tokens": 900, "output_tokens": 150},
|
||
},
|
||
)
|
||
|
||
|
||
def settings(monkeypatch, **env):
|
||
base = {
|
||
"EST_DATABASE__DSN": "sqlite:///:memory:",
|
||
"EST_LLM__ANTHROPIC__API_KEY": "sk-ant",
|
||
"EST_LLM__OPENAI__API_KEY": "sk-oai",
|
||
# estimate and critic run on OpenAI, critic falls back to Anthropic
|
||
"EST_LLM__STEPS__ESTIMATE__PROVIDER": "openai",
|
||
"EST_LLM__STEPS__ESTIMATE__MODEL": "gpt-test",
|
||
"EST_LLM__STEPS__CRITIC__PROVIDER": "openai",
|
||
"EST_LLM__STEPS__CRITIC__MODEL": "gpt-test",
|
||
"EST_LLM__STEPS__CRITIC__FALLBACK__PROVIDER": "anthropic",
|
||
"EST_LLM__STEPS__CRITIC__FALLBACK__MODEL": "claude-fallback",
|
||
}
|
||
base.update(env)
|
||
for k, v in base.items():
|
||
monkeypatch.setenv(k, v)
|
||
return Settings()
|
||
|
||
|
||
async def run(monkeypatch, providers, **env):
|
||
cfg = settings(monkeypatch, **env)
|
||
gw = build_gateway(cfg, transport=httpx.MockTransport(providers))
|
||
|
||
async def instant(_): # no real waiting between retries
|
||
return None
|
||
|
||
for c in gw._clients.values():
|
||
c._sleep = instant
|
||
try:
|
||
pipe = EstimationPipeline(gw, cfg.agent, PromptStore(ROOT / "prompts"))
|
||
return await pipe.run(PipelineInput(requirements="Бот Telegram+MAX+VK с ИИ и amoCRM"))
|
||
finally:
|
||
await gw.aclose()
|
||
|
||
|
||
async def test_two_providers_in_one_run(monkeypatch):
|
||
prov = FakeProviders()
|
||
res = await run(monkeypatch, prov)
|
||
assert res.status == "done"
|
||
assert [(p, s) for p, s, _ in prov.seen] == [
|
||
("anthropic", "ClarifyOutput"),
|
||
("anthropic", "DecomposeOutput"),
|
||
("openai", "EstimateOutput"),
|
||
("openai", "CriticOutput"),
|
||
]
|
||
assert [s.provider for s in res.steps] == ["anthropic", "anthropic", "openai", "openai"]
|
||
assert res.steps[2].model == "gpt-test" and not any(s.via_fallback for s in res.steps)
|
||
assert len(res.tasks) == len(DOC_WBS) + 1 # critic added documentation
|
||
assert res.total_tokens == (900 + 900 + 1000 + 1000, 150 + 150 + 200 + 200)
|
||
p = price(res.tasks, {"analyst": 1500, "backend": 1800, "qa": 1200, "devops": 1800})
|
||
assert p.hours > 144.7 # the doc example plus the critic's task
|
||
|
||
|
||
async def test_openai_outage_falls_back_for_the_critic_only(monkeypatch):
|
||
prov = FakeProviders(openai_down=True)
|
||
# estimate stays on Anthropic so only the critic depends on the broken provider
|
||
res = await run(
|
||
monkeypatch,
|
||
prov,
|
||
EST_LLM__STEPS__ESTIMATE__PROVIDER="anthropic",
|
||
EST_LLM__STEPS__ESTIMATE__MODEL="claude-x",
|
||
)
|
||
critic_log = res.steps[-1]
|
||
assert critic_log.step == "critic" and critic_log.via_fallback
|
||
assert critic_log.provider == "anthropic" and critic_log.model == "claude-fallback"
|
||
assert any(p == "openai" for p, _, _ in prov.seen) # it did try OpenAI first
|
||
assert res.status == "done"
|
||
|
||
|
||
async def test_strict_schema_is_what_goes_over_the_wire(monkeypatch):
|
||
seen = []
|
||
|
||
def handler(request: httpx.Request) -> httpx.Response:
|
||
seen.append(json.loads(request.content))
|
||
return FakeProviders()(request)
|
||
|
||
cfg = settings(monkeypatch)
|
||
gw = build_gateway(cfg, transport=httpx.MockTransport(handler))
|
||
try:
|
||
await EstimationPipeline(gw, cfg.agent, PromptStore(ROOT / "prompts")).run(
|
||
PipelineInput(requirements="бот")
|
||
)
|
||
finally:
|
||
await gw.aclose()
|
||
for body in seen:
|
||
schema = body.get("response_format", {}).get("json_schema", {}).get("schema") or body["output_config"]["format"]["schema"]
|
||
assert schema["additionalProperties"] is False and "$defs" not in schema
|
||
assert pytest.approx(len(seen)) == 4
|
||
|
||
|
||
async def test_outage_without_fallback_is_not_hidden(monkeypatch):
|
||
"""Estimate has no fallback: an OpenAI outage must surface, not return a partial estimate."""
|
||
from estimator.domain.errors import LlmTransientError
|
||
|
||
with pytest.raises(LlmTransientError):
|
||
await run(monkeypatch, FakeProviders(openai_down=True))
|