Files
estimator/tests/integration/test_pipeline_over_http.py
T

171 lines
6.4 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Whole stack with only the network faked: Settings -> router -> real OpenAI and Anthropic
clients -> pipeline. Two providers, per-step routing, and a fallback after an outage."""
import json
from pathlib import Path
import httpx
import pytest
from estimator.agent import EstimationPipeline, PipelineInput, PromptStore
from estimator.config.settings import Settings
from estimator.infra.llm import build_gateway
from estimator.pricing import price
from tests.unit.agent.conftest import (
DOC_WBS,
clarify_out,
critic_out,
decompose_out,
estimate_out,
missing,
)
ROOT = Path(__file__).resolve().parents[2]
CANNED = {
"ClarifyOutput": clarify_out(),
"DecomposeOutput": decompose_out(),
"EstimateOutput": estimate_out(),
"CriticOutput": critic_out(missing=[missing("Документация")]),
}
class FakeProviders:
"""Answers like OpenAI and Anthropic, keyed by the schema the request asks for."""
def __init__(self, openai_down=False):
self.openai_down = openai_down
self.seen: list[tuple[str, str, str]] = [] # (provider, schema, model)
def __call__(self, request: httpx.Request) -> httpx.Response:
body = json.loads(request.content)
if request.url.host == "api.openai.com":
schema = body["response_format"]["json_schema"]["name"]
self.seen.append(("openai", schema, body["model"]))
if self.openai_down:
return httpx.Response(503, json={"error": {"message": "down"}})
text = CANNED[schema].model_dump_json()
return httpx.Response(
200,
json={
"choices": [{"message": {"content": text}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 1000, "completion_tokens": 200},
},
)
# Anthropic requests do not name the schema: recognise it by its content
wire = json.dumps(body["output_config"]["format"]["schema"])
if '"missing_tasks"' in wire:
schema = "CriticOutput"
elif '"task_id"' in wire:
schema = "EstimateOutput"
elif '"summary"' in wire:
schema = "ClarifyOutput"
else:
schema = "DecomposeOutput"
self.seen.append(("anthropic", schema, body["model"]))
return httpx.Response(
200,
json={
"content": [{"type": "text", "text": CANNED[schema].model_dump_json()}],
"stop_reason": "end_turn",
"usage": {"input_tokens": 900, "output_tokens": 150},
},
)
def settings(monkeypatch, **env):
base = {
"EST_DATABASE__DSN": "sqlite:///:memory:",
"EST_LLM__ANTHROPIC__API_KEY": "sk-ant",
"EST_LLM__OPENAI__API_KEY": "sk-oai",
# estimate and critic run on OpenAI, critic falls back to Anthropic
"EST_LLM__STEPS__ESTIMATE__PROVIDER": "openai",
"EST_LLM__STEPS__ESTIMATE__MODEL": "gpt-test",
"EST_LLM__STEPS__CRITIC__PROVIDER": "openai",
"EST_LLM__STEPS__CRITIC__MODEL": "gpt-test",
"EST_LLM__STEPS__CRITIC__FALLBACK__PROVIDER": "anthropic",
"EST_LLM__STEPS__CRITIC__FALLBACK__MODEL": "claude-fallback",
}
base.update(env)
for k, v in base.items():
monkeypatch.setenv(k, v)
return Settings()
async def run(monkeypatch, providers, **env):
cfg = settings(monkeypatch, **env)
gw = build_gateway(cfg, transport=httpx.MockTransport(providers))
async def instant(_): # no real waiting between retries
return None
for c in gw._clients.values():
c._sleep = instant
try:
pipe = EstimationPipeline(gw, cfg.agent, PromptStore(ROOT / "prompts"))
return await pipe.run(PipelineInput(requirements="Бот Telegram+MAX+VK с ИИ и amoCRM"))
finally:
await gw.aclose()
async def test_two_providers_in_one_run(monkeypatch):
prov = FakeProviders()
res = await run(monkeypatch, prov)
assert res.status == "done"
assert [(p, s) for p, s, _ in prov.seen] == [
("anthropic", "ClarifyOutput"),
("anthropic", "DecomposeOutput"),
("openai", "EstimateOutput"),
("openai", "CriticOutput"),
]
assert [s.provider for s in res.steps] == ["anthropic", "anthropic", "openai", "openai"]
assert res.steps[2].model == "gpt-test" and not any(s.via_fallback for s in res.steps)
assert len(res.tasks) == len(DOC_WBS) + 1 # critic added documentation
assert res.total_tokens == (900 + 900 + 1000 + 1000, 150 + 150 + 200 + 200)
p = price(res.tasks, {"analyst": 1500, "backend": 1800, "qa": 1200, "devops": 1800})
assert p.hours > 144.7 # the doc example plus the critic's task
async def test_openai_outage_falls_back_for_the_critic_only(monkeypatch):
prov = FakeProviders(openai_down=True)
# estimate stays on Anthropic so only the critic depends on the broken provider
res = await run(
monkeypatch,
prov,
EST_LLM__STEPS__ESTIMATE__PROVIDER="anthropic",
EST_LLM__STEPS__ESTIMATE__MODEL="claude-x",
)
critic_log = res.steps[-1]
assert critic_log.step == "critic" and critic_log.via_fallback
assert critic_log.provider == "anthropic" and critic_log.model == "claude-fallback"
assert any(p == "openai" for p, _, _ in prov.seen) # it did try OpenAI first
assert res.status == "done"
async def test_strict_schema_is_what_goes_over_the_wire(monkeypatch):
seen = []
def handler(request: httpx.Request) -> httpx.Response:
seen.append(json.loads(request.content))
return FakeProviders()(request)
cfg = settings(monkeypatch)
gw = build_gateway(cfg, transport=httpx.MockTransport(handler))
try:
await EstimationPipeline(gw, cfg.agent, PromptStore(ROOT / "prompts")).run(
PipelineInput(requirements="бот")
)
finally:
await gw.aclose()
for body in seen:
schema = body.get("response_format", {}).get("json_schema", {}).get("schema") or body["output_config"]["format"]["schema"]
assert schema["additionalProperties"] is False and "$defs" not in schema
assert pytest.approx(len(seen)) == 4
async def test_outage_without_fallback_is_not_hidden(monkeypatch):
"""Estimate has no fallback: an OpenAI outage must surface, not return a partial estimate."""
from estimator.domain.errors import LlmTransientError
with pytest.raises(LlmTransientError):
await run(monkeypatch, FakeProviders(openai_down=True))