"""Whole stack with only the network faked: Settings -> router -> real OpenAI and Anthropic clients -> pipeline. Two providers, per-step routing, and a fallback after an outage.""" import json from pathlib import Path import httpx import pytest from estimator.agent import EstimationPipeline, PipelineInput, PromptStore from estimator.config.settings import Settings from estimator.infra.llm import build_gateway from estimator.pricing import price from tests.unit.agent.conftest import ( DOC_WBS, clarify_out, critic_out, decompose_out, estimate_out, missing, ) ROOT = Path(__file__).resolve().parents[2] CANNED = { "ClarifyOutput": clarify_out(), "DecomposeOutput": decompose_out(), "EstimateOutput": estimate_out(), "CriticOutput": critic_out(missing=[missing("Документация")]), } class FakeProviders: """Answers like OpenAI and Anthropic, keyed by the schema the request asks for.""" def __init__(self, openai_down=False): self.openai_down = openai_down self.seen: list[tuple[str, str, str]] = [] # (provider, schema, model) def __call__(self, request: httpx.Request) -> httpx.Response: body = json.loads(request.content) if request.url.host == "api.openai.com": schema = body["response_format"]["json_schema"]["name"] self.seen.append(("openai", schema, body["model"])) if self.openai_down: return httpx.Response(503, json={"error": {"message": "down"}}) text = CANNED[schema].model_dump_json() return httpx.Response( 200, json={ "choices": [{"message": {"content": text}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 1000, "completion_tokens": 200}, }, ) # Anthropic requests do not name the schema: recognise it by its content wire = json.dumps(body["output_config"]["format"]["schema"]) if '"missing_tasks"' in wire: schema = "CriticOutput" elif '"task_id"' in wire: schema = "EstimateOutput" elif '"summary"' in wire: schema = "ClarifyOutput" else: schema = "DecomposeOutput" self.seen.append(("anthropic", schema, body["model"])) return httpx.Response( 200, json={ "content": [{"type": "text", "text": CANNED[schema].model_dump_json()}], "stop_reason": "end_turn", "usage": {"input_tokens": 900, "output_tokens": 150}, }, ) def settings(monkeypatch, **env): base = { "EST_DATABASE__DSN": "sqlite:///:memory:", "EST_LLM__ANTHROPIC__API_KEY": "sk-ant", "EST_LLM__OPENAI__API_KEY": "sk-oai", # estimate and critic run on OpenAI, critic falls back to Anthropic "EST_LLM__STEPS__ESTIMATE__PROVIDER": "openai", "EST_LLM__STEPS__ESTIMATE__MODEL": "gpt-test", "EST_LLM__STEPS__CRITIC__PROVIDER": "openai", "EST_LLM__STEPS__CRITIC__MODEL": "gpt-test", "EST_LLM__STEPS__CRITIC__FALLBACK__PROVIDER": "anthropic", "EST_LLM__STEPS__CRITIC__FALLBACK__MODEL": "claude-fallback", } base.update(env) for k, v in base.items(): monkeypatch.setenv(k, v) return Settings() async def run(monkeypatch, providers, **env): cfg = settings(monkeypatch, **env) gw = build_gateway(cfg, transport=httpx.MockTransport(providers)) async def instant(_): # no real waiting between retries return None for c in gw._clients.values(): c._sleep = instant try: pipe = EstimationPipeline(gw, cfg.agent, PromptStore(ROOT / "prompts")) return await pipe.run(PipelineInput(requirements="Бот Telegram+MAX+VK с ИИ и amoCRM")) finally: await gw.aclose() async def test_two_providers_in_one_run(monkeypatch): prov = FakeProviders() res = await run(monkeypatch, prov) assert res.status == "done" assert [(p, s) for p, s, _ in prov.seen] == [ ("anthropic", "ClarifyOutput"), ("anthropic", "DecomposeOutput"), ("openai", "EstimateOutput"), ("openai", "CriticOutput"), ] assert [s.provider for s in res.steps] == ["anthropic", "anthropic", "openai", "openai"] assert res.steps[2].model == "gpt-test" and not any(s.via_fallback for s in res.steps) assert len(res.tasks) == len(DOC_WBS) + 1 # critic added documentation assert res.total_tokens == (900 + 900 + 1000 + 1000, 150 + 150 + 200 + 200) p = price(res.tasks, {"analyst": 1500, "backend": 1800, "qa": 1200, "devops": 1800}) assert p.hours > 144.7 # the doc example plus the critic's task async def test_openai_outage_falls_back_for_the_critic_only(monkeypatch): prov = FakeProviders(openai_down=True) # estimate stays on Anthropic so only the critic depends on the broken provider res = await run( monkeypatch, prov, EST_LLM__STEPS__ESTIMATE__PROVIDER="anthropic", EST_LLM__STEPS__ESTIMATE__MODEL="claude-x", ) critic_log = res.steps[-1] assert critic_log.step == "critic" and critic_log.via_fallback assert critic_log.provider == "anthropic" and critic_log.model == "claude-fallback" assert any(p == "openai" for p, _, _ in prov.seen) # it did try OpenAI first assert res.status == "done" async def test_strict_schema_is_what_goes_over_the_wire(monkeypatch): seen = [] def handler(request: httpx.Request) -> httpx.Response: seen.append(json.loads(request.content)) return FakeProviders()(request) cfg = settings(monkeypatch) gw = build_gateway(cfg, transport=httpx.MockTransport(handler)) try: await EstimationPipeline(gw, cfg.agent, PromptStore(ROOT / "prompts")).run( PipelineInput(requirements="бот") ) finally: await gw.aclose() for body in seen: schema = body.get("response_format", {}).get("json_schema", {}).get("schema") or body["output_config"]["format"]["schema"] assert schema["additionalProperties"] is False and "$defs" not in schema assert pytest.approx(len(seen)) == 4 async def test_outage_without_fallback_is_not_hidden(monkeypatch): """Estimate has no fallback: an OpenAI outage must surface, not return a partial estimate.""" from estimator.domain.errors import LlmTransientError with pytest.raises(LlmTransientError): await run(monkeypatch, FakeProviders(openai_down=True))