This commit is contained in:
2026-03-27 15:51:10 +03:00
parent 15586f9a8c
commit 51378c5d66
1234 changed files with 95644 additions and 543076 deletions
@@ -16,10 +16,12 @@ def repo_context() -> RepoContext:
RagLayer.CODE_DEPENDENCY_GRAPH,
RagLayer.CODE_SEMANTIC_ROLES,
RagLayer.CODE_SOURCE_CHUNKS,
RagLayer.DOCS_MODULE_CATALOG,
RagLayer.DOCS_DOC_CHUNKS,
RagLayer.DOCS_DOCUMENT_CATALOG,
RagLayer.DOCS_FACT_INDEX,
RagLayer.DOCS_SECTION_INDEX,
RagLayer.DOCS_POLICY_INDEX,
RagLayer.DOCS_ENTITY_CATALOG,
RagLayer.DOCS_WORKFLOW_INDEX,
RagLayer.DOCS_RELATION_GRAPH,
],
)
@@ -2,40 +2,78 @@ from app.modules.rag.contracts.enums import RagLayer
from app.modules.rag.indexing.docs.pipeline import DocsIndexingPipeline
def test_docs_pipeline_builds_catalog_facts_sections_and_policy() -> None:
def test_docs_pipeline_builds_new_d0_to_d5_layers() -> None:
pipeline = DocsIndexingPipeline()
content = """---
id: api.billing.create_invoice
type: policy
domain: billing
type: api_method
name: create_invoice
title: Create Invoice API
module: billing
layer: application
status: draft
updated_at: 2026-03-23
tags: [billing, api]
entities: [Invoice]
parent: billing_api
children: []
links:
calls_api:
- api.billing.validate_invoice
tags: [billing]
status: active
- type: related_api
target: api.billing.validate_invoice
---
# Create Invoice
## Spec Summary
# Summary
Creates an invoice in billing.
## Request Contract
# Details
## Описание
Создает счет на оплату.
## Сценарий
**Название:**
Create invoice
**Предусловия:**
- billing service is available
**Триггер:**
- client sends create invoice request
**Основной сценарий:**
1. Validate payload.
2. Create invoice.
**Альтернативный сценарий:**
1. Reject invalid payload.
**Обработка ошибок:**
1. Return validation error.
**Постусловие:**
- Invoice is created.
## Контракт
### Входные параметры
| field | type | required | validation |
| --- | --- | --- | --- |
| amount | decimal | yes | > 0 |
## Error Matrix
### Выходные параметры
| field | type | required |
| --- | --- | --- |
| invoice_id | string | yes |
## Ошибки
| status | error | client action |
| --- | --- | --- |
| 400 | invalid_amount | fix request |
## Rules
- metric: billing.invoice.created
- rule: amount must be positive
"""
docs = pipeline.index_file(
repo_id="acme/proj",
@@ -45,19 +83,31 @@ Creates an invoice in billing.
)
layers = {doc.layer for doc in docs}
assert RagLayer.DOCS_MODULE_CATALOG in layers
assert RagLayer.DOCS_DOC_CHUNKS in layers
assert RagLayer.DOCS_DOCUMENT_CATALOG in layers
assert RagLayer.DOCS_FACT_INDEX in layers
assert RagLayer.DOCS_SECTION_INDEX in layers
assert RagLayer.DOCS_POLICY_INDEX in layers
assert RagLayer.DOCS_ENTITY_CATALOG in layers
assert RagLayer.DOCS_WORKFLOW_INDEX in layers
assert RagLayer.DOCS_RELATION_GRAPH in layers
module_doc = next(doc for doc in docs if doc.layer == RagLayer.DOCS_MODULE_CATALOG)
assert module_doc.metadata["module_id"] == "api.billing.create_invoice"
assert module_doc.metadata["type"] == "policy"
catalog_doc = next(doc for doc in docs if doc.layer == RagLayer.DOCS_DOCUMENT_CATALOG)
assert catalog_doc.metadata["document_id"] == "api.billing.create_invoice"
assert catalog_doc.metadata["module"] == "billing"
fact_texts = [doc.text for doc in docs if doc.layer == RagLayer.DOCS_FACT_INDEX]
assert any("calls_api" in text for text in fact_texts)
assert any("has_field" in text for text in fact_texts)
assert any("returns_error" in text for text in fact_texts)
assert any("has_field amount" in text for text in fact_texts)
assert any("field_required amount:yes" in text for text in fact_texts)
assert any("returns_error invalid_amount" in text for text in fact_texts)
section_doc = next(doc for doc in docs if doc.layer == RagLayer.DOCS_SECTION_INDEX)
assert section_doc.metadata["section_path"]
entity_doc = next(doc for doc in docs if doc.layer == RagLayer.DOCS_ENTITY_CATALOG)
assert entity_doc.metadata["entity_name"] == "Invoice"
workflow_doc = next(doc for doc in docs if doc.layer == RagLayer.DOCS_WORKFLOW_INDEX)
assert workflow_doc.metadata["workflow_name"] == "Create invoice"
relation_targets = [doc.metadata["target_id"] for doc in docs if doc.layer == RagLayer.DOCS_RELATION_GRAPH]
assert "billing_api" in relation_targets
assert "api.billing.validate_invoice" in relation_targets
chunk_doc = next(doc for doc in docs if doc.layer == RagLayer.DOCS_DOC_CHUNKS)
assert chunk_doc.metadata["section_path"]
@@ -0,0 +1,146 @@
from __future__ import annotations
from app.modules.agent.intent_router_v2 import IntentRouterV2
from app.modules.agent.llm.prompt_loader import PromptLoader
from app.modules.agent.runtime.docs_qa_pipeline import DocsQAPipelineRunner
from app.modules.agent.runtime.docs_qa_pipeline.openapi_postprocessor import OpenAPIPostprocessor
from app.modules.agent.runtime.docs_qa_pipeline.prompt_payload_builder import DocsPromptPayloadBuilder
from app.modules.agent.runtime.steps.generation import RuntimePromptSelector
from tests.docs_qa_eval.fixture_adapter import InMemoryDocsRetrievalAdapter
from tests.unit_tests.rag.intent_router_testkit import repo_context
class FakeLlm:
def __init__(self, response: str) -> None:
self.response = response
self.calls: list[tuple[str, str]] = []
def generate(self, prompt_name: str, user_input: str, *, log_context: str | None = None) -> str:
self.calls.append((prompt_name, user_input))
return self.response
def test_docs_prompt_registry_contains_new_prompts() -> None:
loader = PromptLoader()
for name in (
"docs_explain_answer",
"docs_general_answer",
"docs_openapi_answer",
"docs_openapi_fragment_answer",
):
assert loader.load(name)
def test_prompt_selector_uses_docs_prompts_only() -> None:
selector = RuntimePromptSelector()
assert selector.select(intent="DOCUMENTATION_EXPLAIN", sub_intent="COMPONENT_EXPLAIN", answer_mode="normal") == "docs_explain_answer"
assert selector.select(intent="GENERAL_QA", sub_intent="GENERIC_QA", answer_mode="degraded") == "docs_general_answer"
assert selector.select(intent="OPENAPI_GENERATION", sub_intent="OPENAPI_METHOD_GENERATE", answer_mode="normal") == "docs_openapi_answer"
assert selector.select(intent="OPENAPI_GENERATION", sub_intent="OPENAPI_FRAGMENT_GENERATE", answer_mode="normal") == "docs_openapi_fragment_answer"
def test_docs_prompt_payload_contains_required_contract() -> None:
builder = DocsPromptPayloadBuilder()
from app.modules.agent.runtime.docs_qa_pipeline.models import DocsEvidenceBundle, OpenAPIResult
payload = builder.build(
question="Объясни billing",
intent="DOCUMENTATION_EXPLAIN",
sub_intent="COMPONENT_EXPLAIN",
evidence_bundle=DocsEvidenceBundle(
intent="DOCUMENTATION_EXPLAIN",
sub_intent="COMPONENT_EXPLAIN",
documents=[{"title": "Billing"}],
facts=[{"content": "Handles payments"}],
relations=[{"title": "Billing -> Orders"}],
),
api_contract=OpenAPIResult(path="/orders", method="post"),
)
assert '"question": "Объясни billing"' in payload
assert '"intent": "DOCUMENTATION_EXPLAIN"' in payload
assert '"sub_intent": "COMPONENT_EXPLAIN"' in payload
assert '"documents"' in payload
assert '"facts"' in payload
assert '"relations"' in payload
assert '"api_contract"' in payload
def test_openapi_postprocessor_requires_paths_for_full_spec() -> None:
validator = OpenAPIPostprocessor()
assert validator.validate("paths:\n /orders:\n post:\n responses: {}", require_paths=True)[0] is True
assert validator.validate("type: object\nproperties:\n id: {}", require_paths=True)[0] is False
assert validator.validate("type: object\nproperties:\n id: {}", require_paths=False)[0] is True
def test_docs_pipeline_uses_llm_prompt_and_validates_openapi() -> None:
rows = [
{
"layer": "D2_FACT_INDEX",
"path": "docs/api/orders-create.md",
"title": "POST /orders",
"content": "Create order",
"metadata": {
"endpoint": "/orders",
"http_method": "post",
"request_schema": {"type": "object", "properties": {"customer_id": {}}},
"response_schema": {"type": "object", "properties": {"order_id": {}}},
},
}
]
llm = FakeLlm("not yaml")
runner = DocsQAPipelineRunner(
IntentRouterV2(),
InMemoryDocsRetrievalAdapter(rows),
repo_context=repo_context(),
llm=llm,
)
result = runner.run("Сгенерируй openapi spec для создания заказа", "docs-session")
assert llm.calls
assert llm.calls[0][0] == "docs_openapi_answer"
assert result.prompt_name == "docs_openapi_answer"
assert result.output_valid is False
assert "paths:" in result.answer
assert result.diagnostics.prompt_used == "docs_openapi_answer"
assert result.diagnostics.llm_mode == "yaml"
def test_pre_llm_mode_skips_llm_calls() -> None:
rows = [
{
"layer": "D2_FACT_INDEX",
"path": "docs/api/orders-create.md",
"title": "POST /orders",
"content": "Create order",
"metadata": {
"endpoint": "/orders",
"http_method": "post",
"request_schema": {"type": "object", "properties": {"customer_id": {}}},
},
}
]
llm = FakeLlm("unused")
runner = DocsQAPipelineRunner(
IntentRouterV2(),
InMemoryDocsRetrievalAdapter(rows),
repo_context=repo_context(),
llm=llm,
)
result = runner.run("Сгенерируй openapi для /orders", "docs-session", mode="pre_llm_only")
assert llm.calls == []
assert result.answer
assert result.mode == "pre_llm_only"
assert result.diagnostics.gate_decision == "partial"
assert result.answer_mode == "ready_partial"
assert "paths:" in result.answer
assert result.llm_request["prompt_name"] == "docs_openapi_answer"
assert "user_prompt" in result.llm_request
assert "system_prompt" in result.llm_request
assert result.diagnostics.prompt["prompt_name"] == "docs_openapi_answer"
@@ -0,0 +1,439 @@
from __future__ import annotations
from app.modules.agent.intent_router_v2 import IntentRouterV2
from app.modules.agent.runtime.docs_qa_pipeline import DocsQAPipelineRunner
from tests.docs_qa_eval.fixture_adapter import InMemoryDocsRetrievalAdapter
from tests.unit_tests.rag.intent_router_testkit import repo_context
def test_docs_pipeline_uses_synthesizer_for_explain() -> None:
rows = [
{
"layer": "D2_FACT_INDEX",
"path": "docs/components/billing.md",
"title": "Billing facts",
"content": "Billing обновляет статус заказа после оплаты.",
}
]
runner = DocsQAPipelineRunner(IntentRouterV2(), InMemoryDocsRetrievalAdapter(rows), repo_context=repo_context())
result = runner.run("Объясни модуль billing в документации", "docs-session")
assert result.router_result.intent == "DOCUMENTATION_EXPLAIN"
assert "Billing" in result.answer
assert result.openapi_result is None
assert result.diagnostics.facts_found == 1
def test_docs_pipeline_uses_openapi_branch() -> None:
rows = [
{
"layer": "D2_FACT_INDEX",
"path": "docs/api/orders-create.md",
"title": "POST /orders",
"content": "Create order",
"metadata": {
"endpoint": "/orders",
"http_method": "post",
"request_schema": {"type": "object", "properties": {"customer_id": {}}},
"response_schema": {"type": "object", "properties": {"order_id": {}}},
},
}
]
runner = DocsQAPipelineRunner(IntentRouterV2(), InMemoryDocsRetrievalAdapter(rows), repo_context=repo_context())
result = runner.run("Сгенерируй openapi spec для создания заказа", "docs-session")
assert result.router_result.intent == "OPENAPI_GENERATION"
assert result.openapi_result is not None
assert result.openapi_result.path == "/orders"
assert result.openapi_result.method == "post"
assert result.answer_mode != "degraded"
assert result.diagnostics.gate_decision != "reject"
assert "paths:" in result.answer
assert "/orders" in result.answer
def test_docs_pipeline_keeps_query_candidates_for_not_found_entity() -> None:
runner = DocsQAPipelineRunner(IntentRouterV2(), InMemoryDocsRetrievalAdapter([]), repo_context=repo_context())
result = runner.run("Что такое сущность PaymentTransaction?", "docs-session")
assert result.answer_mode == "degraded"
assert "PaymentTransaction" in result.diagnostics.query_entity_candidates
assert result.diagnostics.resolved_entity_candidates == []
assert result.diagnostics.degraded_reason == "not_found_exact_anchor"
def test_docs_pipeline_reports_canonical_doc_ids_and_layers() -> None:
rows = [
{
"layer": "D5_RELATION_GRAPH",
"path": "docs/api/health.md",
"title": "Health relations",
"content": "Health -> runtime health",
"metadata": {"doc_id": "api.health_endpoint", "target_doc_id": "domain.runtime_health", "endpoint": "/health"},
},
{
"layer": "D1_DOCUMENT_CATALOG",
"path": "docs/domain/runtime-health.md",
"title": "Runtime health",
"content": "Runtime health overview",
"metadata": {"doc_id": "domain.runtime_health"},
},
]
runner = DocsQAPipelineRunner(IntentRouterV2(), InMemoryDocsRetrievalAdapter(rows), repo_context=repo_context())
result = runner.run("Какие документы связаны с endpoint /health?", "docs-session")
assert result.diagnostics.doc_ids[:2] == ["api.health_endpoint", "domain.runtime_health"]
assert result.diagnostics.planned_layers == ["D5_RELATION_GRAPH", "D1_DOCUMENT_CATALOG", "D0_DOC_CHUNKS"]
assert result.diagnostics.executed_layers == ["D5_RELATION_GRAPH", "D1_DOCUMENT_CATALOG", "D0_DOC_CHUNKS"]
assert result.diagnostics.non_empty_layers == ["D5_RELATION_GRAPH"]
assert result.diagnostics.relation_hits_count == 1
assert result.diagnostics.relation_targets == ["domain.runtime_health"]
assert result.diagnostics.answer_mode == "answered"
assert result.diagnostics.layer_diagnostics["D5_RELATION_GRAPH"]["hits"] == 1
assert result.diagnostics.gate_decision == "allow"
def test_related_docs_uses_relation_graph_as_primary_layer() -> None:
runner = DocsQAPipelineRunner(IntentRouterV2(), InMemoryDocsRetrievalAdapter([]), repo_context=repo_context())
result = runner.run("Что еще посмотреть по теме billing?", "docs-session")
assert result.router_result.query_plan.sub_intent == "RELATED_DOCS_EXPLAIN"
assert result.diagnostics.planned_layers[0] == "D5_RELATION_GRAPH"
assert result.diagnostics.executed_layers[0] == "D5_RELATION_GRAPH"
def test_openapi_partial_contract_returns_partial_mode() -> None:
rows = [
{
"layer": "D2_FACT_INDEX",
"path": "docs/api/orders-create.md",
"title": "POST /orders",
"content": "Create order",
"metadata": {
"endpoint": "/orders",
"http_method": "post",
"request_schema": {"type": "object", "properties": {"customer_id": {}}},
"doc_id": "api.orders_create",
},
}
]
runner = DocsQAPipelineRunner(IntentRouterV2(), InMemoryDocsRetrievalAdapter(rows), repo_context=repo_context())
result = runner.run("Сгенерируй openapi для /orders", "docs-session")
assert result.answer_mode == "structured_spec_partial"
assert result.degraded_reason == "answered_with_gaps"
assert result.openapi_result is not None
assert result.openapi_result.path == "/orders"
assert result.diagnostics.gate_decision == "partial"
assert "paths:" in result.answer
assert "/orders" in result.answer
def test_pre_llm_mode_returns_diagnostic_only_without_answer_generation() -> None:
rows = [
{
"layer": "D2_FACT_INDEX",
"path": "docs/components/runtime-manager.md",
"title": "RuntimeManager",
"content": "Coordinates runtime orchestration.",
"metadata": {"doc_id": "architecture.runtime_manager", "component": "RuntimeManager"},
}
]
runner = DocsQAPipelineRunner(IntentRouterV2(), InMemoryDocsRetrievalAdapter(rows), repo_context=repo_context())
result = runner.run("Какую роль в системе играет RuntimeManager?", "docs-session", mode="pre_llm_only")
assert result.mode == "pre_llm_only"
assert result.answer == ""
assert result.diagnostics.gate_decision == "allow"
assert "RuntimeManager" in result.diagnostics.query_entity_candidates
assert "RuntimeManager" in result.diagnostics.resolved_entity_candidates
def test_pre_llm_mode_detects_path_anchor_candidates() -> None:
rows = [
{
"layer": "D2_FACT_INDEX",
"path": "docs/api/health.md",
"title": "GET /health",
"content": "Health endpoint",
"metadata": {"doc_id": "api.health_endpoint", "endpoint": "/health", "http_method": "get"},
}
]
runner = DocsQAPipelineRunner(IntentRouterV2(), InMemoryDocsRetrievalAdapter(rows), repo_context=repo_context())
result = runner.run("Объясни API метод /health", "docs-session", mode="pre_llm_only")
assert "/health" in result.diagnostics.query_anchor_candidates
assert "/health" in result.diagnostics.resolved_anchor_candidates
assert result.diagnostics.planned_layers == ["D2_FACT_INDEX", "D4_WORKFLOW_INDEX", "D1_DOCUMENT_CATALOG", "D0_DOC_CHUNKS"]
assert set(result.diagnostics.executed_layers) == {"D1_DOCUMENT_CATALOG", "D2_FACT_INDEX", "D4_WORKFLOW_INDEX", "D0_DOC_CHUNKS"}
def test_pre_llm_openapi_gate_reports_missing_signals() -> None:
rows = [
{
"layer": "D2_FACT_INDEX",
"path": "docs/api/orders.md",
"title": "POST /orders",
"content": "Create order",
"metadata": {"doc_id": "api.orders_create", "endpoint": "/orders", "http_method": "post"},
}
]
runner = DocsQAPipelineRunner(IntentRouterV2(), InMemoryDocsRetrievalAdapter(rows), repo_context=repo_context())
result = runner.run("Сгенерируй openapi для /orders", "docs-session", mode="pre_llm_only")
assert result.diagnostics.gate_decision == "partial"
assert result.diagnostics.gate_decision_reason == "answered_with_gaps"
assert result.answer_mode == "ready_partial"
assert result.answer
assert "paths:" in result.answer
assert "path_found" in result.diagnostics.gate_satisfied_requirements
assert "http_method_found" in result.diagnostics.gate_satisfied_requirements
assert "contract_fields_found" in result.diagnostics.gate_missing_requirements
assert result.diagnostics.evidence_summary["openapi_signals"]["path_found"] is True
assert result.diagnostics.openapi_evidence["path_found"] is True
assert result.diagnostics.layer_diagnostics["D2_FACT_INDEX"]["hits"] == 1
def test_related_docs_keeps_relation_hits_when_exact_anchor_is_missing() -> None:
rows = [
{
"layer": "D5_RELATION_GRAPH",
"path": "docs/relations/health-links.md",
"title": "Health relations",
"content": "Runtime health and service overview",
"metadata": {
"document_id": "api.health_endpoint",
"target_document_id": "domain.runtime_health",
},
}
]
runner = DocsQAPipelineRunner(IntentRouterV2(), InMemoryDocsRetrievalAdapter(rows), repo_context=repo_context())
result = runner.run("Какие документы связаны с endpoint /health?", "docs-session", mode="pre_llm_only")
assert result.router_result.query_plan.sub_intent == "RELATED_DOCS_EXPLAIN"
assert result.diagnostics.doc_ids[:2] == ["api.health_endpoint", "domain.runtime_health"]
assert result.diagnostics.relation_hits_count == 1
assert result.diagnostics.exact_anchor_match is False
assert result.diagnostics.gate_decision == "allow"
assert result.diagnostics.gate_decision_reason == "relation_evidence_available"
def test_related_docs_exact_path_outranks_neighboring_api_doc() -> None:
rows = [
{
"layer": "D5_RELATION_GRAPH",
"path": "docs/documentation/api/control-actions-endpoint.md",
"title": "HTTP API /actions/{action}",
"content": "Nearby API relation",
"metadata": {
"document_id": "api.control_actions_endpoint",
"target_document_id": "architecture.telegram_notify_app",
},
},
{
"layer": "D5_RELATION_GRAPH",
"path": "docs/documentation/api/health-endpoint.md",
"title": "HTTP API /health",
"content": "Health relation",
"metadata": {
"document_id": "api.health_endpoint",
"target_document_id": "domain.runtime_health",
"endpoint": "/health",
},
},
]
runner = DocsQAPipelineRunner(IntentRouterV2(), InMemoryDocsRetrievalAdapter(rows), repo_context=repo_context())
result = runner.run("Какие документы связаны с endpoint /health?", "docs-session", mode="pre_llm_only")
assert result.diagnostics.anchor_candidates[0] == "api.health_endpoint"
assert result.diagnostics.selected_anchor == "api.health_endpoint"
assert result.diagnostics.anchor_match_type == "exact_path"
assert result.diagnostics.anchor_selection_reason == "metadata.endpoint exact match"
assert result.diagnostics.doc_ids[0] == "api.health_endpoint"
def test_pre_llm_targeted_chunk_fallback_enriches_api_method_layers() -> None:
rows = [
{
"layer": "D1_DOCUMENT_CATALOG",
"path": "docs/api/health.md",
"title": "GET /health",
"content": "Health endpoint",
"metadata": {"doc_id": "api.health_endpoint"},
},
{
"layer": "D0_DOC_CHUNKS",
"path": "docs/api/health.md",
"title": "api.health_endpoint:Scenario",
"content": "request -> check runtime -> return status",
"metadata": {"doc_id": "api.health_endpoint"},
},
]
runner = DocsQAPipelineRunner(IntentRouterV2(), InMemoryDocsRetrievalAdapter(rows), repo_context=repo_context())
result = runner.run("Объясни API метод /health", "docs-session", mode="pre_llm_only")
assert "D0_DOC_CHUNKS" in result.diagnostics.executed_layers
assert "D0_DOC_CHUNKS" in result.diagnostics.non_empty_layers
assert result.diagnostics.layer_diagnostics["D0_DOC_CHUNKS"]["hits"] == 1
assert result.diagnostics.selected_chunk_ids == ["api.health_endpoint"]
def test_openapi_request_fragment_uses_fragment_aware_gate() -> None:
rows = [
{
"layer": "D2_FACT_INDEX",
"path": "docs/api/send.md",
"title": "POST /send",
"content": "Request payload contains message and chat_id fields.",
"metadata": {
"document_id": "api.send_message_endpoint",
"endpoint": "/send",
"http_method": "post",
},
}
]
runner = DocsQAPipelineRunner(IntentRouterV2(), InMemoryDocsRetrievalAdapter(rows), repo_context=repo_context())
result = runner.run("Сгенерируй request schema для endpoint /send", "docs-session", mode="pre_llm_only")
assert result.router_result.query_plan.sub_intent == "OPENAPI_FRAGMENT_GENERATE"
assert result.diagnostics.requested_fragment_type == "request_schema"
assert "path" in result.diagnostics.fragment_evidence_found
assert "payload_description" in result.diagnostics.fragment_evidence_found
assert result.diagnostics.gate_decision in {"allow", "partial"}
assert result.diagnostics.gate_decision_reason in {"evidence_sufficient", "fragment_payload_only"}
assert result.diagnostics.fragment_missing_requirements == []
assert result.answer_mode in {"ready", "ready_partial"}
assert result.answer
assert "type: object" in result.answer
assert "message:" in result.answer
assert "chat_id:" in result.answer
def test_openapi_method_with_only_path_is_rejected() -> None:
rows = [
{
"layer": "D1_DOCUMENT_CATALOG",
"path": "docs/api/health.md",
"title": "/health",
"content": "",
"metadata": {"endpoint": "/health", "doc_id": "api.health_endpoint"},
}
]
runner = DocsQAPipelineRunner(IntentRouterV2(), InMemoryDocsRetrievalAdapter(rows), repo_context=repo_context())
result = runner.run("Сгенерируй OpenAPI для endpoint /health", "docs-session", mode="pre_llm_only")
assert result.diagnostics.gate_decision == "reject"
assert result.answer_mode == "degraded"
def test_openapi_response_fragment_does_not_require_request_payload() -> None:
rows = [
{
"layer": "D2_FACT_INDEX",
"path": "docs/api/health.md",
"title": "GET /health",
"content": "Returns 200 with status and uptime fields.",
"metadata": {
"endpoint": "/health",
"http_method": "get",
"response_schema": {"type": "object", "properties": {"status": {}, "uptime": {}}},
},
}
]
runner = DocsQAPipelineRunner(IntentRouterV2(), InMemoryDocsRetrievalAdapter(rows), repo_context=repo_context())
result = runner.run("Сгенерируй response schema для endpoint /health", "docs-session", mode="pre_llm_only")
assert result.diagnostics.requested_fragment_type == "response_schema"
assert result.diagnostics.gate_decision != "reject"
assert result.answer_mode != "degraded"
assert "properties:" in result.answer
def test_system_flow_evidence_prefers_workflows_and_relations() -> None:
rows = [
{
"layer": "D1_DOCUMENT_CATALOG",
"path": "docs/workflows/checkout.md",
"title": "Checkout overview",
"content": "Overview doc",
"metadata": {"doc_id": "workflow.checkout"},
},
{
"layer": "D4_WORKFLOW_INDEX",
"path": "docs/workflows/checkout.md",
"title": "Checkout steps",
"content": "cart -> payment -> confirm",
"metadata": {"workflow_id": "workflow.checkout"},
},
{
"layer": "D5_RELATION_GRAPH",
"path": "docs/workflows/checkout.md",
"title": "Checkout relations",
"content": "Related to payment entity",
"metadata": {"doc_id": "workflow.checkout", "target_doc_id": "domain.payment"},
},
{
"layer": "D0_DOC_CHUNKS",
"path": "docs/workflows/checkout.md",
"title": "workflow.checkout:chunk",
"content": "payment step persists order",
"metadata": {"chunk_id": "chunk.checkout.1", "doc_id": "workflow.checkout"},
},
]
runner = DocsQAPipelineRunner(IntentRouterV2(), InMemoryDocsRetrievalAdapter(rows), repo_context=repo_context())
result = runner.run("Объясни как работает checkout workflow", "docs-session", mode="pre_llm_only")
assert result.diagnostics.planned_layers == ["D4_WORKFLOW_INDEX", "D5_RELATION_GRAPH", "D1_DOCUMENT_CATALOG", "D0_DOC_CHUNKS"]
assert result.diagnostics.selected_workflow_ids == ["workflow.checkout"]
assert result.diagnostics.selected_relation_ids == ["domain.payment"]
assert result.diagnostics.selected_chunk_ids == ["chunk.checkout.1"]
def test_component_evidence_prefers_facts_and_targeted_chunks() -> None:
rows = [
{
"layer": "D1_DOCUMENT_CATALOG",
"path": "docs/components/runtime-manager.md",
"title": "Runtime manager",
"content": "Overview",
"metadata": {"doc_id": "architecture.runtime_manager"},
},
{
"layer": "D2_FACT_INDEX",
"path": "docs/components/runtime-manager.md",
"title": "Runtime facts",
"content": "Coordinates runtime orchestration",
"metadata": {"fact_id": "fact.runtime_manager.role", "doc_id": "architecture.runtime_manager"},
},
{
"layer": "D0_DOC_CHUNKS",
"path": "docs/components/runtime-manager.md",
"title": "RuntimeManager chunk",
"content": "Starts worker loops and supervises control channels",
"metadata": {"chunk_id": "chunk.runtime_manager.1", "doc_id": "architecture.runtime_manager"},
},
]
runner = DocsQAPipelineRunner(IntentRouterV2(), InMemoryDocsRetrievalAdapter(rows), repo_context=repo_context())
result = runner.run("Какую роль в системе играет RuntimeManager?", "docs-session", mode="pre_llm_only")
assert result.diagnostics.planned_layers == ["D2_FACT_INDEX", "D5_RELATION_GRAPH", "D1_DOCUMENT_CATALOG", "D0_DOC_CHUNKS"]
assert result.diagnostics.selected_fact_ids == ["fact.runtime_manager.role"]
assert result.diagnostics.selected_doc_ids == ["architecture.runtime_manager"]
assert result.diagnostics.selected_chunk_ids == ["chunk.runtime_manager.1"]
@@ -53,7 +53,7 @@ def test_e2e_docs_switch_from_code_topic() -> None:
)
assert_intent(first, "CODE_QA")
assert_intent(second, "DOCS_QA")
assert_intent(second, "DOCUMENTATION_EXPLAIN")
assert second.conversation_mode == "SWITCH"
assert_domains(second, ["DOCS"])
carried = [
@@ -56,8 +56,8 @@ def test_invariant_inline_code_span_routes_to_code_and_extracts_symbol() -> None
def test_invariant_docs_cyrillic_path_with_quotes() -> None:
result = run_sequence(["Что сказано в «docs/архитектура.md»?"])[0]
assert_intent(result, "DOCS_QA")
assert_sub_intent(result, "EXPLAIN")
assert_intent(result, "DOCUMENTATION_EXPLAIN")
assert_sub_intent(result, "COMPONENT_EXPLAIN")
assert_domains(result, ["DOCS"])
assert "docs/архитектура.md" in result.query_plan.normalized
assert_has_file_path(result, "docs/архитектура.md")
@@ -113,8 +113,92 @@ def test_invariant_open_file_sub_intent_uses_narrow_retrieval_profile() -> None:
def test_invariant_docs_question_routes_to_docs() -> None:
result = run_sequence(["Что сказано в документации?"])[0]
assert_intent(result, "DOCS_QA")
assert_intent(result, "DOCUMENTATION_EXPLAIN")
assert_domains(result, ["DOCS"])
assert_domain_layer_prefixes(result)
assert result.query_plan.keyword_hints
assert any(item in result.query_plan.expansions for item in result.query_plan.keyword_hints)
def test_invariant_docs_flow_sub_intent_uses_workflow_layers() -> None:
result = run_sequence(["Как работает процесс создания заказа по документации?"])[0]
assert_intent(result, "DOCUMENTATION_EXPLAIN")
assert_sub_intent(result, "SYSTEM_FLOW_EXPLAIN")
layer_ids = [item.layer_id for item in result.retrieval_spec.layer_queries]
assert layer_ids == ["D4_WORKFLOW_INDEX", "D5_RELATION_GRAPH", "D1_DOCUMENT_CATALOG", "D0_DOC_CHUNKS"]
def test_invariant_docs_entity_sub_intent_uses_entity_layers() -> None:
result = run_sequence(["Что такое сущность Order в документации?"])[0]
assert_intent(result, "DOCUMENTATION_EXPLAIN")
assert_sub_intent(result, "ENTITY_EXPLAIN")
layer_ids = [item.layer_id for item in result.retrieval_spec.layer_queries]
assert layer_ids == ["D3_ENTITY_CATALOG", "D5_RELATION_GRAPH", "D1_DOCUMENT_CATALOG", "D0_DOC_CHUNKS"]
def test_invariant_entity_like_camel_case_prefers_entity_explain() -> None:
result = run_sequence(["Что такое WorkerHealth?"])[0]
assert_intent(result, "DOCUMENTATION_EXPLAIN")
assert_sub_intent(result, "ENTITY_EXPLAIN")
def test_invariant_related_docs_routes_to_docs_explain() -> None:
result = run_sequence(["Найди документацию по billing"])[0]
assert_intent(result, "DOCUMENTATION_EXPLAIN")
assert_sub_intent(result, "RELATED_DOCS_EXPLAIN")
layer_ids = [item.layer_id for item in result.retrieval_spec.layer_queries]
assert layer_ids == ["D5_RELATION_GRAPH", "D1_DOCUMENT_CATALOG", "D0_DOC_CHUNKS"]
def test_invariant_docs_navigation_uses_related_docs_explain() -> None:
result = run_sequence(["Что связано с checkout документацией?"])[0]
assert_intent(result, "DOCUMENTATION_EXPLAIN")
assert_sub_intent(result, "RELATED_DOCS_EXPLAIN")
layer_ids = [item.layer_id for item in result.retrieval_spec.layer_queries]
assert layer_ids == ["D5_RELATION_GRAPH", "D1_DOCUMENT_CATALOG", "D0_DOC_CHUNKS"]
def test_invariant_openapi_routes_to_docs_layers_with_api_filter() -> None:
result = run_sequence(["Сгенерируй openapi yaml для создания заказа"])[0]
assert_intent(result, "OPENAPI_GENERATION")
assert_sub_intent(result, "OPENAPI_METHOD_GENERATE")
layer_ids = [item.layer_id for item in result.retrieval_spec.layer_queries]
assert layer_ids == ["D1_DOCUMENT_CATALOG", "D2_FACT_INDEX", "D0_DOC_CHUNKS"]
assert result.retrieval_spec.filters.doc_type == "api_method"
def test_invariant_general_qa_routes_to_generic_docs_layers() -> None:
result = run_sequence(["Помоги разобраться"])[0]
assert_intent(result, "GENERAL_QA")
assert_sub_intent(result, "GENERIC_QA")
assert_domains(result, ["DOCS"])
layer_ids = [item.layer_id for item in result.retrieval_spec.layer_queries]
assert layer_ids == ["D1_DOCUMENT_CATALOG", "D0_DOC_CHUNKS"]
def test_invariant_component_like_manager_routes_to_component_explain() -> None:
result = run_sequence(["Какую роль в системе играет RuntimeManager?"])[0]
assert_intent(result, "DOCUMENTATION_EXPLAIN")
assert_sub_intent(result, "COMPONENT_EXPLAIN")
def test_invariant_cycle_query_routes_to_system_flow_explain() -> None:
result = run_sequence(["Объясни как работает цикл отправки уведомлений"])[0]
assert_intent(result, "DOCUMENTATION_EXPLAIN")
assert_sub_intent(result, "SYSTEM_FLOW_EXPLAIN")
def test_invariant_overview_question_routes_to_general_qa() -> None:
result = run_sequence(["Что вообще описано в документации по этому сервису?"])[0]
assert_intent(result, "GENERAL_QA")
assert_sub_intent(result, "GENERIC_QA")
@@ -0,0 +1,95 @@
from __future__ import annotations
import json
from app.modules.agent.intent_router_v2 import ConversationState, IntentRouterV2
from app.modules.agent.intent_router_v2.intent.classifier import IntentClassifierV2
from app.modules.agent.intent_router_v2.intent.llm_disambiguator import DocsLlmDisambiguator
from tests.unit_tests.rag.intent_router_testkit import repo_context
class FakeLlm:
def __init__(self, response: str, *, fail: bool = False) -> None:
self.response = response
self.fail = fail
self.calls: list[tuple[str, str]] = []
def generate(self, prompt_name: str, user_input: str, *, log_context: str | None = None) -> str:
self.calls.append((prompt_name, user_input))
if self.fail:
raise RuntimeError("llm unavailable")
return self.response
def test_technical_query_keeps_deterministic_routing_without_llm_call() -> None:
llm = FakeLlm('{"sub_intent":"GENERIC_QA","reason":"unused","confidence":"low"}')
router = IntentRouterV2(
classifier=IntentClassifierV2(),
llm_disambiguator=DocsLlmDisambiguator(llm),
enable_llm_disambiguation=True,
)
result = router.route("Объясни endpoint /health", ConversationState(), repo_context())
assert result.query_plan.sub_intent == "API_METHOD_EXPLAIN"
assert result.is_ambiguous is False
assert result.routing_mode == "deterministic"
assert result.llm_router_used is False
assert llm.calls == []
def test_ambiguous_query_can_be_resolved_by_llm() -> None:
llm = FakeLlm('{"sub_intent":"ENTITY_EXPLAIN","reason":"runtime health is a concept/entity here","confidence":"medium"}')
router = IntentRouterV2(
classifier=IntentClassifierV2(),
llm_disambiguator=DocsLlmDisambiguator(llm),
enable_llm_disambiguation=True,
)
result = router.route("Объясни runtime health", ConversationState(), repo_context())
assert result.is_ambiguous is True
assert result.routing_mode == "llm_disambiguation"
assert result.llm_router_used is True
assert result.deterministic_selected_sub_intent
assert result.llm_router_selected_sub_intent == "ENTITY_EXPLAIN"
assert result.query_plan.sub_intent == "ENTITY_EXPLAIN"
def test_ambiguous_query_falls_back_to_deterministic_when_llm_fails() -> None:
llm = FakeLlm("{}", fail=True)
router = IntentRouterV2(
classifier=IntentClassifierV2(),
llm_disambiguator=DocsLlmDisambiguator(llm),
enable_llm_disambiguation=True,
)
result = router.route("Как работает health check runtime?", ConversationState(), repo_context())
assert result.is_ambiguous is True
assert result.routing_mode == "deterministic_fallback"
assert result.llm_router_used is False
assert result.llm_router_error == "llm unavailable"
assert result.query_plan.sub_intent == result.deterministic_selected_sub_intent
def test_overview_query_stays_in_generic_qa() -> None:
llm = FakeLlm('{"sub_intent":"GENERIC_QA","reason":"overview query","confidence":"high"}')
router = IntentRouterV2(
classifier=IntentClassifierV2(),
llm_disambiguator=DocsLlmDisambiguator(llm),
enable_llm_disambiguation=True,
)
result = router.route("Какая структура документации?", ConversationState(), repo_context())
assert result.is_ambiguous is False or result.query_plan.sub_intent == "GENERIC_QA"
assert result.intent == "GENERAL_QA"
assert result.query_plan.sub_intent == "GENERIC_QA"
def test_llm_disambiguator_rejects_unknown_labels() -> None:
llm = FakeLlm(json.dumps({"sub_intent": "MADE_UP", "reason": "bad", "confidence": "high"}))
disambiguator = DocsLlmDisambiguator(llm)
assert disambiguator.choose({"query": "test"}) is None