43 lines
1.6 KiB
Python
43 lines
1.6 KiB
Python
import asyncio
|
|
|
|
from app.modules.agent.engine.orchestrator.models import OutputContract, OutputSection, RoutingMeta, Scenario, TaskConstraints, TaskSpec
|
|
from app.modules.agent.engine.orchestrator.service import OrchestratorService
|
|
|
|
|
|
def test_quality_metrics_present_and_scored() -> None:
|
|
service = OrchestratorService()
|
|
task = TaskSpec(
|
|
task_id="quality-1",
|
|
dialog_session_id="dialog-1",
|
|
rag_session_id="rag-1",
|
|
mode="auto",
|
|
user_message="Explain architecture",
|
|
scenario=Scenario.EXPLAIN_PART,
|
|
routing=RoutingMeta(domain_id="project", process_id="qa", confidence=0.9, reason="test"),
|
|
constraints=TaskConstraints(allow_writes=False),
|
|
output_contract=OutputContract(
|
|
result_type="answer",
|
|
sections=[
|
|
OutputSection(name="sequence_diagram", format="mermaid"),
|
|
OutputSection(name="use_cases", format="markdown"),
|
|
OutputSection(name="summary", format="markdown"),
|
|
],
|
|
),
|
|
metadata={"rag_context": "A\nB", "confluence_context": "", "files_map": {}},
|
|
)
|
|
|
|
result = asyncio.run(
|
|
service.run(
|
|
task=task,
|
|
graph_resolver=lambda _d, _p: object(),
|
|
graph_invoker=lambda _g, _s, _id: {"answer": "unused", "changeset": []},
|
|
)
|
|
)
|
|
|
|
quality = result.meta.get("quality", {})
|
|
assert quality
|
|
assert quality.get("faithfulness", {}).get("score") is not None
|
|
assert quality.get("coverage", {}).get("score") is not None
|
|
assert quality.get("status") in {"ok", "needs_review", "fail"}
|
|
assert quality.get("coverage", {}).get("covered_count", 0) >= 1
|