Фиксируем состояние

This commit is contained in:
2026-03-12 20:40:29 +03:00
parent 095d354112
commit b1f825e6b9
220 changed files with 23993 additions and 92568 deletions

View File

@@ -66,8 +66,8 @@ class SummaryComposer:
"",
f"Passed: {passed}/{len(results)}",
"",
"| File | Case | Mode | Query | Actual sub-intent | RAG layers | Pass |",
"|------|------|------|-------|-------------------|------------|------|",
"| File | Case | Mode | Query | Actual sub-intent | RAG layers | Tokens | Pass |",
"|------|------|------|-------|-------------------|------------|--------|------|",
]
lines.extend(self._rows(results))
failures = [item for item in results if not item.passed]
@@ -84,7 +84,7 @@ class SummaryComposer:
rows.append(
f"| {item.case.source_file.name} | {item.case.case_id} | {item.case.mode} | "
f"{self._cell(item.case.query)} | {item.actual.get('sub_intent') or ''} | "
f"{self._layer_text(item.details)} | {'' if item.passed else ''} |"
f"{self._layer_text(item.details)} | {self._token_text(item.details)} | {'' if item.passed else ''} |"
)
return rows
@@ -104,6 +104,16 @@ class SummaryComposer:
return compact
return compact[: limit - 1].rstrip() + ""
def _token_text(self, details: dict) -> str:
token_usage = dict(details.get("token_usage") or {})
direct = token_usage.get("tokens_in_estimate")
if direct is not None:
return str(direct)
prompt = dict(details.get("diagnostics", {}).get("prompt") or {})
stats = dict(prompt.get("prompt_stats") or {})
value = stats.get("tokens_in_estimate")
return str(value) if value is not None else ""
def _llm_section(self, results: list[V3CaseResult]) -> list[str]:
llm_results = [item for item in results if str(item.actual.get("llm_answer") or "").strip()]
if not llm_results:

View File

@@ -2,6 +2,7 @@ from __future__ import annotations
from pathlib import Path
from tests.pipeline_setup_v3.core.artifacts import SummaryComposer
from tests.pipeline_setup_v3.core.models import CaseExpectations, LlmExpectation, RetrievalExpectation, RouterExpectation, V3Case
from tests.pipeline_setup_v3.core.validators import CaseValidator
@@ -66,3 +67,29 @@ def test_llm_contains_and_excludes_checks() -> None:
}
assert CaseValidator().validate(case, actual, {}) == []
def test_summary_includes_token_usage_column() -> None:
case = V3Case(
case_id="llm-quality",
runner="agent_runtime",
mode="full_chain",
query="Где health endpoint?",
source_file=Path("cases.yaml"),
)
result_text = SummaryComposer().compose([
type(
"Result",
(),
{
"case": case,
"actual": {"sub_intent": "FIND_ENTRYPOINTS"},
"details": {"token_usage": {"tokens_in_estimate": 321}, "rag_rows": []},
"passed": True,
"mismatches": [],
},
)()
])
assert "Tokens" in result_text
assert "321" in result_text