Фиксируем состояние
This commit is contained in:
@@ -65,8 +65,8 @@ class SummaryComposer:
|
||||
"",
|
||||
f"Passed: {passed}/{len(results)}",
|
||||
"",
|
||||
"| File | Case | Query | Expected sub-intent | Intent | Actual sub-intent | RAG layers | Pass |",
|
||||
"|------|------|-------|---------------------|--------|-------------------|------------|------|",
|
||||
"| File | Case | Query | Expected sub-intent | Intent | Actual sub-intent | RAG layers | Tokens | Pass |",
|
||||
"|------|------|-------|---------------------|--------|-------------------|------------|--------|------|",
|
||||
]
|
||||
lines.extend(self._result_rows(results))
|
||||
lines.extend(self._failure_section(results))
|
||||
@@ -80,7 +80,7 @@ class SummaryComposer:
|
||||
rows.append(
|
||||
f"| {item.case.source_file.name} | {item.case.case_id} | {self._table_text(item.case.query)} | "
|
||||
f"{item.case.expectations.router.sub_intent or '—'} | {actual.get('intent') or '—'} | "
|
||||
f"{actual.get('sub_intent') or '—'} | {self._rag_layers_text(item)} | {'✓' if item.passed else '✗'} |"
|
||||
f"{actual.get('sub_intent') or '—'} | {self._rag_layers_text(item)} | {self._token_text(item)} | {'✓' if item.passed else '✗'} |"
|
||||
)
|
||||
return rows
|
||||
|
||||
@@ -142,3 +142,10 @@ class SummaryComposer:
|
||||
return "—"
|
||||
parts = [f"{layer}:{counts[layer]}" for layer in sorted(counts)]
|
||||
return self._table_text(", ".join(parts), limit=120)
|
||||
|
||||
def _token_text(self, item: V2CaseResult) -> str:
|
||||
diagnostics = dict(item.details.get("diagnostics") or {})
|
||||
prompt = dict(diagnostics.get("prompt") or {})
|
||||
stats = dict(prompt.get("prompt_stats") or {})
|
||||
value = stats.get("tokens_in_estimate")
|
||||
return str(value) if value is not None else "—"
|
||||
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from tests.pipeline_setup_v2.core.artifacts import SummaryComposer
|
||||
from tests.pipeline_setup_v2.core.models import CaseExpectations, LlmExpectation, RetrievalExpectation, RouterExpectation, V2Case
|
||||
from tests.pipeline_setup_v2.core.validators import CaseValidator
|
||||
|
||||
@@ -108,3 +109,29 @@ def test_llm_contains_and_excludes_checks() -> None:
|
||||
mismatches = CaseValidator().validate(case, actual, {})
|
||||
|
||||
assert mismatches == []
|
||||
|
||||
|
||||
def test_summary_includes_token_usage_column() -> None:
|
||||
case = V2Case(
|
||||
case_id="llm-quality",
|
||||
runner="runtime",
|
||||
mode="full_chain",
|
||||
query="Где health endpoint?",
|
||||
source_file=Path("cases.yaml"),
|
||||
)
|
||||
result_text = SummaryComposer().compose([
|
||||
type(
|
||||
"Result",
|
||||
(),
|
||||
{
|
||||
"case": case,
|
||||
"actual": {"intent": "CODE_QA", "sub_intent": "FIND_ENTRYPOINTS"},
|
||||
"details": {"diagnostics": {"prompt": {"prompt_stats": {"tokens_in_estimate": 654}}}, "rag_rows": []},
|
||||
"passed": True,
|
||||
"mismatches": [],
|
||||
},
|
||||
)()
|
||||
])
|
||||
|
||||
assert "Tokens" in result_text
|
||||
assert "654" in result_text
|
||||
|
||||
Reference in New Issue
Block a user