73 lines
2.0 KiB
Python
73 lines
2.0 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Literal
|
|
|
|
|
|
ComponentKind = Literal[
|
|
"process_v2_intent_router",
|
|
"process_v2_retrieval_policy_resolver",
|
|
"process_v2_router_plus_retrieval_policy",
|
|
"process_v2_router_plus_retrieval_policy_rag",
|
|
"process_v2_full_chain",
|
|
]
|
|
|
|
|
|
@dataclass(slots=True, frozen=True)
|
|
class RouterExpectation:
|
|
domain: str | None = None
|
|
intent: str | None = None
|
|
sub_intent: str | None = None
|
|
|
|
|
|
@dataclass(slots=True, frozen=True)
|
|
class RetrievalPlanExpectation:
|
|
profile: str | None = None
|
|
layers: tuple[str, ...] = ()
|
|
limit: int | None = None
|
|
filters: dict[str, object] = field(default_factory=dict)
|
|
|
|
|
|
@dataclass(slots=True, frozen=True)
|
|
class CaseExpectations:
|
|
router: RouterExpectation = RouterExpectation()
|
|
retrieval_plan: RetrievalPlanExpectation = field(default_factory=RetrievalPlanExpectation)
|
|
route_assertions: dict[str, object] = field(default_factory=dict)
|
|
retrieval_plan_assertions: dict[str, object] = field(default_factory=dict)
|
|
rag_assertions: dict[str, object] = field(default_factory=dict)
|
|
pipeline_assertions: dict[str, object] = field(default_factory=dict)
|
|
llm_assertions: dict[str, object] = field(default_factory=dict)
|
|
|
|
|
|
@dataclass(slots=True, frozen=True)
|
|
class V4Case:
|
|
case_id: str
|
|
component: ComponentKind
|
|
source_file: Path
|
|
query: str = ""
|
|
rag_session_id: str | None = None
|
|
route: dict[str, object] = field(default_factory=dict)
|
|
expectations: CaseExpectations = field(default_factory=CaseExpectations)
|
|
notes: str = ""
|
|
tags: tuple[str, ...] = ()
|
|
|
|
@property
|
|
def display_input(self) -> str:
|
|
return self.query or self.route.get("user_query") or self.case_id
|
|
|
|
|
|
@dataclass(slots=True, frozen=True)
|
|
class ExecutionPayload:
|
|
actual: dict
|
|
details: dict
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class V4CaseResult:
|
|
case: V4Case
|
|
actual: dict
|
|
details: dict
|
|
passed: bool
|
|
mismatches: list[str] = field(default_factory=list)
|