88 lines
2.2 KiB
Python
88 lines
2.2 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Literal
|
|
|
|
|
|
RunnerKind = Literal["agent_runtime", "process_v2"]
|
|
ModeKind = Literal["router_only", "router_rag", "full_chain"]
|
|
|
|
|
|
@dataclass(slots=True, frozen=True)
|
|
class CaseInput:
|
|
repo_path: Path | None = None
|
|
project_id: str | None = None
|
|
rag_session_id: str | None = None
|
|
|
|
|
|
@dataclass(slots=True, frozen=True)
|
|
class RouterExpectation:
|
|
domain: str | None = None
|
|
intent: str | None = None
|
|
sub_intent: str | None = None
|
|
graph_id: str | None = None
|
|
conversation_mode: str | None = None
|
|
|
|
|
|
@dataclass(slots=True, frozen=True)
|
|
class RetrievalExpectation:
|
|
non_empty: bool | None = None
|
|
min_rows: int | None = None
|
|
direct_symbol_test_hits_max: int | None = None
|
|
path_scope_contains: tuple[str, ...] = ()
|
|
doc_scope_contains: tuple[str, ...] = ()
|
|
symbol_candidates_contain: tuple[str, ...] = ()
|
|
entity_candidates_contain: tuple[str, ...] = ()
|
|
layers_include: tuple[str, ...] = ()
|
|
filters_contain: dict[str, str] = field(default_factory=dict)
|
|
|
|
|
|
@dataclass(slots=True, frozen=True)
|
|
class LlmExpectation:
|
|
non_empty: bool | None = None
|
|
contains_all: tuple[str, ...] = ()
|
|
excludes: tuple[str, ...] = ()
|
|
answer_mode: str | None = None
|
|
|
|
|
|
@dataclass(slots=True, frozen=True)
|
|
class PipelineExpectation:
|
|
answer_mode: str | None = None
|
|
|
|
|
|
@dataclass(slots=True, frozen=True)
|
|
class CaseExpectations:
|
|
router: RouterExpectation = RouterExpectation()
|
|
retrieval: RetrievalExpectation = RetrievalExpectation()
|
|
llm: LlmExpectation = LlmExpectation()
|
|
pipeline: PipelineExpectation = PipelineExpectation()
|
|
|
|
|
|
@dataclass(slots=True, frozen=True)
|
|
class V3Case:
|
|
case_id: str
|
|
runner: RunnerKind
|
|
mode: ModeKind
|
|
query: str
|
|
source_file: Path
|
|
input: CaseInput = CaseInput()
|
|
expectations: CaseExpectations = CaseExpectations()
|
|
notes: str = ""
|
|
tags: tuple[str, ...] = ()
|
|
|
|
|
|
@dataclass(slots=True, frozen=True)
|
|
class ExecutionPayload:
|
|
actual: dict
|
|
details: dict
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class V3CaseResult:
|
|
case: V3Case
|
|
actual: dict
|
|
details: dict
|
|
passed: bool
|
|
mismatches: list[str] = field(default_factory=list)
|