Роутер работает нормально в process v2

This commit is contained in:
2026-04-07 14:09:51 +03:00
parent 5d77ab1a88
commit 6b74d410cd
1748 changed files with 216679 additions and 14208 deletions
+22 -2
View File
@@ -2,11 +2,12 @@ from __future__ import annotations
class InMemoryDocsRetrievalAdapter:
def __init__(self, rows: list[dict]) -> None:
def __init__(self, rows: list[dict], materialized_rows: list[dict] | None = None) -> None:
self._rows = list(rows)
self._materialized_rows = list(materialized_rows or [])
self._report: dict = {}
def retrieve_with_plan(self, rag_session_id: str, query: str, retrieval_spec, retrieval_constraints=None, *, query_plan=None) -> list[dict]:
def retrieve_with_plan(self, rag_session_id: str, query: str, retrieval_spec, retrieval_constraints=None, *, query_plan=None, trace=None) -> list[dict]:
planned_layers = [str(item.layer_id) for item in retrieval_spec.layer_queries]
query_sub_intent = str(getattr(query_plan, "sub_intent", "") or "")
relation_rows = [row for row in self._rows if str(row.get("layer") or "") == "D5_RELATION_GRAPH"]
@@ -60,6 +61,25 @@ class InMemoryDocsRetrievalAdapter:
def retrieve_exact_files(self, rag_session_id: str, *, repo_id=None, paths: list[str], layers=None, limit: int = 200, query: str = "", ranking_profile: str = "") -> list[dict]:
return []
def materialize_document_ids(self, rag_session_id: str, *, document_ids: list[str], layers=None, limit: int = 200) -> list[dict]:
allowed_ids = {str(item).strip() for item in document_ids if str(item).strip()}
if not allowed_ids:
return []
allowed_layers = {str(item).strip() for item in list(layers or []) if str(item).strip()}
result = []
for row in [*self._rows, *self._materialized_rows]:
if allowed_layers and str(row.get("layer") or "") not in allowed_layers:
continue
metadata = dict(row.get("metadata") or {})
candidates = {
str(metadata.get("document_id") or "").strip(),
str(metadata.get("doc_id") or "").strip(),
str(metadata.get("subject_id") or "").strip(),
}
if candidates & allowed_ids:
result.append(row)
return self._dedupe(result)[: max(1, int(limit))]
def hydrate_resolved_symbol_sources(self, rag_session_id: str, base_query: str, rag_rows: list[dict], symbol_resolution: dict, retrieval_spec, retrieval_constraints=None) -> list[dict]:
return list(rag_rows)
+2 -2
View File
@@ -2,8 +2,8 @@ from __future__ import annotations
from dataclasses import dataclass
from app.modules.agent.intent_router_v2 import IntentRouterV2
from app.modules.agent.runtime.docs_qa_pipeline import DocsQAPipelineRunner
from app.core.agent.intent_router import IntentRouterV2
from app.core.agent.runtime.docs_qa_pipeline import DocsQAPipelineRunner
from tests.docs_qa_eval.config import DocsEvalConfig
from tests.docs_qa_eval.fixture_adapter import InMemoryDocsRetrievalAdapter
from tests.docs_qa_eval.golden_loader import DocsGoldenCase, load_cases
@@ -0,0 +1,421 @@
defaults:
runner: process_v2
mode: full_chain
input:
rag_session_id: "fdf3ff03-81f0-4772-b68e-250147960243"
cases:
- id: v2-general-01-what-is-service
query: "Что это за сервис?"
expected:
router:
domain: GENERAL
intent: GENERAL_QA
sub_intent: SUMMARY
pipeline:
answer_mode: grounded_summary
llm:
non_empty: true
contains_all: ["Telegram", "сервис"]
- id: v2-general-02-purpose
query: "Для чего нужен test_echo_app?"
expected:
router:
domain: GENERAL
intent: GENERAL_QA
sub_intent: SUMMARY
pipeline:
answer_mode: grounded_summary
llm:
non_empty: true
contains_all: ["test_echo_app"]
- id: v2-general-03-problem
query: "Какую задачу решает это приложение?"
expected:
router:
domain: GENERAL
intent: GENERAL_QA
sub_intent: SUMMARY
pipeline:
answer_mode: grounded_summary
llm:
non_empty: true
contains_all: ["Telegram"]
- id: v2-general-04-docs-contents
query: "Что входит в документацию этого проекта?"
expected:
router:
domain: GENERAL
intent: GENERAL_QA
sub_intent: SUMMARY
llm:
non_empty: true
contains_all: ["документац", "архитект"]
- id: v2-general-05-main-parts
query: "Какие основные части есть у системы?"
expected:
router:
domain: GENERAL
intent: GENERAL_QA
sub_intent: SUMMARY
llm:
non_empty: true
contains_all: ["систем", "Telegram"]
- id: v2-general-06-high-level-architecture
query: "Как в целом устроено приложение?"
expected:
router:
domain: GENERAL
intent: GENERAL_QA
sub_intent: SUMMARY
llm:
non_empty: true
contains_all: ["приложен"]
- id: v2-general-07-besides-api
query: "Что тут есть кроме API?"
expected:
router:
domain: GENERAL
intent: GENERAL_QA
sub_intent: SUMMARY
llm:
non_empty: true
contains_all: ["API"]
- id: v2-general-08-components
query: "Из чего состоит Telegram Notify App?"
expected:
router:
domain: GENERAL
intent: GENERAL_QA
sub_intent: SUMMARY
llm:
non_empty: true
contains_all: ["Telegram Notify App"]
- id: v2-general-09-reading-order
query: "Какие документы стоит читать сначала?"
expected:
router:
domain: GENERAL
intent: GENERAL_QA
sub_intent: SUMMARY
llm:
non_empty: true
contains_all: ["README", "архитект"]
- id: v2-general-10-short-summary
query: "Дай короткое summary по проектной документации"
expected:
router:
domain: GENERAL
intent: GENERAL_QA
sub_intent: SUMMARY
llm:
non_empty: true
contains_all: ["документац"]
- id: v2-docs-summary-01-health
query: "Кратко объясни по документации, как работает `/health`"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: SUMMARY
retrieval:
non_empty: true
layers_include: ["D1_DOCUMENT_CATALOG", "D3_ENTITY_CATALOG", "D0_DOC_CHUNKS"]
doc_scope_contains: ["docs/README.md", "docs/architecture/telegram-notify-app-overview.md"]
llm:
non_empty: true
contains_all: ["/health", "status", "components"]
pipeline:
answer_mode: grounded_summary
- id: v2-docs-summary-02-send
query: "Что делает endpoint `/send`?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: SUMMARY
retrieval:
non_empty: true
layers_include: ["D1_DOCUMENT_CATALOG", "D3_ENTITY_CATALOG", "D0_DOC_CHUNKS"]
doc_scope_contains: ["docs/api/send-message-endpoint.md"]
llm:
non_empty: true
contains_all: ["/send", "message", "Telegram"]
- id: v2-docs-summary-03-actions
query: "Объясни назначение `/actions/{action}`"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: SUMMARY
retrieval:
non_empty: true
doc_scope_contains: ["docs/api/control-actions-endpoint.md"]
llm:
non_empty: true
contains_all: ["/actions/{action}"]
- id: v2-docs-summary-04-runtime-health
query: "Что такое runtime health в этой документации?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: SUMMARY
retrieval:
non_empty: true
doc_scope_contains: ["docs/domains/runtime-health-entity.md"]
llm:
non_empty: true
contains_all: ["runtime health"]
- id: v2-docs-summary-05-notification-loop
query: "Кратко опиши цикл отправки уведомлений"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: SUMMARY
retrieval:
non_empty: true
doc_scope_contains: ["docs/logic/telegram-notification-loop.md"]
llm:
non_empty: true
contains_all: ["worker", "interval", "degraded"]
- id: v2-docs-summary-06-architecture
query: "Как устроена архитектура Telegram Notify App?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: SUMMARY
retrieval:
non_empty: true
doc_scope_contains: ["docs/architecture/telegram-notify-app-overview.md"]
llm:
non_empty: true
contains_all: ["RuntimeManager", "TelegramControlChannel"]
- id: v2-docs-summary-07-worker
query: "Что делает worker в этом приложении?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: SUMMARY
retrieval:
non_empty: true
doc_scope_contains: ["docs/logic/telegram-notification-loop.md", "docs/architecture/telegram-notify-app-overview.md"]
llm:
non_empty: true
contains_all: ["worker", "Telegram"]
- id: v2-docs-summary-08-health-statuses
query: "Какие статусы здоровья есть у runtime?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: SUMMARY
retrieval:
non_empty: true
doc_scope_contains: ["docs/domains/runtime-health-entity.md", "docs/api/health-endpoint.md"]
llm:
non_empty: true
contains_all: ["ok", "degraded", "unhealthy", "unknown"]
- id: v2-docs-summary-09-manual-send
query: "Как в приложении устроена ручная отправка сообщения?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: SUMMARY
retrieval:
non_empty: true
doc_scope_contains: ["docs/api/send-message-endpoint.md"]
llm:
non_empty: true
contains_all: ["/send", "message", "TelegramSendService"]
- id: v2-docs-summary-10-telegram-integration
query: "Как сервис взаимодействует с Telegram в целом?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: SUMMARY
retrieval:
non_empty: true
min_rows: 2
doc_scope_contains: ["docs/architecture/telegram-notify-app-overview.md", "docs/api/send-message-endpoint.md"]
llm:
non_empty: true
contains_all: ["Telegram", "worker", "/send"]
- id: v2-find-files-01-health
query: "В каком файле описан `/health`?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: FIND_FILES
retrieval:
non_empty: true
pipeline:
answer_mode: deterministic
llm:
non_empty: true
contains_all: ["docs/api/health-endpoint.md"]
- id: v2-find-files-02-send
query: "Покажи файл с описанием endpoint `/send`"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: FIND_FILES
retrieval:
non_empty: true
pipeline:
answer_mode: deterministic
llm:
non_empty: true
contains_all: ["docs/api/send-message-endpoint.md"]
- id: v2-find-files-03-actions
query: "Где находится документация по `/actions/{action}`?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: FIND_FILES
retrieval:
non_empty: true
pipeline:
answer_mode: deterministic
llm:
non_empty: true
contains_all: ["docs/api/control-actions-endpoint.md"]
- id: v2-find-files-04-runtime-health
query: "В каком документе описан runtime health?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: FIND_FILES
retrieval:
non_empty: true
pipeline:
answer_mode: deterministic
llm:
non_empty: true
contains_all: ["docs/domains/runtime-health-entity.md"]
- id: v2-find-files-05-loop
query: "Найди файл с логикой цикла отправки уведомлений"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: FIND_FILES
retrieval:
non_empty: true
pipeline:
answer_mode: deterministic
llm:
non_empty: true
contains_all: ["docs/logic/telegram-notification-loop.md"]
- id: v2-find-files-06-architecture
query: "Где описана архитектура приложения?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: FIND_FILES
retrieval:
non_empty: true
pipeline:
answer_mode: deterministic
llm:
non_empty: true
contains_all: ["docs/architecture/telegram-notify-app-overview.md"]
- id: v2-find-files-07-errors-catalog
query: "В каком файле лежит каталог ошибок?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: FIND_FILES
retrieval:
non_empty: true
pipeline:
answer_mode: deterministic
llm:
non_empty: true
contains_all: ["docs/errors/catalog.yaml"]
- id: v2-find-files-08-docs-index
query: "Покажи файл-индекс документации проекта"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: FIND_FILES
retrieval:
non_empty: true
pipeline:
answer_mode: deterministic
llm:
non_empty: true
contains_all: ["docs/README.md"]
- id: v2-find-files-09-telegram-bot-api
query: "Какие файлы относятся к Telegram Bot API интеграции?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: FIND_FILES
retrieval:
non_empty: true
min_rows: 2
pipeline:
answer_mode: deterministic
llm:
non_empty: true
contains_all:
["docs/architecture/telegram-notify-app-overview.md", "docs/api/send-message-endpoint.md", "docs/logic/telegram-notification-loop.md"]
- id: v2-find-files-10-telegram-notify-worker
query: "В каких файлах описан worker `telegram_notify`?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: FIND_FILES
retrieval:
non_empty: true
min_rows: 2
pipeline:
answer_mode: deterministic
llm:
non_empty: true
contains_all:
["docs/architecture/telegram-notify-app-overview.md", "docs/logic/telegram-notification-loop.md", "docs/domains/runtime-health-entity.md"]
@@ -0,0 +1,244 @@
defaults:
runner: process_v2
mode: router_only
cases:
- id: v2-router-general-01-what-is-service
query: "Что это за сервис?"
expected:
router:
domain: GENERAL
intent: GENERAL_QA
sub_intent: SUMMARY
- id: v2-router-general-02-purpose
query: "Для чего нужен test_echo_app?"
expected:
router:
domain: GENERAL
intent: GENERAL_QA
sub_intent: SUMMARY
- id: v2-router-general-03-problem
query: "Какую задачу решает это приложение?"
expected:
router:
domain: GENERAL
intent: GENERAL_QA
sub_intent: SUMMARY
- id: v2-router-general-04-docs-contents
query: "Что входит в документацию этого проекта?"
expected:
router:
domain: GENERAL
intent: GENERAL_QA
sub_intent: SUMMARY
- id: v2-router-general-05-main-parts
query: "Какие основные части есть у системы?"
expected:
router:
domain: GENERAL
intent: GENERAL_QA
sub_intent: SUMMARY
- id: v2-router-general-06-high-level-architecture
query: "Как в целом устроено приложение?"
expected:
router:
domain: GENERAL
intent: GENERAL_QA
sub_intent: SUMMARY
- id: v2-router-general-07-besides-api
query: "Что тут есть кроме API?"
expected:
router:
domain: GENERAL
intent: GENERAL_QA
sub_intent: SUMMARY
- id: v2-router-general-08-components
query: "Из чего состоит Telegram Notify App?"
expected:
router:
domain: GENERAL
intent: GENERAL_QA
sub_intent: SUMMARY
- id: v2-router-general-09-reading-order
query: "Какие документы стоит читать сначала?"
expected:
router:
domain: GENERAL
intent: GENERAL_QA
sub_intent: SUMMARY
- id: v2-router-general-10-short-summary
query: "Дай короткое summary по проектной документации"
expected:
router:
domain: GENERAL
intent: GENERAL_QA
sub_intent: SUMMARY
- id: v2-router-docs-summary-01-health
query: "Кратко объясни по документации, как работает `/health`"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: SUMMARY
- id: v2-router-docs-summary-02-send
query: "Что делает endpoint `/send`?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: SUMMARY
- id: v2-router-docs-summary-03-actions
query: "Объясни назначение `/actions/{action}`"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: SUMMARY
- id: v2-router-docs-summary-04-runtime-health
query: "Что такое runtime health в этой документации?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: SUMMARY
- id: v2-router-docs-summary-05-notification-loop
query: "Кратко опиши цикл отправки уведомлений"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: SUMMARY
- id: v2-router-docs-summary-06-architecture
query: "Как устроена архитектура Telegram Notify App?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: SUMMARY
- id: v2-router-docs-summary-07-worker
query: "Что делает worker в этом приложении?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: SUMMARY
- id: v2-router-docs-summary-08-health-statuses
query: "Какие статусы здоровья есть у runtime?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: SUMMARY
- id: v2-router-docs-summary-09-manual-send
query: "Как в приложении устроена ручная отправка сообщения?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: SUMMARY
- id: v2-router-docs-summary-10-telegram-integration
query: "Как сервис взаимодействует с Telegram в целом?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: SUMMARY
- id: v2-router-find-files-01-health
query: "В каком файле описан `/health`?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: FIND_FILES
- id: v2-router-find-files-02-send
query: "Покажи файл с описанием endpoint `/send`"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: FIND_FILES
- id: v2-router-find-files-03-actions
query: "Где находится документация по `/actions/{action}`?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: FIND_FILES
- id: v2-router-find-files-04-runtime-health
query: "В каком документе описан runtime health?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: FIND_FILES
- id: v2-router-find-files-05-loop
query: "Найди файл с логикой цикла отправки уведомлений"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: FIND_FILES
- id: v2-router-find-files-06-architecture
query: "Где описана архитектура приложения?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: FIND_FILES
- id: v2-router-find-files-07-errors-catalog
query: "В каком файле лежит каталог ошибок?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: FIND_FILES
- id: v2-router-find-files-08-docs-index
query: "Покажи файл-индекс документации проекта"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: FIND_FILES
- id: v2-router-find-files-09-telegram-bot-api
query: "Какие файлы относятся к Telegram Bot API интеграции?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: FIND_FILES
- id: v2-router-find-files-10-telegram-notify-worker
query: "В каких файлах описан worker `telegram_notify`?"
expected:
router:
domain: DOCS
intent: DOC_EXPLAIN
sub_intent: FIND_FILES
+1 -1
View File
@@ -10,7 +10,7 @@ _src = _agent_root / "src"
if _src.exists() and str(_src) not in sys.path:
sys.path.insert(0, str(_src))
from app.modules.rag.persistence.repository import RagRepository
from app.core.rag.persistence.repository import RagRepository
from tests.pipeline_setup_v3.shared.env_loader import load_pipeline_setup_env
from tests.pipeline_setup_v3.shared.rag_indexer import RagSessionIndexer
@@ -80,6 +80,7 @@ class CaseDirectoryLoader:
pipeline = dict(raw.get("pipeline") or {})
return CaseExpectations(
router=RouterExpectation(
domain=str(router.get("domain") or "").strip() or None,
intent=str(router.get("intent") or "").strip() or None,
sub_intent=str(router.get("sub_intent") or "").strip() or None,
graph_id=str(router.get("graph_id") or "").strip() or None,
@@ -114,4 +115,6 @@ class CaseDirectoryLoader:
def _normalize_runner(self, value: str) -> str:
if value in {"agent_runtime", "runtime", "code_qa_eval"}:
return "agent_runtime"
if value in {"process_v2", "v2_process", "v2"}:
return "process_v2"
return value
+2 -1
View File
@@ -5,7 +5,7 @@ from pathlib import Path
from typing import Literal
RunnerKind = Literal["agent_runtime"]
RunnerKind = Literal["agent_runtime", "process_v2"]
ModeKind = Literal["router_only", "router_rag", "full_chain"]
@@ -18,6 +18,7 @@ class CaseInput:
@dataclass(slots=True, frozen=True)
class RouterExpectation:
domain: str | None = None
intent: str | None = None
sub_intent: str | None = None
graph_id: str | None = None
+14 -3
View File
@@ -19,10 +19,12 @@ class V3Runner:
*,
pipeline_mode: str = "full",
router_llm_mode: str = "deterministic",
workflow_llm_enabled: bool = True,
) -> None:
self._cases_dir = cases_dir
self._pipeline_mode = pipeline_mode
self._router_llm_mode = router_llm_mode
self._workflow_llm_enabled = workflow_llm_enabled
self._validator = CaseValidator()
self._sessions = RagSessionProvider()
self._agent_runtime = None
@@ -50,9 +52,11 @@ class V3Runner:
return results, self._writer.write_summary(results)
def _execute(self, case, rag_session_id):
if case.runner != "agent_runtime":
raise ValueError(f"Unsupported runner: {case.runner}")
return self._agent_runtime_adapter().execute(case, rag_session_id)
if case.runner == "agent_runtime":
return self._agent_runtime_adapter().execute(case, rag_session_id)
if case.runner == "process_v2":
return self._v2_process_adapter().execute(case, rag_session_id)
raise ValueError(f"Unsupported runner: {case.runner}")
def _agent_runtime_adapter(self):
if self._agent_runtime is None:
@@ -63,3 +67,10 @@ class V3Runner:
router_llm_mode=self._router_llm_mode,
)
return self._agent_runtime
def _v2_process_adapter(self):
if not hasattr(self, "_v2_process"):
from tests.pipeline_setup_v3.runtime.v2_process_adapter import V2ProcessAdapter
self._v2_process = V2ProcessAdapter(workflow_llm_enabled=self._workflow_llm_enabled)
return self._v2_process
@@ -24,7 +24,7 @@ class RagSessionProvider:
return self._cache[key]
def _build_indexer(self):
from app.modules.rag.persistence.repository import RagRepository
from app.core.rag.persistence.repository import RagRepository
from tests.pipeline_setup_v3.shared.rag_indexer import RagSessionIndexer
if self._repository is None:
@@ -6,6 +6,7 @@ from tests.pipeline_setup_v3.core.models import V3Case
class CaseValidator:
def validate(self, case: V3Case, actual: dict, details: dict | None = None) -> list[str]:
mismatches: list[str] = []
self._check(case.expectations.router.domain, actual.get("domain"), "domain", mismatches)
self._check(case.expectations.router.intent, actual.get("intent"), "intent", mismatches)
self._check(case.expectations.router.sub_intent, actual.get("sub_intent"), "sub_intent", mismatches)
self._check(case.expectations.router.graph_id, actual.get("graph_id"), "graph_id", mismatches)
+7
View File
@@ -21,6 +21,12 @@ def main(argv: list[str] | None = None) -> int:
parser.add_argument("--run-name", default="manual_run", help="Output directory prefix")
parser.add_argument("--results-dir", default=None, help="Override results root directory")
parser.add_argument("--pipeline-mode", default="full", choices=["full", "pre_llm_only"], help="Pipeline execution mode")
parser.add_argument(
"--workflow-llm-enabled",
default="true",
choices=["true", "false"],
help="Enable or disable final workflow LLM calls where supported",
)
parser.add_argument(
"--router-llm-mode",
default="deterministic",
@@ -39,6 +45,7 @@ def main(argv: list[str] | None = None) -> int:
run_name=str(ns.run_name).strip() or "manual_run",
pipeline_mode=str(ns.pipeline_mode).strip() or "full",
router_llm_mode=str(ns.router_llm_mode).strip() or "deterministic",
workflow_llm_enabled=str(ns.workflow_llm_enabled).strip().lower() == "true",
)
print(f"Cases dir: {cases_dir}")
print(f"Run dir: {runner.run_dir}")
+4 -3
View File
@@ -1,3 +1,4 @@
from tests.pipeline_setup_v3.runtime.agent_runtime_adapter import AgentRuntimeAdapter
__all__ = ["AgentRuntimeAdapter"]
__all__ = [
"agent_runtime_adapter",
"v2_process_adapter",
]
@@ -2,20 +2,20 @@ from __future__ import annotations
import math
from app.modules.agent.runtime import (
from app.core.agent.runtime import (
AgentRuntimeExecutor,
DocsQAPipelineRunner,
RuntimeRepoContextFactory,
RuntimeRetrievalAdapter,
)
from app.modules.agent.llm import AgentLlmService
from app.modules.agent.llm.prompt_loader import PromptLoader
from app.modules.agent.runtime.steps.context import build_retrieval_request, build_retrieval_result
from app.modules.agent.intent_router_v2 import ConversationState, IntentRouterV2
from app.modules.agent.intent_router_v2.factory import GigaChatIntentRouterFactory
from app.modules.shared.gigachat.client import GigaChatClient
from app.modules.shared.gigachat.settings import GigaChatSettings
from app.modules.shared.gigachat.token_provider import GigaChatTokenProvider
from app.core.agent.llm import AgentLlmService
from app.core.agent.llm.prompt_loader import PromptLoader
from app.core.agent.runtime.steps.context import build_retrieval_request, build_retrieval_result
from app.core.agent.intent_router import ConversationState, IntentRouterV2
from app.core.agent.intent_router.factory import GigaChatIntentRouterFactory
from app.core.shared.gigachat.client import GigaChatClient
from app.core.shared.gigachat.settings import GigaChatSettings
from app.core.shared.gigachat.token_provider import GigaChatTokenProvider
from tests.pipeline_setup_v3.core.models import ExecutionPayload, V3Case
@@ -190,6 +190,7 @@ class AgentRuntimeAdapter:
) -> dict:
route_dump = route.model_dump(mode="json") if route else {}
return {
"domain": route_dump.get("routing_domain") or route_dump.get("domain") or _infer_domain(route_dump.get("intent")),
"intent": route_dump.get("intent"),
"sub_intent": dict(route_dump.get("query_plan") or {}).get("sub_intent"),
"graph_id": route_dump.get("graph_id"),
@@ -262,6 +263,7 @@ class AgentRuntimeAdapter:
entity_candidates = tuple(diagnostics.query_entity_candidates or self._docs_entity_candidates(result.raw_rows))
doc_scope = tuple(diagnostics.doc_ids or self._docs_scope(result.raw_rows))
return {
"domain": getattr(route, "routing_domain", None) or getattr(route, "domain", None) or _infer_domain(route.intent),
"intent": route.intent,
"sub_intent": route.query_plan.sub_intent,
"graph_id": route.graph_id,
@@ -287,6 +289,7 @@ class AgentRuntimeAdapter:
def _stubbed_code_payload(self, case: V3Case, route) -> ExecutionPayload:
actual = {
"domain": _infer_domain(route.intent),
"intent": route.intent,
"sub_intent": route.query_plan.sub_intent,
"graph_id": route.graph_id,
@@ -493,3 +496,12 @@ def _input_step(query: str) -> dict:
"input": {"query": query},
"output": {"query": query},
}
def _infer_domain(intent: str | None) -> str | None:
value = str(intent or "").strip()
if value == "GENERAL_QA":
return "GENERAL"
if value in {"DOCUMENTATION_EXPLAIN", "DOC_EXPLAIN"}:
return "DOCS"
return None
@@ -0,0 +1,420 @@
from __future__ import annotations
import asyncio
from dataclasses import asdict, dataclass
from pathlib import Path
from app.core.agent.processes.v2.anchor_signals import route_anchor_summary
from app.core.agent.processes.v2 import V2IntentRouter
from app.core.agent.processes.v2.evidence.assembler import DocsEvidenceAssembler
from app.core.agent.processes.v2.evidence.gate import DocsEvidenceGate
from app.core.agent.processes.v2.models import RetrievedFile, RetrievedSummary, V2Intent, V2Subintent
from app.core.agent.processes.v2.retrieval import DocsMetadataLookupIndex
from app.core.agent.processes.v2.retrieval.policy_resolver import V2RetrievalPolicyResolver
from app.core.agent.processes.v2.retrieval.v2_rag_adapter import V2RagRetrievalAdapter
from app.core.agent.processes.v2.workflows.docs_explain_find_files.context import DocsExplainFindFilesContext
from app.core.agent.processes.v2.workflows.docs_explain_find_files.graph import DocsExplainFindFilesGraph
from app.core.agent.processes.v2.workflows.docs_explain_summary.context import DocsExplainSummaryContext
from app.core.agent.processes.v2.workflows.docs_explain_summary.graph import DocsExplainSummaryGraph
from app.core.agent.processes.v2.workflows.general_summary.context import GeneralSummaryContext
from app.core.agent.processes.v2.workflows.general_summary.graph import GeneralSummaryGraph
from app.core.agent.utils.llm import AgentLlmService, PromptLoader
from app.core.rag.embedding.gigachat_embedder import GigaChatEmbedder
from app.core.rag.persistence import RagRepository
from app.core.rag.retrieval.session_retriever import RagSessionRetriever
from app.core.shared.gigachat.client import GigaChatClient
from app.core.shared.gigachat.settings import GigaChatSettings
from app.core.shared.gigachat.token_provider import GigaChatTokenProvider
from app.infra.observability.module_trace import RequestTraceContext
from tests.pipeline_setup_v3.core.models import ExecutionPayload, V3Case
class V2ProcessAdapter:
def __init__(self, *, workflow_llm_enabled: bool = True) -> None:
self._workflow_llm_enabled = workflow_llm_enabled
self._router = V2IntentRouter(llm=_build_v2_llm())
self._policy = V2RetrievalPolicyResolver()
retriever = RagSessionRetriever(repository=RagRepository(), embedder=GigaChatEmbedder(_build_client()))
self._retrieval = V2RagRetrievalAdapter(retriever)
self._evidence = DocsEvidenceAssembler()
self._gate = DocsEvidenceGate()
self._summary_graph = DocsExplainSummaryGraph(_build_v2_llm())
self._find_files_graph = DocsExplainFindFilesGraph()
self._general_graph = GeneralSummaryGraph(_build_v2_llm())
def execute(self, case: V3Case, rag_session_id: str | None) -> ExecutionPayload:
return asyncio.run(self._execute_async(case, rag_session_id))
async def _execute_async(self, case: V3Case, rag_session_id: str | None) -> ExecutionPayload:
runtime = _RuntimeStub(query=case.query)
route = self._router.route(case.query)
_log_pipeline_step(
runtime,
"router_resolved",
{
"domain": route.routing_domain,
"intent": route.intent,
"subintent": route.subintent,
"confidence": route.confidence,
},
)
_log_pipeline_step(
runtime,
"anchors_extracted",
{
"signal_types": route_anchor_summary(route)["signal_types"],
"endpoint_paths": route.anchors.endpoint_paths,
"target_doc_hints": route.anchors.target_doc_hints,
"matched_aliases": route.anchors.matched_aliases,
},
)
_log_pipeline_step(
runtime,
"alias_resolution",
{
"resolved_aliases": route.anchors.matched_aliases,
"target_doc_hints": route.anchors.target_doc_hints,
},
)
if case.mode == "router_only":
return ExecutionPayload(
actual=_actual_from_v2(route),
details=_details(case.query, route=route, pipeline_steps=_build_pipeline_steps(runtime.logs)),
)
plan = self._policy.resolve(route)
_log_pipeline_step(
runtime,
"retrieval_profile_selected",
{"profile": plan.profile, "layers": plan.layers, "filters": plan.filters},
)
semantic_rows = await self._retrieve_rows(route, rag_session_id, plan)
seeded_rows = await self._seed_candidates_from_target_hints(route, rag_session_id, plan)
metadata_rows = self._metadata_lookup_candidates([*seeded_rows, *semantic_rows], route)
rows = self._merge_candidate_rows(seeded_rows, metadata_rows, semantic_rows)
_log_pipeline_step(
runtime,
"candidate_generation",
{
"resolved_aliases": route.anchors.matched_aliases,
"target_doc_hints": route.anchors.target_doc_hints,
"candidate_docs_before_ranking": [self._trace_row(row) for row in rows[:8]],
"sources": {
"seeded": [self._trace_row(row) for row in seeded_rows[:5]],
"metadata_lookup": [self._trace_row(row) for row in metadata_rows[:5]],
"semantic": [self._trace_row(row) for row in semantic_rows[:5]],
},
},
)
_log_pipeline_step(
runtime,
"retrieval_executed",
{
"query": case.query,
"profile": plan.profile,
"row_count": len(rows),
"target_doc_hints": route.anchors.target_doc_hints,
"top_results": [self._trace_row(row) for row in rows[:5]],
},
)
if case.mode == "router_rag":
return ExecutionPayload(
actual=_actual_from_v2(route, rows=rows, plan=plan, answer_mode="partial"),
details=_details(case.query, route=route, plan=plan, rows=rows, pipeline_steps=_build_pipeline_steps(runtime.logs)),
)
answer, evidence, gate = await self._run_workflow(runtime, route, rag_session_id, rows)
answer_mode = gate.answer_mode
_log_pipeline_step(
runtime,
"answer_generated",
{"answer_mode": answer_mode, "answer_length": len(answer)},
)
return ExecutionPayload(
actual=_actual_from_v2(route, rows=rows, plan=plan, answer=answer, answer_mode=answer_mode),
details=_details(
case.query,
route=route,
plan=plan,
rows=rows,
evidence=evidence,
answer=answer,
logs=runtime.logs,
pipeline_steps=_build_pipeline_steps(runtime.logs),
),
)
async def _retrieve_rows(self, route, rag_session_id: str | None, plan) -> list[dict]:
if not rag_session_id:
if route.intent == V2Intent.GENERAL_QA:
return []
raise ValueError("process_v2 cases with DOCS intent require rag_session_id")
return await self._retrieval.fetch_rows(rag_session_id, route.normalized_query, plan)
async def _seed_candidates_from_target_hints(self, route, rag_session_id: str | None, plan) -> list[dict]:
if not rag_session_id or not route.anchors.target_doc_hints:
return []
return await self._retrieval.fetch_exact_paths(rag_session_id, paths=route.anchors.target_doc_hints, layers=plan.layers)
def _metadata_lookup_candidates(self, rows: list[dict], route) -> list[dict]:
return DocsMetadataLookupIndex(rows).lookup(route)
def _merge_candidate_rows(self, *groups: list[dict]) -> list[dict]:
merged: list[dict] = []
seen: set[tuple[str, str, str]] = set()
for rows in groups:
for row in rows:
key = (
str(row.get("path") or ""),
str(row.get("layer") or ""),
str(dict(row.get("metadata") or {}).get("section_path") or ""),
)
if key in seen:
continue
seen.add(key)
merged.append(row)
return merged
async def _run_workflow(
self,
runtime: "_RuntimeStub",
route,
rag_session_id: str | None,
rows: list[dict],
) -> tuple[str, dict, object]:
if route.intent == V2Intent.GENERAL_QA:
documents = self._evidence.assemble_summaries(rows, route)
gate = self._gate.check_summaries(route, documents)
_log_pipeline_step(
runtime,
"evidence_assembled",
{"mode": "summary", "primary_doc": documents[0].path if documents else None, "document_count": len(documents)},
)
self._log_ranking(runtime, documents)
_log_pipeline_step(
runtime,
"evidence_gate_checked",
{"passed": gate.passed, "reason": gate.reason, "answer_mode": gate.answer_mode},
)
context = GeneralSummaryContext(runtime=runtime, route=route, prompt_name="v2_general.summary_answer")
context.workflow_llm_enabled = self._workflow_llm_enabled
context.documents = documents
context.gate_decision = gate
final = await self._general_graph.run(context)
return final.answer, {"documents": [_serialize_summary(item) for item in documents], "files": []}, gate
if route.subintent == V2Subintent.FIND_FILES:
files = self._evidence.assemble_files(rows, route)
gate = self._gate.check_files(route, files)
_log_pipeline_step(
runtime,
"evidence_assembled",
{"mode": "find_files", "primary_file": files[0].path if files else None, "file_count": len(files)},
)
self._log_ranking(runtime, files)
_log_pipeline_step(
runtime,
"evidence_gate_checked",
{"passed": gate.passed, "reason": gate.reason, "answer_mode": gate.answer_mode},
)
context = DocsExplainFindFilesContext(
runtime=runtime,
route=route,
rag_session_id=rag_session_id or "",
files=files,
gate_decision=gate,
)
final = await self._find_files_graph.run(context)
return final.answer, {"documents": [], "files": [_serialize_file(item) for item in files]}, gate
documents = self._evidence.assemble_summaries(rows, route)
gate = self._gate.check_summaries(route, documents)
_log_pipeline_step(
runtime,
"evidence_assembled",
{"mode": "summary", "primary_doc": documents[0].path if documents else None, "document_count": len(documents)},
)
self._log_ranking(runtime, documents)
_log_pipeline_step(
runtime,
"evidence_gate_checked",
{"passed": gate.passed, "reason": gate.reason, "answer_mode": gate.answer_mode},
)
context = DocsExplainSummaryContext(
runtime=runtime,
route=route,
rag_session_id=rag_session_id or "",
prompt_name="v2_docs_explain.summary_answer",
workflow_llm_enabled=self._workflow_llm_enabled,
documents=documents,
gate_decision=gate,
)
final = await self._summary_graph.run(context)
return final.answer, {"documents": [_serialize_summary(item) for item in documents], "files": []}, gate
def _trace_row(self, row: dict) -> dict[str, object]:
metadata = dict(row.get("metadata") or {})
return {
"path": str(row.get("path") or ""),
"layer": str(row.get("layer") or ""),
"title": str(row.get("title") or ""),
"document_id": str(metadata.get("document_id") or metadata.get("doc_id") or ""),
}
def _log_ranking(self, runtime: "_RuntimeStub", items: list) -> None:
top_docs: list[dict[str, object]] = []
for item in items[:4]:
top_docs.append(
{
"doc": getattr(item, "path", ""),
"score": getattr(item, "score", 0),
"match_reason": getattr(item, "match_reason", ""),
}
)
_log_pipeline_step(
runtime,
"ranking_explained",
{
"doc": getattr(item, "path", ""),
"score": getattr(item, "score", 0),
"score_breakdown": getattr(item, "score_breakdown", {}),
"match_reason": getattr(item, "match_reason", ""),
},
)
_log_pipeline_step(
runtime,
"ranking_explained",
{
"top_docs_after_ranking": top_docs,
"ranking_score_breakdown": [
{
"doc": getattr(item, "path", ""),
"score_breakdown": getattr(item, "score_breakdown", {}),
}
for item in items[:4]
],
},
)
@dataclass(slots=True)
class _RequestStub:
request_id: str
message: str
@dataclass(slots=True)
class _SessionStub:
active_rag_session_id: str | None = None
class _PublisherStub:
async def publish_status(self, request_id: str, source: str, message: str, payload: dict | None = None) -> None:
return None
class _TraceLoggerStub:
def __init__(self, store: list[dict]) -> None:
self._store = store
def log_module(self, request_id: str, module: str, title: str, payload: dict | None = None) -> None:
self._store.append(
{"request_id": request_id, "module": module, "event": title, "payload": dict(payload or {})}
)
class _RuntimeStub:
def __init__(self, *, query: str) -> None:
self.logs: list[dict] = []
self.request = _RequestStub(request_id="pipeline_setup_v3", message=query)
self.session = _SessionStub()
self.publisher = _PublisherStub()
self.trace = RequestTraceContext(request_id=self.request.request_id, logger=_TraceLoggerStub(self.logs))
def _build_client() -> GigaChatClient:
settings = GigaChatSettings.from_env()
return GigaChatClient(settings, GigaChatTokenProvider(settings))
def _build_v2_llm() -> AgentLlmService:
prompt_paths = [
Path(__file__).resolve().parents[3] / "src/app/core/agent/processes/v2/prompts.yml",
Path(__file__).resolve().parents[3] / "src/app/core/agent/processes/v2/general_prompts.yml",
Path(__file__).resolve().parents[3] / "src/app/core/agent/processes/v2/intent_router/routers/prompts.yml",
]
return AgentLlmService(client=_build_client(), prompts=PromptLoader(prompt_paths))
def _actual_from_v2(route, *, rows: list[dict] | None = None, plan=None, answer: str = "", answer_mode: str = "partial") -> dict:
return {
"domain": route.routing_domain,
"intent": route.intent,
"sub_intent": route.subintent,
"rag_count": len(rows or []),
"llm_answer": answer,
"answer_mode": answer_mode,
"path_scope": tuple(),
"symbol_candidates": tuple(),
"entity_candidates": tuple(_entity_candidates(rows or [])),
"doc_scope": tuple(_doc_scope(rows or [])),
"layers": tuple(getattr(plan, "layers", []) or []),
"filters": dict(getattr(plan, "filters", {}) or {}),
}
def _details(query: str, **payload) -> dict:
details = {"query": query}
for key, value in payload.items():
if key == "route":
details["router_result"] = asdict(value)
elif key == "plan":
details["retrieval_plan"] = asdict(value)
else:
details[key] = value
return details
def _doc_scope(rows: list[dict]) -> list[str]:
values: list[str] = []
for row in rows:
metadata = dict(row.get("metadata") or {})
for candidate in (metadata.get("document_id"), metadata.get("doc_id"), row.get("path")):
value = str(candidate or "").strip()
if value and value not in values:
values.append(value)
return values
def _entity_candidates(rows: list[dict]) -> list[str]:
values: list[str] = []
for row in rows:
metadata = dict(row.get("metadata") or {})
value = str(metadata.get("entity_name") or row.get("title") or "").strip()
if value and value not in values and str(row.get("layer") or "") == "D3_ENTITY_CATALOG":
values.append(value)
return values
def _serialize_summary(item: RetrievedSummary) -> dict:
return asdict(item)
def _serialize_file(item: RetrievedFile) -> dict:
return asdict(item)
def _build_pipeline_steps(logs: list[dict]) -> list[dict]:
steps: list[dict] = []
for item in logs:
if item.get("module") != "process.v2.pipeline":
continue
steps.append({"step": item.get("event"), "output": item.get("payload") or {}})
return steps
def _log_pipeline_step(runtime: _RuntimeStub, step: str, payload: dict[str, object]) -> None:
runtime.logs.append(
{
"request_id": runtime.request.request_id,
"module": "process.v2.pipeline",
"event": step,
"payload": payload,
}
)
+1 -1
View File
@@ -3,7 +3,7 @@ from __future__ import annotations
import os
from pathlib import Path
from app.modules.shared.env_loader import load_workspace_env
from app.core.shared.config import load_workspace_env
def load_pipeline_setup_env(start_dir: str | Path | None = None) -> list[Path]:
@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING
from uuid import uuid4
if TYPE_CHECKING:
from app.modules.rag.persistence.repository import RagRepository
from app.core.rag.persistence.repository import RagRepository
class LocalRepoFileCollector:
@@ -77,7 +77,7 @@ class DeterministicEmbedder:
class RagSessionIndexer:
def __init__(self, repository: "RagRepository") -> None:
from app.modules.rag.services.rag_service import RagService
from app.core.rag.indexing.service import RagService
self._repository = repository
self._rag = RagService(embedder=DeterministicEmbedder(), repository=repository)
@@ -90,7 +90,7 @@ class RagSessionIndexer:
files = LocalRepoFileCollector(repo_path).collect()
if not files:
raise ValueError(f"No indexable text files found under: {repo_path}")
logger = logging.getLogger("app.modules.rag.services.rag_service")
logger = logging.getLogger("app.core.rag.indexing.service")
previous_level = logger.level
logger.setLevel(logging.ERROR)
try:
@@ -0,0 +1,43 @@
{
"meta": {
"case_id": "v2-docs-summary-01-health",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"llm: expected answer to contain 'status'",
"llm: expected answer to contain 'components'"
],
"actual": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"rag_count": 12,
"llm_answer": "Информация о том, как именно работает `/health`, не представлена в предоставленных SUMMARY-блоках. \n\nФайлы, на которые я опирался:\n- docs/architecture/telegram-notify-app-overview.md",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramNotifyModule",
"TelegramNotifyWorker",
"RuntimeManager",
"TelegramSendService",
"TelegramControlChannel"
],
"doc_scope": [
"index.test_echo_app_docs",
"docs/README.md",
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,51 @@
# v2-docs-summary-01-health
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Кратко объясни по документации, как работает `/health`
## Actual
{
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"rag_count": 12,
"llm_answer": "Информация о том, как именно работает `/health`, не представлена в предоставленных SUMMARY-блоках. \n\nФайлы, на которые я опирался:\n- docs/architecture/telegram-notify-app-overview.md",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramNotifyModule",
"TelegramNotifyWorker",
"RuntimeManager",
"TelegramSendService",
"TelegramControlChannel"
],
"doc_scope": [
"index.test_echo_app_docs",
"docs/README.md",
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- llm: expected answer to contain 'status'
- llm: expected answer to contain 'components'
@@ -0,0 +1,43 @@
{
"meta": {
"case_id": "v2-docs-summary-02-send",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']",
"llm: expected answer to contain 'message'"
],
"actual": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"rag_count": 12,
"llm_answer": "На основе представленного SUMMARY-блока, документация о том, что именно делает endpoint `/send`, отсутствует. В контексте указано, что он связан с отправкой сообщений, но подробностей его работы не приведено.\n\nФайлы, на которые опирался ответ:\n- docs/architecture/telegram-notify-app-overview.md",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramSendService",
"TelegramNotifyModule",
"TelegramControlChannel",
"TelegramNotifyWorker",
"RuntimeManager"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,51 @@
# v2-docs-summary-02-send
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Что делает endpoint `/send`?
## Actual
{
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"rag_count": 12,
"llm_answer": "На основе представленного SUMMARY-блока, документация о том, что именно делает endpoint `/send`, отсутствует. В контексте указано, что он связан с отправкой сообщений, но подробностей его работы не приведено.\n\nФайлы, на которые опирался ответ:\n- docs/architecture/telegram-notify-app-overview.md",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramSendService",
"TelegramNotifyModule",
"TelegramControlChannel",
"TelegramNotifyWorker",
"RuntimeManager"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
- llm: expected answer to contain 'message'
@@ -0,0 +1,30 @@
{
"meta": {
"case_id": "v2-docs-summary-03-actions",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"domain: expected DOCS, got GENERAL",
"intent: expected DOC_EXPLAIN, got GENERAL_QA",
"retrieval: expected non-empty rag, got 0 rows",
"doc_scope should contain 'docs/api/control-actions-endpoint.md', got []"
],
"actual": {
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "`/actions/{action}` обычно используется в API для выполнения конкретных действий или операций, где `{action}` заменяется названием нужного действия. Это позволяет организовывать и вызывать различные функции через единый интерфейс.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,38 @@
# v2-docs-summary-03-actions
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Объясни назначение `/actions/{action}`
## Actual
{
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "`/actions/{action}` обычно используется в API для выполнения конкретных действий или операций, где `{action}` заменяется названием нужного действия. Это позволяет организовывать и вызывать различные функции через единый интерфейс.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- domain: expected DOCS, got GENERAL
- intent: expected DOC_EXPLAIN, got GENERAL_QA
- retrieval: expected non-empty rag, got 0 rows
- doc_scope should contain 'docs/api/control-actions-endpoint.md', got []
@@ -0,0 +1,30 @@
{
"meta": {
"case_id": "v2-docs-summary-04-runtime-health",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"domain: expected DOCS, got GENERAL",
"intent: expected DOC_EXPLAIN, got GENERAL_QA",
"retrieval: expected non-empty rag, got 0 rows",
"doc_scope should contain 'docs/domains/runtime-health-entity.md', got []"
],
"actual": {
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Runtime health — это процесс мониторинга и обеспечения стабильности работы системы или приложения во время их выполнения (runtime), позволяющий своевременно обнаруживать и устранять возникающие проблемы.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,38 @@
# v2-docs-summary-04-runtime-health
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Что такое runtime health в этой документации?
## Actual
{
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Runtime health — это процесс мониторинга и обеспечения стабильности работы системы или приложения во время их выполнения (runtime), позволяющий своевременно обнаруживать и устранять возникающие проблемы.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- domain: expected DOCS, got GENERAL
- intent: expected DOC_EXPLAIN, got GENERAL_QA
- retrieval: expected non-empty rag, got 0 rows
- doc_scope should contain 'docs/domains/runtime-health-entity.md', got []
@@ -0,0 +1,33 @@
{
"meta": {
"case_id": "v2-docs-summary-05-notification-loop",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"domain: expected DOCS, got GENERAL",
"intent: expected DOC_EXPLAIN, got GENERAL_QA",
"retrieval: expected non-empty rag, got 0 rows",
"doc_scope should contain 'docs/logic/telegram-notification-loop.md', got []",
"llm: expected answer to contain 'worker'",
"llm: expected answer to contain 'interval'",
"llm: expected answer to contain 'degraded'"
],
"actual": {
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Цикл отправки уведомлений включает следующие шаги: отлавливание события, обработка данных, выбор получателей, формирование сообщения и его отправка выбранным пользователям.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,41 @@
# v2-docs-summary-05-notification-loop
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Кратко опиши цикл отправки уведомлений
## Actual
{
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Цикл отправки уведомлений включает следующие шаги: отлавливание события, обработка данных, выбор получателей, формирование сообщения и его отправка выбранным пользователям.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- domain: expected DOCS, got GENERAL
- intent: expected DOC_EXPLAIN, got GENERAL_QA
- retrieval: expected non-empty rag, got 0 rows
- doc_scope should contain 'docs/logic/telegram-notification-loop.md', got []
- llm: expected answer to contain 'worker'
- llm: expected answer to contain 'interval'
- llm: expected answer to contain 'degraded'
@@ -0,0 +1,32 @@
{
"meta": {
"case_id": "v2-docs-summary-06-architecture",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"domain: expected DOCS, got GENERAL",
"intent: expected DOC_EXPLAIN, got GENERAL_QA",
"retrieval: expected non-empty rag, got 0 rows",
"doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []",
"llm: expected answer to contain 'RuntimeManager'",
"llm: expected answer to contain 'TelegramControlChannel'"
],
"actual": {
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Архитектура Telegram Notify App основана на клиент-серверной модели, где серверное приложение обрабатывает входящие данные и отправляет уведомления через Telegram API. Клиентское приложение взаимодействует с сервером для отправки уведомлений. Пользователь настраивает параметры уведомлений в приложении, а сервер передает их в виде сообщений в указанный Telegram-канал или пользователю.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,40 @@
# v2-docs-summary-06-architecture
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Как устроена архитектура Telegram Notify App?
## Actual
{
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Архитектура Telegram Notify App основана на клиент-серверной модели, где серверное приложение обрабатывает входящие данные и отправляет уведомления через Telegram API. Клиентское приложение взаимодействует с сервером для отправки уведомлений. Пользователь настраивает параметры уведомлений в приложении, а сервер передает их в виде сообщений в указанный Telegram-канал или пользователю.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- domain: expected DOCS, got GENERAL
- intent: expected DOC_EXPLAIN, got GENERAL_QA
- retrieval: expected non-empty rag, got 0 rows
- doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []
- llm: expected answer to contain 'RuntimeManager'
- llm: expected answer to contain 'TelegramControlChannel'
@@ -0,0 +1,32 @@
{
"meta": {
"case_id": "v2-docs-summary-07-worker",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"domain: expected DOCS, got GENERAL",
"intent: expected DOC_EXPLAIN, got GENERAL_QA",
"retrieval: expected non-empty rag, got 0 rows",
"doc_scope should contain 'docs/logic/telegram-notification-loop.md', got []",
"doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []",
"llm: expected answer to contain 'Telegram'"
],
"actual": {
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Worker в этом приложении выполняет фоновые задачи или процессы, позволяя основному приложению работать быстро и плавно. Это может включать обработку данных, отправку уведомлений, загрузку файлов и другие ресурсоемкие операции.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,40 @@
# v2-docs-summary-07-worker
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Что делает worker в этом приложении?
## Actual
{
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Worker в этом приложении выполняет фоновые задачи или процессы, позволяя основному приложению работать быстро и плавно. Это может включать обработку данных, отправку уведомлений, загрузку файлов и другие ресурсоемкие операции.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- domain: expected DOCS, got GENERAL
- intent: expected DOC_EXPLAIN, got GENERAL_QA
- retrieval: expected non-empty rag, got 0 rows
- doc_scope should contain 'docs/logic/telegram-notification-loop.md', got []
- doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []
- llm: expected answer to contain 'Telegram'
@@ -0,0 +1,47 @@
{
"meta": {
"case_id": "v2-docs-summary-08-health-statuses",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"doc_scope should contain 'docs/domains/runtime-health-entity.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']",
"doc_scope should contain 'docs/api/health-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']",
"llm: expected answer to contain 'ok'",
"llm: expected answer to contain 'degraded'",
"llm: expected answer to contain 'unhealthy'",
"llm: expected answer to contain 'unknown'"
],
"actual": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"rag_count": 12,
"llm_answer": "На основе представленного SUMMARY-блока конкретной информации о статусах здоровья runtime нет. Файл, связанный с доменом \"runtime health\" (../domains/runtime-health-entity.md), вероятно, содержит эту информацию, но она не включена в текущий контекст.\n\nФайлы, на которые опирался ответ:\n- docs/architecture/telegram-notify-app-overview.md",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"RuntimeManager",
"TelegramNotifyWorker",
"TelegramSendService",
"TelegramNotifyModule",
"TelegramControlChannel"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,55 @@
# v2-docs-summary-08-health-statuses
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Какие статусы здоровья есть у runtime?
## Actual
{
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"rag_count": 12,
"llm_answer": "На основе представленного SUMMARY-блока конкретной информации о статусах здоровья runtime нет. Файл, связанный с доменом \"runtime health\" (../domains/runtime-health-entity.md), вероятно, содержит эту информацию, но она не включена в текущий контекст.\n\nФайлы, на которые опирался ответ:\n- docs/architecture/telegram-notify-app-overview.md",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"RuntimeManager",
"TelegramNotifyWorker",
"TelegramSendService",
"TelegramNotifyModule",
"TelegramControlChannel"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- doc_scope should contain 'docs/domains/runtime-health-entity.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
- doc_scope should contain 'docs/api/health-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
- llm: expected answer to contain 'ok'
- llm: expected answer to contain 'degraded'
- llm: expected answer to contain 'unhealthy'
- llm: expected answer to contain 'unknown'
@@ -0,0 +1,33 @@
{
"meta": {
"case_id": "v2-docs-summary-09-manual-send",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"domain: expected DOCS, got GENERAL",
"intent: expected DOC_EXPLAIN, got GENERAL_QA",
"retrieval: expected non-empty rag, got 0 rows",
"doc_scope should contain 'docs/api/send-message-endpoint.md', got []",
"llm: expected answer to contain '/send'",
"llm: expected answer to contain 'message'",
"llm: expected answer to contain 'TelegramSendService'"
],
"actual": {
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Ручная отправка сообщения в приложении обычно осуществляется нажатием кнопки \"Отправить\" или аналогичного элемента интерфейса после того, как пользователь введет текст сообщения.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,41 @@
# v2-docs-summary-09-manual-send
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Как в приложении устроена ручная отправка сообщения?
## Actual
{
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Ручная отправка сообщения в приложении обычно осуществляется нажатием кнопки \"Отправить\" или аналогичного элемента интерфейса после того, как пользователь введет текст сообщения.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- domain: expected DOCS, got GENERAL
- intent: expected DOC_EXPLAIN, got GENERAL_QA
- retrieval: expected non-empty rag, got 0 rows
- doc_scope should contain 'docs/api/send-message-endpoint.md', got []
- llm: expected answer to contain '/send'
- llm: expected answer to contain 'message'
- llm: expected answer to contain 'TelegramSendService'
@@ -0,0 +1,34 @@
{
"meta": {
"case_id": "v2-docs-summary-10-telegram-integration",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"domain: expected DOCS, got GENERAL",
"intent: expected DOC_EXPLAIN, got GENERAL_QA",
"retrieval: expected non-empty rag, got 0 rows",
"retrieval: expected at least 2 rows, got 0",
"doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []",
"doc_scope should contain 'docs/api/send-message-endpoint.md', got []",
"llm: expected answer to contain 'worker'",
"llm: expected answer to contain '/send'"
],
"actual": {
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Сервис может отправлять и получать сообщения, реагировать на команды и запросы пользователей через Telegram-бота. Взаимодействие происходит посредством отправки запросов и получения ответов от бота в чате.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,42 @@
# v2-docs-summary-10-telegram-integration
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Как сервис взаимодействует с Telegram в целом?
## Actual
{
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Сервис может отправлять и получать сообщения, реагировать на команды и запросы пользователей через Telegram-бота. Взаимодействие происходит посредством отправки запросов и получения ответов от бота в чате.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- domain: expected DOCS, got GENERAL
- intent: expected DOC_EXPLAIN, got GENERAL_QA
- retrieval: expected non-empty rag, got 0 rows
- retrieval: expected at least 2 rows, got 0
- doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []
- doc_scope should contain 'docs/api/send-message-endpoint.md', got []
- llm: expected answer to contain 'worker'
- llm: expected answer to contain '/send'
@@ -0,0 +1,42 @@
{
"meta": {
"case_id": "v2-find-files-01-health",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"llm: expected answer to contain 'docs/api/health-endpoint.md'"
],
"actual": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "FIND_FILES",
"rag_count": 20,
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
"answer_mode": "deterministic",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramNotifyModule",
"RuntimeManager",
"TelegramControlChannel",
"TelegramNotifyWorker",
"TelegramSendService"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D3_ENTITY_CATALOG",
"D1_DOCUMENT_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,50 @@
# v2-find-files-01-health
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
В каком файле описан `/health`?
## Actual
{
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "FIND_FILES",
"rag_count": 20,
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
"answer_mode": "deterministic",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramNotifyModule",
"RuntimeManager",
"TelegramControlChannel",
"TelegramNotifyWorker",
"TelegramSendService"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D3_ENTITY_CATALOG",
"D1_DOCUMENT_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- llm: expected answer to contain 'docs/api/health-endpoint.md'
@@ -0,0 +1,42 @@
{
"meta": {
"case_id": "v2-find-files-02-send",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"llm: expected answer to contain 'docs/api/send-message-endpoint.md'"
],
"actual": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "FIND_FILES",
"rag_count": 20,
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
"answer_mode": "deterministic",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramSendService",
"TelegramNotifyModule",
"TelegramNotifyWorker",
"TelegramControlChannel",
"RuntimeManager"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D3_ENTITY_CATALOG",
"D1_DOCUMENT_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,50 @@
# v2-find-files-02-send
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Покажи файл с описанием endpoint `/send`
## Actual
{
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "FIND_FILES",
"rag_count": 20,
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
"answer_mode": "deterministic",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramSendService",
"TelegramNotifyModule",
"TelegramNotifyWorker",
"TelegramControlChannel",
"RuntimeManager"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D3_ENTITY_CATALOG",
"D1_DOCUMENT_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- llm: expected answer to contain 'docs/api/send-message-endpoint.md'
@@ -0,0 +1,42 @@
{
"meta": {
"case_id": "v2-find-files-03-actions",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"llm: expected answer to contain 'docs/api/control-actions-endpoint.md'"
],
"actual": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "FIND_FILES",
"rag_count": 20,
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
"answer_mode": "deterministic",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"RuntimeManager",
"TelegramNotifyModule",
"TelegramControlChannel",
"TelegramNotifyWorker",
"TelegramSendService"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D3_ENTITY_CATALOG",
"D1_DOCUMENT_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,50 @@
# v2-find-files-03-actions
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Где находится документация по `/actions/{action}`?
## Actual
{
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "FIND_FILES",
"rag_count": 20,
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
"answer_mode": "deterministic",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"RuntimeManager",
"TelegramNotifyModule",
"TelegramControlChannel",
"TelegramNotifyWorker",
"TelegramSendService"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D3_ENTITY_CATALOG",
"D1_DOCUMENT_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- llm: expected answer to contain 'docs/api/control-actions-endpoint.md'
@@ -0,0 +1,42 @@
{
"meta": {
"case_id": "v2-find-files-04-runtime-health",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"llm: expected answer to contain 'docs/domains/runtime-health-entity.md'"
],
"actual": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "FIND_FILES",
"rag_count": 20,
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/architecture/telegram-notify-app-overview.md`\n- `docs/README.md`",
"answer_mode": "deterministic",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"RuntimeManager",
"TelegramNotifyWorker",
"TelegramNotifyModule",
"TelegramControlChannel",
"TelegramSendService"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D3_ENTITY_CATALOG",
"D1_DOCUMENT_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,50 @@
# v2-find-files-04-runtime-health
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
В каком документе описан runtime health?
## Actual
{
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "FIND_FILES",
"rag_count": 20,
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/architecture/telegram-notify-app-overview.md`\n- `docs/README.md`",
"answer_mode": "deterministic",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"RuntimeManager",
"TelegramNotifyWorker",
"TelegramNotifyModule",
"TelegramControlChannel",
"TelegramSendService"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D3_ENTITY_CATALOG",
"D1_DOCUMENT_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- llm: expected answer to contain 'docs/domains/runtime-health-entity.md'
@@ -0,0 +1,42 @@
{
"meta": {
"case_id": "v2-find-files-05-loop",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'"
],
"actual": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "FIND_FILES",
"rag_count": 20,
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
"answer_mode": "deterministic",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramNotifyModule",
"TelegramNotifyWorker",
"TelegramSendService",
"RuntimeManager",
"TelegramControlChannel"
],
"doc_scope": [
"index.test_echo_app_docs",
"docs/README.md",
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md"
],
"layers": [
"D3_ENTITY_CATALOG",
"D1_DOCUMENT_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,50 @@
# v2-find-files-05-loop
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Найди файл с логикой цикла отправки уведомлений
## Actual
{
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "FIND_FILES",
"rag_count": 20,
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
"answer_mode": "deterministic",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramNotifyModule",
"TelegramNotifyWorker",
"TelegramSendService",
"RuntimeManager",
"TelegramControlChannel"
],
"doc_scope": [
"index.test_echo_app_docs",
"docs/README.md",
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md"
],
"layers": [
"D3_ENTITY_CATALOG",
"D1_DOCUMENT_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'
@@ -0,0 +1,40 @@
{
"meta": {
"case_id": "v2-find-files-06-architecture",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": true,
"mismatches": [],
"actual": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "FIND_FILES",
"rag_count": 20,
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
"answer_mode": "deterministic",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"RuntimeManager",
"TelegramSendService",
"TelegramNotifyModule",
"TelegramControlChannel",
"TelegramNotifyWorker"
],
"doc_scope": [
"index.test_echo_app_docs",
"docs/README.md",
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md"
],
"layers": [
"D3_ENTITY_CATALOG",
"D1_DOCUMENT_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,50 @@
# v2-find-files-06-architecture
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: True
## Query
Где описана архитектура приложения?
## Actual
{
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "FIND_FILES",
"rag_count": 20,
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
"answer_mode": "deterministic",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"RuntimeManager",
"TelegramSendService",
"TelegramNotifyModule",
"TelegramControlChannel",
"TelegramNotifyWorker"
],
"doc_scope": [
"index.test_echo_app_docs",
"docs/README.md",
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md"
],
"layers": [
"D3_ENTITY_CATALOG",
"D1_DOCUMENT_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- none
@@ -0,0 +1,42 @@
{
"meta": {
"case_id": "v2-find-files-07-errors-catalog",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"llm: expected answer to contain 'docs/errors/catalog.yaml'"
],
"actual": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "FIND_FILES",
"rag_count": 20,
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`",
"answer_mode": "deterministic",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"RuntimeManager",
"TelegramControlChannel",
"TelegramNotifyModule",
"TelegramSendService",
"TelegramNotifyWorker"
],
"doc_scope": [
"index.test_echo_app_docs",
"docs/README.md",
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md"
],
"layers": [
"D3_ENTITY_CATALOG",
"D1_DOCUMENT_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,50 @@
# v2-find-files-07-errors-catalog
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
В каком файле лежит каталог ошибок?
## Actual
{
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "FIND_FILES",
"rag_count": 20,
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`",
"answer_mode": "deterministic",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"RuntimeManager",
"TelegramControlChannel",
"TelegramNotifyModule",
"TelegramSendService",
"TelegramNotifyWorker"
],
"doc_scope": [
"index.test_echo_app_docs",
"docs/README.md",
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md"
],
"layers": [
"D3_ENTITY_CATALOG",
"D1_DOCUMENT_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- llm: expected answer to contain 'docs/errors/catalog.yaml'
@@ -0,0 +1,40 @@
{
"meta": {
"case_id": "v2-find-files-08-docs-index",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": true,
"mismatches": [],
"actual": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "FIND_FILES",
"rag_count": 20,
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`",
"answer_mode": "deterministic",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramNotifyModule",
"TelegramNotifyWorker",
"TelegramControlChannel",
"RuntimeManager",
"TelegramSendService"
],
"doc_scope": [
"index.test_echo_app_docs",
"docs/README.md",
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md"
],
"layers": [
"D3_ENTITY_CATALOG",
"D1_DOCUMENT_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,50 @@
# v2-find-files-08-docs-index
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: True
## Query
Покажи файл-индекс документации проекта
## Actual
{
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "FIND_FILES",
"rag_count": 20,
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`",
"answer_mode": "deterministic",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramNotifyModule",
"TelegramNotifyWorker",
"TelegramControlChannel",
"RuntimeManager",
"TelegramSendService"
],
"doc_scope": [
"index.test_echo_app_docs",
"docs/README.md",
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md"
],
"layers": [
"D3_ENTITY_CATALOG",
"D1_DOCUMENT_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- none
@@ -0,0 +1,43 @@
{
"meta": {
"case_id": "v2-find-files-09-telegram-bot-api",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"llm: expected answer to contain 'docs/api/send-message-endpoint.md'",
"llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'"
],
"actual": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "FIND_FILES",
"rag_count": 20,
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
"answer_mode": "deterministic",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramNotifyModule",
"TelegramSendService",
"TelegramControlChannel",
"TelegramNotifyWorker",
"RuntimeManager"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D3_ENTITY_CATALOG",
"D1_DOCUMENT_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,51 @@
# v2-find-files-09-telegram-bot-api
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Какие файлы относятся к Telegram Bot API интеграции?
## Actual
{
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "FIND_FILES",
"rag_count": 20,
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
"answer_mode": "deterministic",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramNotifyModule",
"TelegramSendService",
"TelegramControlChannel",
"TelegramNotifyWorker",
"RuntimeManager"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D3_ENTITY_CATALOG",
"D1_DOCUMENT_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- llm: expected answer to contain 'docs/api/send-message-endpoint.md'
- llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'
@@ -0,0 +1,43 @@
{
"meta": {
"case_id": "v2-find-files-10-telegram-notify-worker",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'",
"llm: expected answer to contain 'docs/domains/runtime-health-entity.md'"
],
"actual": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "FIND_FILES",
"rag_count": 20,
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/architecture/telegram-notify-app-overview.md`",
"answer_mode": "deterministic",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramNotifyWorker",
"TelegramNotifyModule",
"TelegramSendService",
"TelegramControlChannel",
"RuntimeManager"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D3_ENTITY_CATALOG",
"D1_DOCUMENT_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,51 @@
# v2-find-files-10-telegram-notify-worker
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
В каких файлах описан worker `telegram_notify`?
## Actual
{
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "FIND_FILES",
"rag_count": 20,
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/architecture/telegram-notify-app-overview.md`",
"answer_mode": "deterministic",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramNotifyWorker",
"TelegramNotifyModule",
"TelegramSendService",
"TelegramControlChannel",
"RuntimeManager"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D3_ENTITY_CATALOG",
"D1_DOCUMENT_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'
- llm: expected answer to contain 'docs/domains/runtime-health-entity.md'
@@ -0,0 +1,27 @@
{
"meta": {
"case_id": "v2-general-01-what-is-service",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"llm: expected answer to contain 'Telegram'"
],
"actual": {
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Это зависит от контекста. Уточни, пожалуйста, о каком именно сервисе идет речь.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,35 @@
# v2-general-01-what-is-service
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Что это за сервис?
## Actual
{
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Это зависит от контекста. Уточни, пожалуйста, о каком именно сервисе идет речь.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- llm: expected answer to contain 'Telegram'
@@ -0,0 +1,25 @@
{
"meta": {
"case_id": "v2-general-02-purpose",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": true,
"mismatches": [],
"actual": {
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Test_echo_app нужен для тестирования и отладки приложений и систем, повторяя входящий запрос в качестве ответа, чтобы убедиться в правильной работе связи и функциональности.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,35 @@
# v2-general-02-purpose
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: True
## Query
Для чего нужен test_echo_app?
## Actual
{
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Test_echo_app нужен для тестирования и отладки приложений и систем, повторяя входящий запрос в качестве ответа, чтобы убедиться в правильной работе связи и функциональности.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- none
@@ -0,0 +1,27 @@
{
"meta": {
"case_id": "v2-general-03-problem",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"llm: expected answer to contain 'Telegram'"
],
"actual": {
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Приложение решает задачу (указать конкретную задачу, например, управления контактами, планирования задач, учета финансов и т.д.) для пользователей путем предоставления удобного и интуитивно понятного интерфейса.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,35 @@
# v2-general-03-problem
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Какую задачу решает это приложение?
## Actual
{
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Приложение решает задачу (указать конкретную задачу, например, управления контактами, планирования задач, учета финансов и т.д.) для пользователей путем предоставления удобного и интуитивно понятного интерфейса.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- llm: expected answer to contain 'Telegram'
@@ -0,0 +1,43 @@
{
"meta": {
"case_id": "v2-general-04-docs-contents",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"domain: expected GENERAL, got DOCS",
"intent: expected GENERAL_QA, got DOC_EXPLAIN"
],
"actual": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"rag_count": 12,
"llm_answer": "Документация проекта включает следующие компоненты:\n\n- Архитектура приложения `test_echo_app`\n- Описание HTTP API (control plane)\n- Цикл отправки уведомлений\n- Health-модель\n- Каталог ошибок\n\nСтруктура документации канонически разделена на несколько секций: `docs/architecture`, `docs/api`, `docs/logic`, `docs/domains`, `docs/errors`. Основная родительская документация — \"Архитектура Telegram Notify App\" (находится по пути ./architecture/telegram-notify-app-overview.md). Документы взаимосвязаны с помощью ссылок `related_docs`, `parent`/`children` и обычных markdown-ссылок без повторения одной и той же информации.\n\nФайлы, использованные для ответа: docs/README.md",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramControlChannel",
"TelegramNotifyModule",
"TelegramSendService",
"TelegramNotifyWorker",
"RuntimeManager"
],
"doc_scope": [
"index.test_echo_app_docs",
"docs/README.md",
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,51 @@
# v2-general-04-docs-contents
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Что входит в документацию этого проекта?
## Actual
{
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"rag_count": 12,
"llm_answer": "Документация проекта включает следующие компоненты:\n\n- Архитектура приложения `test_echo_app`\n- Описание HTTP API (control plane)\n- Цикл отправки уведомлений\n- Health-модель\n- Каталог ошибок\n\nСтруктура документации канонически разделена на несколько секций: `docs/architecture`, `docs/api`, `docs/logic`, `docs/domains`, `docs/errors`. Основная родительская документация — \"Архитектура Telegram Notify App\" (находится по пути ./architecture/telegram-notify-app-overview.md). Документы взаимосвязаны с помощью ссылок `related_docs`, `parent`/`children` и обычных markdown-ссылок без повторения одной и той же информации.\n\nФайлы, использованные для ответа: docs/README.md",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramControlChannel",
"TelegramNotifyModule",
"TelegramSendService",
"TelegramNotifyWorker",
"RuntimeManager"
],
"doc_scope": [
"index.test_echo_app_docs",
"docs/README.md",
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- domain: expected GENERAL, got DOCS
- intent: expected GENERAL_QA, got DOC_EXPLAIN
@@ -0,0 +1,27 @@
{
"meta": {
"case_id": "v2-general-05-main-parts",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"llm: expected answer to contain 'Telegram'"
],
"actual": {
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Основные части любой системы включают вход, процессоры (обработчики), память и выход.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,35 @@
# v2-general-05-main-parts
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Какие основные части есть у системы?
## Actual
{
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Основные части любой системы включают вход, процессоры (обработчики), память и выход.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- llm: expected answer to contain 'Telegram'
@@ -0,0 +1,25 @@
{
"meta": {
"case_id": "v2-general-06-high-level-architecture",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": true,
"mismatches": [],
"actual": {
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Приложение обычно состоит из пользовательского интерфейса (то, что видит и с чем взаимодействует пользователь), логики приложения (код, который обрабатывает действия пользователя) и базы данных (место хранения информации). Все эти части связаны между собой для обеспечения удобной работы пользователя.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,35 @@
# v2-general-06-high-level-architecture
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: True
## Query
Как в целом устроено приложение?
## Actual
{
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Приложение обычно состоит из пользовательского интерфейса (то, что видит и с чем взаимодействует пользователь), логики приложения (код, который обрабатывает действия пользователя) и базы данных (место хранения информации). Все эти части связаны между собой для обеспечения удобной работы пользователя.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- none
@@ -0,0 +1,25 @@
{
"meta": {
"case_id": "v2-general-07-besides-api",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": true,
"mismatches": [],
"actual": {
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Кроме API, здесь есть дружелюбная атмосфера, готовность помочь и интересные беседы на различные темы.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,35 @@
# v2-general-07-besides-api
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: True
## Query
Что тут есть кроме API?
## Actual
{
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Кроме API, здесь есть дружелюбная атмосфера, готовность помочь и интересные беседы на различные темы.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- none
@@ -0,0 +1,25 @@
{
"meta": {
"case_id": "v2-general-08-components",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": true,
"mismatches": [],
"actual": {
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Telegram Notify App состоит из простого пользовательского интерфейса для отправки уведомлений, механизма обработки сообщений и интеграции с API Telegram, позволяющей отправлять сообщения в выбранные чаты или каналы.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,35 @@
# v2-general-08-components
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: True
## Query
Из чего состоит Telegram Notify App?
## Actual
{
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Telegram Notify App состоит из простого пользовательского интерфейса для отправки уведомлений, механизма обработки сообщений и интеграции с API Telegram, позволяющей отправлять сообщения в выбранные чаты или каналы.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- none
@@ -0,0 +1,45 @@
{
"meta": {
"case_id": "v2-general-09-reading-order",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"domain: expected GENERAL, got DOCS",
"intent: expected GENERAL_QA, got DOC_EXPLAIN",
"llm: expected answer to contain 'README'",
"llm: expected answer to contain 'архитект'"
],
"actual": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"rag_count": 12,
"llm_answer": "Не нашёл подходящих SUMMARY-блоков в документации по этому запросу.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramNotifyModule",
"TelegramSendService",
"TelegramNotifyWorker",
"TelegramControlChannel",
"RuntimeManager"
],
"doc_scope": [
"index.test_echo_app_docs",
"docs/README.md",
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,53 @@
# v2-general-09-reading-order
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Какие документы стоит читать сначала?
## Actual
{
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"rag_count": 12,
"llm_answer": "Не нашёл подходящих SUMMARY-блоков в документации по этому запросу.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramNotifyModule",
"TelegramSendService",
"TelegramNotifyWorker",
"TelegramControlChannel",
"RuntimeManager"
],
"doc_scope": [
"index.test_echo_app_docs",
"docs/README.md",
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- domain: expected GENERAL, got DOCS
- intent: expected GENERAL_QA, got DOC_EXPLAIN
- llm: expected answer to contain 'README'
- llm: expected answer to contain 'архитект'
@@ -0,0 +1,25 @@
{
"meta": {
"case_id": "v2-general-10-short-summary",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": true,
"mismatches": [],
"actual": {
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Проектная документация включает описание целей, задач, методов, графика и бюджета проекта, а также список необходимых ресурсов и команду исполнителей.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
},
"pipeline_steps": []
}
@@ -0,0 +1,35 @@
# v2-general-10-short-summary
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: True
## Query
Дай короткое summary по проектной документации
## Actual
{
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"rag_count": 0,
"llm_answer": "Проектная документация включает описание целей, задач, методов, графика и бюджета проекта, а также список необходимых ресурсов и команду исполнителей.",
"answer_mode": "answered",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [],
"layers": [],
"filters": {}
}
## Pipeline Steps
- none
## Diagnostics
{}
## Mismatches
- none
@@ -0,0 +1,153 @@
# pipeline_setup_v3 summary
Passed: 7/30
| File | Case | Mode | Query | Actual sub-intent | RAG layers | Tokens | Pass |
|------|------|------|-------|-------------------|------------|--------|------|
| full_chain_docs_v2_matrix.yaml | v2-general-01-what-is-service | full_chain | Что это за сервис? | SUMMARY | — | — | ✗ |
| full_chain_docs_v2_matrix.yaml | v2-general-02-purpose | full_chain | Для чего нужен test_echo_app? | SUMMARY | — | — | ✓ |
| full_chain_docs_v2_matrix.yaml | v2-general-03-problem | full_chain | Какую задачу решает это приложение? | SUMMARY | — | — | ✗ |
| full_chain_docs_v2_matrix.yaml | v2-general-04-docs-contents | full_chain | Что входит в документацию этого проекта? | SUMMARY | — | — | ✗ |
| full_chain_docs_v2_matrix.yaml | v2-general-05-main-parts | full_chain | Какие основные части есть у системы? | SUMMARY | — | — | ✗ |
| full_chain_docs_v2_matrix.yaml | v2-general-06-high-level-architecture | full_chain | Как в целом устроено приложение? | SUMMARY | — | — | ✓ |
| full_chain_docs_v2_matrix.yaml | v2-general-07-besides-api | full_chain | Что тут есть кроме API? | SUMMARY | — | — | ✓ |
| full_chain_docs_v2_matrix.yaml | v2-general-08-components | full_chain | Из чего состоит Telegram Notify App? | SUMMARY | — | — | ✓ |
| full_chain_docs_v2_matrix.yaml | v2-general-09-reading-order | full_chain | Какие документы стоит читать сначала? | SUMMARY | — | — | ✗ |
| full_chain_docs_v2_matrix.yaml | v2-general-10-short-summary | full_chain | Дай короткое summary по проектной документации | SUMMARY | — | — | ✓ |
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-01-health | full_chain | Кратко объясни по документации, как работает `/health` | SUMMARY | — | — | ✗ |
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-02-send | full_chain | Что делает endpoint `/send`? | SUMMARY | — | — | ✗ |
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-03-actions | full_chain | Объясни назначение `/actions/{action}` | SUMMARY | — | — | ✗ |
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-04-runtime-health | full_chain | Что такое runtime health в этой документации? | SUMMARY | — | — | ✗ |
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-05-notification-loop | full_chain | Кратко опиши цикл отправки уведомлений | SUMMARY | — | — | ✗ |
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-06-architecture | full_chain | Как устроена архитектура Telegram Notify App? | SUMMARY | — | — | ✗ |
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-07-worker | full_chain | Что делает worker в этом приложении? | SUMMARY | — | — | ✗ |
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-08-health-statuses | full_chain | Какие статусы здоровья есть у runtime? | SUMMARY | — | — | ✗ |
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-09-manual-send | full_chain | Как в приложении устроена ручная отправка сообщения? | SUMMARY | — | — | ✗ |
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-10-telegram-integration | full_chain | Как сервис взаимодействует с Telegram в целом? | SUMMARY | — | — | ✗ |
| full_chain_docs_v2_matrix.yaml | v2-find-files-01-health | full_chain | В каком файле описан `/health`? | FIND_FILES | — | — | ✗ |
| full_chain_docs_v2_matrix.yaml | v2-find-files-02-send | full_chain | Покажи файл с описанием endpoint `/send` | FIND_FILES | — | — | ✗ |
| full_chain_docs_v2_matrix.yaml | v2-find-files-03-actions | full_chain | Где находится документация по `/actions/{action}`? | FIND_FILES | — | — | ✗ |
| full_chain_docs_v2_matrix.yaml | v2-find-files-04-runtime-health | full_chain | В каком документе описан runtime health? | FIND_FILES | — | — | ✗ |
| full_chain_docs_v2_matrix.yaml | v2-find-files-05-loop | full_chain | Найди файл с логикой цикла отправки уведомлений | FIND_FILES | — | — | ✗ |
| full_chain_docs_v2_matrix.yaml | v2-find-files-06-architecture | full_chain | Где описана архитектура приложения? | FIND_FILES | — | — | ✓ |
| full_chain_docs_v2_matrix.yaml | v2-find-files-07-errors-catalog | full_chain | В каком файле лежит каталог ошибок? | FIND_FILES | — | — | ✗ |
| full_chain_docs_v2_matrix.yaml | v2-find-files-08-docs-index | full_chain | Покажи файл-индекс документации проекта | FIND_FILES | — | — | ✓ |
| full_chain_docs_v2_matrix.yaml | v2-find-files-09-telegram-bot-api | full_chain | Какие файлы относятся к Telegram Bot API интеграции? | FIND_FILES | — | — | ✗ |
| full_chain_docs_v2_matrix.yaml | v2-find-files-10-telegram-notify-worker | full_chain | В каких файлах описан worker `telegram_notify`? | FIND_FILES | — | — | ✗ |
## Failures
- **v2-general-01-what-is-service**: llm: expected answer to contain 'Telegram'
- **v2-general-03-problem**: llm: expected answer to contain 'Telegram'
- **v2-general-04-docs-contents**: domain: expected GENERAL, got DOCS; intent: expected GENERAL_QA, got DOC_EXPLAIN
- **v2-general-05-main-parts**: llm: expected answer to contain 'Telegram'
- **v2-general-09-reading-order**: domain: expected GENERAL, got DOCS; intent: expected GENERAL_QA, got DOC_EXPLAIN; llm: expected answer to contain 'README'; llm: expected answer to contain 'архитект'
- **v2-docs-summary-01-health**: llm: expected answer to contain 'status'; llm: expected answer to contain 'components'
- **v2-docs-summary-02-send**: doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']; llm: expected answer to contain 'message'
- **v2-docs-summary-03-actions**: domain: expected DOCS, got GENERAL; intent: expected DOC_EXPLAIN, got GENERAL_QA; retrieval: expected non-empty rag, got 0 rows; doc_scope should contain 'docs/api/control-actions-endpoint.md', got []
- **v2-docs-summary-04-runtime-health**: domain: expected DOCS, got GENERAL; intent: expected DOC_EXPLAIN, got GENERAL_QA; retrieval: expected non-empty rag, got 0 rows; doc_scope should contain 'docs/domains/runtime-health-entity.md', got []
- **v2-docs-summary-05-notification-loop**: domain: expected DOCS, got GENERAL; intent: expected DOC_EXPLAIN, got GENERAL_QA; retrieval: expected non-empty rag, got 0 rows; doc_scope should contain 'docs/logic/telegram-notification-loop.md', got []; llm: expected answer to contain 'worker'; llm: expected answer to contain 'interval'; llm: expected answer to contain 'degraded'
- **v2-docs-summary-06-architecture**: domain: expected DOCS, got GENERAL; intent: expected DOC_EXPLAIN, got GENERAL_QA; retrieval: expected non-empty rag, got 0 rows; doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []; llm: expected answer to contain 'RuntimeManager'; llm: expected answer to contain 'TelegramControlChannel'
- **v2-docs-summary-07-worker**: domain: expected DOCS, got GENERAL; intent: expected DOC_EXPLAIN, got GENERAL_QA; retrieval: expected non-empty rag, got 0 rows; doc_scope should contain 'docs/logic/telegram-notification-loop.md', got []; doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []; llm: expected answer to contain 'Telegram'
- **v2-docs-summary-08-health-statuses**: doc_scope should contain 'docs/domains/runtime-health-entity.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']; doc_scope should contain 'docs/api/health-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']; llm: expected answer to contain 'ok'; llm: expected answer to contain 'degraded'; llm: expected answer to contain 'unhealthy'; llm: expected answer to contain 'unknown'
- **v2-docs-summary-09-manual-send**: domain: expected DOCS, got GENERAL; intent: expected DOC_EXPLAIN, got GENERAL_QA; retrieval: expected non-empty rag, got 0 rows; doc_scope should contain 'docs/api/send-message-endpoint.md', got []; llm: expected answer to contain '/send'; llm: expected answer to contain 'message'; llm: expected answer to contain 'TelegramSendService'
- **v2-docs-summary-10-telegram-integration**: domain: expected DOCS, got GENERAL; intent: expected DOC_EXPLAIN, got GENERAL_QA; retrieval: expected non-empty rag, got 0 rows; retrieval: expected at least 2 rows, got 0; doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []; doc_scope should contain 'docs/api/send-message-endpoint.md', got []; llm: expected answer to contain 'worker'; llm: expected answer to contain '/send'
- **v2-find-files-01-health**: llm: expected answer to contain 'docs/api/health-endpoint.md'
- **v2-find-files-02-send**: llm: expected answer to contain 'docs/api/send-message-endpoint.md'
- **v2-find-files-03-actions**: llm: expected answer to contain 'docs/api/control-actions-endpoint.md'
- **v2-find-files-04-runtime-health**: llm: expected answer to contain 'docs/domains/runtime-health-entity.md'
- **v2-find-files-05-loop**: llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'
- **v2-find-files-07-errors-catalog**: llm: expected answer to contain 'docs/errors/catalog.yaml'
- **v2-find-files-09-telegram-bot-api**: llm: expected answer to contain 'docs/api/send-message-endpoint.md'; llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'
- **v2-find-files-10-telegram-notify-worker**: llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'; llm: expected answer to contain 'docs/domains/runtime-health-entity.md'
## LLM Answers
- **v2-general-01-what-is-service**
Query: Что это за сервис?
> Это зависит от контекста. Уточни, пожалуйста, о каком именно сервисе идет речь.
- **v2-general-02-purpose**
Query: Для чего нужен test_echo_app?
> Test\_echo\_app нужен для тестирования и отладки приложений и систем, повторяя входящий запрос в качестве ответа, чтобы убедиться в правильной работе связи и функциональности.
- **v2-general-03-problem**
Query: Какую задачу решает это приложение?
> Приложение решает задачу \(указать конкретную задачу, например, управления контактами, планирования задач, учета финансов и т.д.\) для пользователей путем предоставления удобного и интуитивно понятного интерфейса.
- **v2-general-04-docs-contents**
Query: Что входит в документацию этого проекта?
> Документация проекта включает следующие компоненты: \- Архитектура приложения \`test\_echo\_app\` \- Описание HTTP API \(control plane\) \- Цикл отправки уведомлений \- Health\-модель \- Каталог ошибок Структура документации канонически разделена на несколько секций: \`docs/architecture\`, \`docs/api\`, \`docs/logic\`, \`docs/domains\`, \`docs/errors\`. Основная родительская документация — "Архитектура Telegram Notify App" \(находится по пути ./architecture/telegram\-notify\-app\-overview.md\). Документы взаимосвязаны с помощью ссылок \`related\_docs\`, \`parent\`/\`children\` и обычных markdown\-ссылок без повторения одной и той же информации. Файлы, использованные для ответа: docs/README.md
- **v2-general-05-main-parts**
Query: Какие основные части есть у системы?
> Основные части любой системы включают вход, процессоры \(обработчики\), память и выход.
- **v2-general-06-high-level-architecture**
Query: Как в целом устроено приложение?
> Приложение обычно состоит из пользовательского интерфейса \(то, что видит и с чем взаимодействует пользователь\), логики приложения \(код, который обрабатывает действия пользователя\) и базы данных \(место хранения информации\). Все эти части связаны между собой для обеспечения удобной работы пользователя.
- **v2-general-07-besides-api**
Query: Что тут есть кроме API?
> Кроме API, здесь есть дружелюбная атмосфера, готовность помочь и интересные беседы на различные темы.
- **v2-general-08-components**
Query: Из чего состоит Telegram Notify App?
> Telegram Notify App состоит из простого пользовательского интерфейса для отправки уведомлений, механизма обработки сообщений и интеграции с API Telegram, позволяющей отправлять сообщения в выбранные чаты или каналы.
- **v2-general-09-reading-order**
Query: Какие документы стоит читать сначала?
> Не нашёл подходящих SUMMARY\-блоков в документации по этому запросу.
- **v2-general-10-short-summary**
Query: Дай короткое summary по проектной документации
> Проектная документация включает описание целей, задач, методов, графика и бюджета проекта, а также список необходимых ресурсов и команду исполнителей.
- **v2-docs-summary-01-health**
Query: Кратко объясни по документации, как работает `/health`
> Информация о том, как именно работает \`/health\`, не представлена в предоставленных SUMMARY\-блоках. Файлы, на которые я опирался: \- docs/architecture/telegram\-notify\-app\-overview.md
- **v2-docs-summary-02-send**
Query: Что делает endpoint `/send`?
> На основе представленного SUMMARY\-блока, документация о том, что именно делает endpoint \`/send\`, отсутствует. В контексте указано, что он связан с отправкой сообщений, но подробностей его работы не приведено. Файлы, на которые опирался ответ: \- docs/architecture/telegram\-notify\-app\-overview.md
- **v2-docs-summary-03-actions**
Query: Объясни назначение `/actions/{action}`
> \`/actions/\{action\}\` обычно используется в API для выполнения конкретных действий или операций, где \`\{action\}\` заменяется названием нужного действия. Это позволяет организовывать и вызывать различные функции через единый интерфейс.
- **v2-docs-summary-04-runtime-health**
Query: Что такое runtime health в этой документации?
> Runtime health — это процесс мониторинга и обеспечения стабильности работы системы или приложения во время их выполнения \(runtime\), позволяющий своевременно обнаруживать и устранять возникающие проблемы.
- **v2-docs-summary-05-notification-loop**
Query: Кратко опиши цикл отправки уведомлений
> Цикл отправки уведомлений включает следующие шаги: отлавливание события, обработка данных, выбор получателей, формирование сообщения и его отправка выбранным пользователям.
- **v2-docs-summary-06-architecture**
Query: Как устроена архитектура Telegram Notify App?
> Архитектура Telegram Notify App основана на клиент\-серверной модели, где серверное приложение обрабатывает входящие данные и отправляет уведомления через Telegram API. Клиентское приложение взаимодействует с сервером для отправки уведомлений. Пользователь настраивает параметры уведомлений в приложении, а сервер передает их в виде сообщений в указанный Telegram\-канал или пользователю.
- **v2-docs-summary-07-worker**
Query: Что делает worker в этом приложении?
> Worker в этом приложении выполняет фоновые задачи или процессы, позволяя основному приложению работать быстро и плавно. Это может включать обработку данных, отправку уведомлений, загрузку файлов и другие ресурсоемкие операции.
- **v2-docs-summary-08-health-statuses**
Query: Какие статусы здоровья есть у runtime?
> На основе представленного SUMMARY\-блока конкретной информации о статусах здоровья runtime нет. Файл, связанный с доменом "runtime health" \(../domains/runtime\-health\-entity.md\), вероятно, содержит эту информацию, но она не включена в текущий контекст. Файлы, на которые опирался ответ: \- docs/architecture/telegram\-notify\-app\-overview.md
- **v2-docs-summary-09-manual-send**
Query: Как в приложении устроена ручная отправка сообщения?
> Ручная отправка сообщения в приложении обычно осуществляется нажатием кнопки "Отправить" или аналогичного элемента интерфейса после того, как пользователь введет текст сообщения.
- **v2-docs-summary-10-telegram-integration**
Query: Как сервис взаимодействует с Telegram в целом?
> Сервис может отправлять и получать сообщения, реагировать на команды и запросы пользователей через Telegram\-бота. Взаимодействие происходит посредством отправки запросов и получения ответов от бота в чате.
- **v2-find-files-01-health**
Query: В каком файле описан `/health`?
> Нашёл релевантные файлы документации: \- \`docs/README.md\` \- \`docs/architecture/telegram\-notify\-app\-overview.md\`
- **v2-find-files-02-send**
Query: Покажи файл с описанием endpoint `/send`
> Нашёл релевантные файлы документации: \- \`docs/README.md\` \- \`docs/architecture/telegram\-notify\-app\-overview.md\`
- **v2-find-files-03-actions**
Query: Где находится документация по `/actions/{action}`?
> Нашёл релевантные файлы документации: \- \`docs/README.md\` \- \`docs/architecture/telegram\-notify\-app\-overview.md\`
- **v2-find-files-04-runtime-health**
Query: В каком документе описан runtime health?
> Нашёл релевантные файлы документации: \- \`docs/architecture/telegram\-notify\-app\-overview.md\` \- \`docs/README.md\`
- **v2-find-files-05-loop**
Query: Найди файл с логикой цикла отправки уведомлений
> Нашёл релевантные файлы документации: \- \`docs/README.md\` \- \`docs/architecture/telegram\-notify\-app\-overview.md\`
- **v2-find-files-06-architecture**
Query: Где описана архитектура приложения?
> Нашёл релевантные файлы документации: \- \`docs/README.md\` \- \`docs/architecture/telegram\-notify\-app\-overview.md\`
- **v2-find-files-07-errors-catalog**
Query: В каком файле лежит каталог ошибок?
> Нашёл релевантные файлы документации: \- \`docs/README.md\`
- **v2-find-files-08-docs-index**
Query: Покажи файл-индекс документации проекта
> Нашёл релевантные файлы документации: \- \`docs/README.md\`
- **v2-find-files-09-telegram-bot-api**
Query: Какие файлы относятся к Telegram Bot API интеграции?
> Нашёл релевантные файлы документации: \- \`docs/README.md\` \- \`docs/architecture/telegram\-notify\-app\-overview.md\`
- **v2-find-files-10-telegram-notify-worker**
Query: В каких файлах описан worker `telegram_notify`?
> Нашёл релевантные файлы документации: \- \`docs/architecture/telegram\-notify\-app\-overview.md\`
@@ -0,0 +1,153 @@
{
"meta": {
"case_id": "v2-docs-summary-01-health",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"answer_mode: expected grounded_summary, got insufficient_evidence",
"llm: expected answer to contain '/health'",
"llm: expected answer to contain 'status'",
"llm: expected answer to contain 'components'"
],
"actual": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"anchor_type": "API_ENDPOINT",
"rag_count": 8,
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
"answer_mode": "insufficient_evidence",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramNotifyModule",
"TelegramNotifyWorker",
"RuntimeManager",
"TelegramSendService",
"TelegramControlChannel"
],
"doc_scope": [
"index.test_echo_app_docs",
"docs/README.md",
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/api/",
"docs/"
],
"prefer_like_patterns": [
"%health-endpoint.md%",
"%/health%"
],
"target_doc_hints": [
"docs/api/health-endpoint.md"
],
"path_prefixes": [
"docs/api/",
"docs/architecture/",
"docs/"
]
}
}
},
"pipeline_steps": [
{
"step": "router_resolved",
"output": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"subintent": "SUMMARY",
"anchor_type": "API_ENDPOINT",
"confidence": 1.0
}
},
{
"step": "anchors_extracted",
"output": {
"anchor_type": "API_ENDPOINT",
"endpoint_paths": [
"/health"
],
"target_doc_hints": [
"docs/api/health-endpoint.md"
],
"matched_aliases": []
}
},
{
"step": "retrieval_profile_selected",
"output": {
"profile": "docs_summary_api_endpoint",
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/api/",
"docs/"
],
"prefer_like_patterns": [
"%health-endpoint.md%",
"%/health%"
],
"target_doc_hints": [
"docs/api/health-endpoint.md"
],
"path_prefixes": [
"docs/api/",
"docs/architecture/",
"docs/"
]
}
}
},
{
"step": "retrieval_executed",
"output": {
"profile": "docs_summary_api_endpoint",
"row_count": 8,
"top_files": [
"docs/README.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md"
]
}
},
{
"step": "evidence_assembled",
"output": {
"mode": "summary",
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
"document_count": 1
}
},
{
"step": "evidence_gate_checked",
"output": {
"passed": false,
"reason": "target_doc_missing",
"answer_mode": "insufficient_evidence"
}
},
{
"step": "answer_generated",
"output": {
"answer_mode": "insufficient_evidence",
"answer_length": 131
}
}
]
}
@@ -0,0 +1,186 @@
# v2-docs-summary-01-health
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Кратко объясни по документации, как работает `/health`
## Actual
{
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"anchor_type": "API_ENDPOINT",
"rag_count": 8,
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
"answer_mode": "insufficient_evidence",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramNotifyModule",
"TelegramNotifyWorker",
"RuntimeManager",
"TelegramSendService",
"TelegramControlChannel"
],
"doc_scope": [
"index.test_echo_app_docs",
"docs/README.md",
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/api/",
"docs/"
],
"prefer_like_patterns": [
"%health-endpoint.md%",
"%/health%"
],
"target_doc_hints": [
"docs/api/health-endpoint.md"
],
"path_prefixes": [
"docs/api/",
"docs/architecture/",
"docs/"
]
}
}
## Pipeline Steps
### router_resolved
```json
{
"input": {},
"output": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"subintent": "SUMMARY",
"anchor_type": "API_ENDPOINT",
"confidence": 1.0
}
}
```
### anchors_extracted
```json
{
"input": {},
"output": {
"anchor_type": "API_ENDPOINT",
"endpoint_paths": [
"/health"
],
"target_doc_hints": [
"docs/api/health-endpoint.md"
],
"matched_aliases": []
}
}
```
### retrieval_profile_selected
```json
{
"input": {},
"output": {
"profile": "docs_summary_api_endpoint",
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/api/",
"docs/"
],
"prefer_like_patterns": [
"%health-endpoint.md%",
"%/health%"
],
"target_doc_hints": [
"docs/api/health-endpoint.md"
],
"path_prefixes": [
"docs/api/",
"docs/architecture/",
"docs/"
]
}
}
}
```
### retrieval_executed
```json
{
"input": {},
"output": {
"profile": "docs_summary_api_endpoint",
"row_count": 8,
"top_files": [
"docs/README.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md"
]
}
}
```
### evidence_assembled
```json
{
"input": {},
"output": {
"mode": "summary",
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
"document_count": 1
}
}
```
### evidence_gate_checked
```json
{
"input": {},
"output": {
"passed": false,
"reason": "target_doc_missing",
"answer_mode": "insufficient_evidence"
}
}
```
### answer_generated
```json
{
"input": {},
"output": {
"answer_mode": "insufficient_evidence",
"answer_length": 131
}
}
```
## Diagnostics
{}
## Mismatches
- answer_mode: expected grounded_summary, got insufficient_evidence
- llm: expected answer to contain '/health'
- llm: expected answer to contain 'status'
- llm: expected answer to contain 'components'
@@ -0,0 +1,152 @@
{
"meta": {
"case_id": "v2-docs-summary-02-send",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']",
"llm: expected answer to contain '/send'",
"llm: expected answer to contain 'message'"
],
"actual": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"anchor_type": "API_ENDPOINT",
"rag_count": 8,
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
"answer_mode": "insufficient_evidence",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramSendService",
"TelegramNotifyModule",
"TelegramControlChannel",
"TelegramNotifyWorker",
"RuntimeManager"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/api/",
"docs/"
],
"prefer_like_patterns": [
"%send-message-endpoint.md%",
"%/send%"
],
"target_doc_hints": [
"docs/api/send-message-endpoint.md"
],
"path_prefixes": [
"docs/api/",
"docs/architecture/",
"docs/"
]
}
}
},
"pipeline_steps": [
{
"step": "router_resolved",
"output": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"subintent": "SUMMARY",
"anchor_type": "API_ENDPOINT",
"confidence": 1.0
}
},
{
"step": "anchors_extracted",
"output": {
"anchor_type": "API_ENDPOINT",
"endpoint_paths": [
"/send"
],
"target_doc_hints": [
"docs/api/send-message-endpoint.md"
],
"matched_aliases": []
}
},
{
"step": "retrieval_profile_selected",
"output": {
"profile": "docs_summary_api_endpoint",
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/api/",
"docs/"
],
"prefer_like_patterns": [
"%send-message-endpoint.md%",
"%/send%"
],
"target_doc_hints": [
"docs/api/send-message-endpoint.md"
],
"path_prefixes": [
"docs/api/",
"docs/architecture/",
"docs/"
]
}
}
},
{
"step": "retrieval_executed",
"output": {
"profile": "docs_summary_api_endpoint",
"row_count": 8,
"top_files": [
"docs/architecture/telegram-notify-app-overview.md",
"docs/README.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md"
]
}
},
{
"step": "evidence_assembled",
"output": {
"mode": "summary",
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
"document_count": 1
}
},
{
"step": "evidence_gate_checked",
"output": {
"passed": false,
"reason": "target_doc_missing",
"answer_mode": "insufficient_evidence"
}
},
{
"step": "answer_generated",
"output": {
"answer_mode": "insufficient_evidence",
"answer_length": 131
}
}
]
}
@@ -0,0 +1,185 @@
# v2-docs-summary-02-send
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Что делает endpoint `/send`?
## Actual
{
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"anchor_type": "API_ENDPOINT",
"rag_count": 8,
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
"answer_mode": "insufficient_evidence",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramSendService",
"TelegramNotifyModule",
"TelegramControlChannel",
"TelegramNotifyWorker",
"RuntimeManager"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/api/",
"docs/"
],
"prefer_like_patterns": [
"%send-message-endpoint.md%",
"%/send%"
],
"target_doc_hints": [
"docs/api/send-message-endpoint.md"
],
"path_prefixes": [
"docs/api/",
"docs/architecture/",
"docs/"
]
}
}
## Pipeline Steps
### router_resolved
```json
{
"input": {},
"output": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"subintent": "SUMMARY",
"anchor_type": "API_ENDPOINT",
"confidence": 1.0
}
}
```
### anchors_extracted
```json
{
"input": {},
"output": {
"anchor_type": "API_ENDPOINT",
"endpoint_paths": [
"/send"
],
"target_doc_hints": [
"docs/api/send-message-endpoint.md"
],
"matched_aliases": []
}
}
```
### retrieval_profile_selected
```json
{
"input": {},
"output": {
"profile": "docs_summary_api_endpoint",
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/api/",
"docs/"
],
"prefer_like_patterns": [
"%send-message-endpoint.md%",
"%/send%"
],
"target_doc_hints": [
"docs/api/send-message-endpoint.md"
],
"path_prefixes": [
"docs/api/",
"docs/architecture/",
"docs/"
]
}
}
}
```
### retrieval_executed
```json
{
"input": {},
"output": {
"profile": "docs_summary_api_endpoint",
"row_count": 8,
"top_files": [
"docs/architecture/telegram-notify-app-overview.md",
"docs/README.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md"
]
}
}
```
### evidence_assembled
```json
{
"input": {},
"output": {
"mode": "summary",
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
"document_count": 1
}
}
```
### evidence_gate_checked
```json
{
"input": {},
"output": {
"passed": false,
"reason": "target_doc_missing",
"answer_mode": "insufficient_evidence"
}
}
```
### answer_generated
```json
{
"input": {},
"output": {
"answer_mode": "insufficient_evidence",
"answer_length": 131
}
}
```
## Diagnostics
{}
## Mismatches
- doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
- llm: expected answer to contain '/send'
- llm: expected answer to contain 'message'
@@ -0,0 +1,153 @@
{
"meta": {
"case_id": "v2-docs-summary-03-actions",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"doc_scope should contain 'docs/api/control-actions-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']",
"llm: expected answer to contain '/actions/{action}'"
],
"actual": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"anchor_type": "API_ENDPOINT",
"rag_count": 8,
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
"answer_mode": "insufficient_evidence",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"RuntimeManager",
"TelegramNotifyModule",
"TelegramNotifyWorker",
"TelegramControlChannel",
"TelegramSendService"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/api/",
"docs/"
],
"prefer_like_patterns": [
"%control-actions-endpoint.md%",
"%/actions/{action}%"
],
"target_doc_hints": [
"docs/api/control-actions-endpoint.md"
],
"path_prefixes": [
"docs/api/",
"docs/architecture/",
"docs/"
]
}
}
},
"pipeline_steps": [
{
"step": "router_resolved",
"output": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"subintent": "SUMMARY",
"anchor_type": "API_ENDPOINT",
"confidence": 1.0
}
},
{
"step": "anchors_extracted",
"output": {
"anchor_type": "API_ENDPOINT",
"endpoint_paths": [
"/actions/{action}"
],
"target_doc_hints": [
"docs/api/control-actions-endpoint.md"
],
"matched_aliases": [
"/actions/{action}"
]
}
},
{
"step": "retrieval_profile_selected",
"output": {
"profile": "docs_summary_api_endpoint",
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/api/",
"docs/"
],
"prefer_like_patterns": [
"%control-actions-endpoint.md%",
"%/actions/{action}%"
],
"target_doc_hints": [
"docs/api/control-actions-endpoint.md"
],
"path_prefixes": [
"docs/api/",
"docs/architecture/",
"docs/"
]
}
}
},
{
"step": "retrieval_executed",
"output": {
"profile": "docs_summary_api_endpoint",
"row_count": 8,
"top_files": [
"docs/architecture/telegram-notify-app-overview.md",
"docs/README.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md"
]
}
},
{
"step": "evidence_assembled",
"output": {
"mode": "summary",
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
"document_count": 1
}
},
{
"step": "evidence_gate_checked",
"output": {
"passed": false,
"reason": "target_doc_missing",
"answer_mode": "insufficient_evidence"
}
},
{
"step": "answer_generated",
"output": {
"answer_mode": "insufficient_evidence",
"answer_length": 131
}
}
]
}
@@ -0,0 +1,186 @@
# v2-docs-summary-03-actions
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Объясни назначение `/actions/{action}`
## Actual
{
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"anchor_type": "API_ENDPOINT",
"rag_count": 8,
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
"answer_mode": "insufficient_evidence",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"RuntimeManager",
"TelegramNotifyModule",
"TelegramNotifyWorker",
"TelegramControlChannel",
"TelegramSendService"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/api/",
"docs/"
],
"prefer_like_patterns": [
"%control-actions-endpoint.md%",
"%/actions/{action}%"
],
"target_doc_hints": [
"docs/api/control-actions-endpoint.md"
],
"path_prefixes": [
"docs/api/",
"docs/architecture/",
"docs/"
]
}
}
## Pipeline Steps
### router_resolved
```json
{
"input": {},
"output": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"subintent": "SUMMARY",
"anchor_type": "API_ENDPOINT",
"confidence": 1.0
}
}
```
### anchors_extracted
```json
{
"input": {},
"output": {
"anchor_type": "API_ENDPOINT",
"endpoint_paths": [
"/actions/{action}"
],
"target_doc_hints": [
"docs/api/control-actions-endpoint.md"
],
"matched_aliases": [
"/actions/{action}"
]
}
}
```
### retrieval_profile_selected
```json
{
"input": {},
"output": {
"profile": "docs_summary_api_endpoint",
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/api/",
"docs/"
],
"prefer_like_patterns": [
"%control-actions-endpoint.md%",
"%/actions/{action}%"
],
"target_doc_hints": [
"docs/api/control-actions-endpoint.md"
],
"path_prefixes": [
"docs/api/",
"docs/architecture/",
"docs/"
]
}
}
}
```
### retrieval_executed
```json
{
"input": {},
"output": {
"profile": "docs_summary_api_endpoint",
"row_count": 8,
"top_files": [
"docs/architecture/telegram-notify-app-overview.md",
"docs/README.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md"
]
}
}
```
### evidence_assembled
```json
{
"input": {},
"output": {
"mode": "summary",
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
"document_count": 1
}
}
```
### evidence_gate_checked
```json
{
"input": {},
"output": {
"passed": false,
"reason": "target_doc_missing",
"answer_mode": "insufficient_evidence"
}
}
```
### answer_generated
```json
{
"input": {},
"output": {
"answer_mode": "insufficient_evidence",
"answer_length": 131
}
}
```
## Diagnostics
{}
## Mismatches
- doc_scope should contain 'docs/api/control-actions-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
- llm: expected answer to contain '/actions/{action}'
@@ -0,0 +1,139 @@
{
"meta": {
"case_id": "v2-docs-summary-04-runtime-health",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"doc_scope should contain 'docs/domains/runtime-health-entity.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md']",
"llm: expected answer to contain 'runtime health'"
],
"actual": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"anchor_type": "DOMAIN_ENTITY",
"rag_count": 8,
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
"answer_mode": "insufficient_evidence",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"RuntimeManager",
"TelegramNotifyModule",
"TelegramNotifyWorker",
"TelegramSendService",
"TelegramControlChannel"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/domains/",
"docs/api/",
"docs/architecture/"
],
"prefer_like_patterns": [
"%runtime-health-entity.md%"
],
"target_doc_hints": [
"docs/domains/runtime-health-entity.md"
]
}
}
},
"pipeline_steps": [
{
"step": "router_resolved",
"output": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"subintent": "SUMMARY",
"anchor_type": "DOMAIN_ENTITY",
"confidence": 1.0
}
},
{
"step": "anchors_extracted",
"output": {
"anchor_type": "DOMAIN_ENTITY",
"endpoint_paths": [],
"target_doc_hints": [
"docs/domains/runtime-health-entity.md"
],
"matched_aliases": [
"runtime-health-entity"
]
}
},
{
"step": "retrieval_profile_selected",
"output": {
"profile": "docs_summary_domain_entity",
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/domains/",
"docs/api/",
"docs/architecture/"
],
"prefer_like_patterns": [
"%runtime-health-entity.md%"
],
"target_doc_hints": [
"docs/domains/runtime-health-entity.md"
]
}
}
},
{
"step": "retrieval_executed",
"output": {
"profile": "docs_summary_domain_entity",
"row_count": 8,
"top_files": [
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md"
]
}
},
{
"step": "evidence_assembled",
"output": {
"mode": "summary",
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
"document_count": 1
}
},
{
"step": "evidence_gate_checked",
"output": {
"passed": false,
"reason": "target_doc_missing",
"answer_mode": "insufficient_evidence"
}
},
{
"step": "answer_generated",
"output": {
"answer_mode": "insufficient_evidence",
"answer_length": 131
}
}
]
}
@@ -0,0 +1,172 @@
# v2-docs-summary-04-runtime-health
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Что такое runtime health в этой документации?
## Actual
{
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"anchor_type": "DOMAIN_ENTITY",
"rag_count": 8,
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
"answer_mode": "insufficient_evidence",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"RuntimeManager",
"TelegramNotifyModule",
"TelegramNotifyWorker",
"TelegramSendService",
"TelegramControlChannel"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/domains/",
"docs/api/",
"docs/architecture/"
],
"prefer_like_patterns": [
"%runtime-health-entity.md%"
],
"target_doc_hints": [
"docs/domains/runtime-health-entity.md"
]
}
}
## Pipeline Steps
### router_resolved
```json
{
"input": {},
"output": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"subintent": "SUMMARY",
"anchor_type": "DOMAIN_ENTITY",
"confidence": 1.0
}
}
```
### anchors_extracted
```json
{
"input": {},
"output": {
"anchor_type": "DOMAIN_ENTITY",
"endpoint_paths": [],
"target_doc_hints": [
"docs/domains/runtime-health-entity.md"
],
"matched_aliases": [
"runtime-health-entity"
]
}
}
```
### retrieval_profile_selected
```json
{
"input": {},
"output": {
"profile": "docs_summary_domain_entity",
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/domains/",
"docs/api/",
"docs/architecture/"
],
"prefer_like_patterns": [
"%runtime-health-entity.md%"
],
"target_doc_hints": [
"docs/domains/runtime-health-entity.md"
]
}
}
}
```
### retrieval_executed
```json
{
"input": {},
"output": {
"profile": "docs_summary_domain_entity",
"row_count": 8,
"top_files": [
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md"
]
}
}
```
### evidence_assembled
```json
{
"input": {},
"output": {
"mode": "summary",
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
"document_count": 1
}
}
```
### evidence_gate_checked
```json
{
"input": {},
"output": {
"passed": false,
"reason": "target_doc_missing",
"answer_mode": "insufficient_evidence"
}
}
```
### answer_generated
```json
{
"input": {},
"output": {
"answer_mode": "insufficient_evidence",
"answer_length": 131
}
}
```
## Diagnostics
{}
## Mismatches
- doc_scope should contain 'docs/domains/runtime-health-entity.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md']
- llm: expected answer to contain 'runtime health'
@@ -0,0 +1,143 @@
{
"meta": {
"case_id": "v2-docs-summary-05-notification-loop",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"doc_scope should contain 'docs/logic/telegram-notification-loop.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']",
"llm: expected answer to contain 'worker'",
"llm: expected answer to contain 'interval'",
"llm: expected answer to contain 'degraded'"
],
"actual": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"anchor_type": "LOGIC_FLOW",
"rag_count": 8,
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md, docs/README.md.",
"answer_mode": "insufficient_evidence",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramNotifyModule",
"TelegramNotifyWorker",
"TelegramSendService",
"TelegramControlChannel",
"RuntimeManager"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/logic/",
"docs/architecture/",
"docs/"
],
"prefer_like_patterns": [
"%telegram-notification-loop.md%"
],
"target_doc_hints": [
"docs/logic/telegram-notification-loop.md"
]
}
}
},
"pipeline_steps": [
{
"step": "router_resolved",
"output": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"subintent": "SUMMARY",
"anchor_type": "LOGIC_FLOW",
"confidence": 1.0
}
},
{
"step": "anchors_extracted",
"output": {
"anchor_type": "LOGIC_FLOW",
"endpoint_paths": [],
"target_doc_hints": [
"docs/logic/telegram-notification-loop.md"
],
"matched_aliases": [
"telegram-notification-loop"
]
}
},
{
"step": "retrieval_profile_selected",
"output": {
"profile": "docs_summary_logic_flow",
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/logic/",
"docs/architecture/",
"docs/"
],
"prefer_like_patterns": [
"%telegram-notification-loop.md%"
],
"target_doc_hints": [
"docs/logic/telegram-notification-loop.md"
]
}
}
},
{
"step": "retrieval_executed",
"output": {
"profile": "docs_summary_logic_flow",
"row_count": 8,
"top_files": [
"docs/architecture/telegram-notify-app-overview.md",
"docs/README.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md"
]
}
},
{
"step": "evidence_assembled",
"output": {
"mode": "summary",
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
"document_count": 2
}
},
{
"step": "evidence_gate_checked",
"output": {
"passed": false,
"reason": "target_doc_missing",
"answer_mode": "insufficient_evidence"
}
},
{
"step": "answer_generated",
"output": {
"answer_mode": "insufficient_evidence",
"answer_length": 147
}
}
]
}
@@ -0,0 +1,176 @@
# v2-docs-summary-05-notification-loop
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Кратко опиши цикл отправки уведомлений
## Actual
{
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"anchor_type": "LOGIC_FLOW",
"rag_count": 8,
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md, docs/README.md.",
"answer_mode": "insufficient_evidence",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramNotifyModule",
"TelegramNotifyWorker",
"TelegramSendService",
"TelegramControlChannel",
"RuntimeManager"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/logic/",
"docs/architecture/",
"docs/"
],
"prefer_like_patterns": [
"%telegram-notification-loop.md%"
],
"target_doc_hints": [
"docs/logic/telegram-notification-loop.md"
]
}
}
## Pipeline Steps
### router_resolved
```json
{
"input": {},
"output": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"subintent": "SUMMARY",
"anchor_type": "LOGIC_FLOW",
"confidence": 1.0
}
}
```
### anchors_extracted
```json
{
"input": {},
"output": {
"anchor_type": "LOGIC_FLOW",
"endpoint_paths": [],
"target_doc_hints": [
"docs/logic/telegram-notification-loop.md"
],
"matched_aliases": [
"telegram-notification-loop"
]
}
}
```
### retrieval_profile_selected
```json
{
"input": {},
"output": {
"profile": "docs_summary_logic_flow",
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/logic/",
"docs/architecture/",
"docs/"
],
"prefer_like_patterns": [
"%telegram-notification-loop.md%"
],
"target_doc_hints": [
"docs/logic/telegram-notification-loop.md"
]
}
}
}
```
### retrieval_executed
```json
{
"input": {},
"output": {
"profile": "docs_summary_logic_flow",
"row_count": 8,
"top_files": [
"docs/architecture/telegram-notify-app-overview.md",
"docs/README.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md"
]
}
}
```
### evidence_assembled
```json
{
"input": {},
"output": {
"mode": "summary",
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
"document_count": 2
}
}
```
### evidence_gate_checked
```json
{
"input": {},
"output": {
"passed": false,
"reason": "target_doc_missing",
"answer_mode": "insufficient_evidence"
}
}
```
### answer_generated
```json
{
"input": {},
"output": {
"answer_mode": "insufficient_evidence",
"answer_length": 147
}
}
```
## Diagnostics
{}
## Mismatches
- doc_scope should contain 'docs/logic/telegram-notification-loop.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
- llm: expected answer to contain 'worker'
- llm: expected answer to contain 'interval'
- llm: expected answer to contain 'degraded'
@@ -0,0 +1,124 @@
{
"meta": {
"case_id": "v2-docs-summary-06-architecture",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": true,
"mismatches": [],
"actual": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"anchor_type": "ARCHITECTURE",
"rag_count": 8,
"llm_answer": "- Purpose: сервис поднимает HTTP control plane и фоновый worker для отправки уведомлений в Telegram.\n- Entry point: `src/telegram_notify_app/main.py`.\n- Main components: `RuntimeManager`, `TelegramControlChannel`, `TelegramNotifyModule`, `TelegramNotifyWorker`, `TelegramSendService`.\n- Configuration: `config/config.yaml` или путь из `CONFIG_PATH`.\n- Related API: [`/health`](../api/health-endpoint.md), [`/actions/{action}`](../api/control-actions-endpoint.md), [`/send`](../api/send-message-endpoint.md).\n- Related logic: [цикл отправки уведомлений](../logic/telegram-notification-loop.md).\n- Related domain: [runtime health](../domains/runtime-health-entity.md).\n\nФайлы-источники:\n- docs/architecture/telegram-notify-app-overview.md\n- docs/README.md",
"answer_mode": "grounded_summary",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramNotifyModule",
"TelegramNotifyWorker",
"TelegramSendService",
"TelegramControlChannel",
"RuntimeManager"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/architecture/",
"docs/"
],
"prefer_like_patterns": [],
"target_doc_hints": []
}
}
},
"pipeline_steps": [
{
"step": "router_resolved",
"output": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"subintent": "SUMMARY",
"anchor_type": "ARCHITECTURE",
"confidence": 1.0
}
},
{
"step": "anchors_extracted",
"output": {
"anchor_type": "ARCHITECTURE",
"endpoint_paths": [],
"target_doc_hints": [],
"matched_aliases": []
}
},
{
"step": "retrieval_profile_selected",
"output": {
"profile": "docs_summary_architecture",
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/architecture/",
"docs/"
],
"prefer_like_patterns": [],
"target_doc_hints": []
}
}
},
{
"step": "retrieval_executed",
"output": {
"profile": "docs_summary_architecture",
"row_count": 8,
"top_files": [
"docs/architecture/telegram-notify-app-overview.md",
"docs/README.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md"
]
}
},
{
"step": "evidence_assembled",
"output": {
"mode": "summary",
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
"document_count": 2
}
},
{
"step": "evidence_gate_checked",
"output": {
"passed": true,
"reason": "target_doc_found",
"answer_mode": "grounded_summary"
}
},
{
"step": "answer_generated",
"output": {
"answer_mode": "grounded_summary",
"answer_length": 753
}
}
]
}
@@ -0,0 +1,159 @@
# v2-docs-summary-06-architecture
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: True
## Query
Как устроена архитектура Telegram Notify App?
## Actual
{
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"anchor_type": "ARCHITECTURE",
"rag_count": 8,
"llm_answer": "- Purpose: сервис поднимает HTTP control plane и фоновый worker для отправки уведомлений в Telegram.\n- Entry point: `src/telegram_notify_app/main.py`.\n- Main components: `RuntimeManager`, `TelegramControlChannel`, `TelegramNotifyModule`, `TelegramNotifyWorker`, `TelegramSendService`.\n- Configuration: `config/config.yaml` или путь из `CONFIG_PATH`.\n- Related API: [`/health`](../api/health-endpoint.md), [`/actions/{action}`](../api/control-actions-endpoint.md), [`/send`](../api/send-message-endpoint.md).\n- Related logic: [цикл отправки уведомлений](../logic/telegram-notification-loop.md).\n- Related domain: [runtime health](../domains/runtime-health-entity.md).\n\nФайлы-источники:\n- docs/architecture/telegram-notify-app-overview.md\n- docs/README.md",
"answer_mode": "grounded_summary",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramNotifyModule",
"TelegramNotifyWorker",
"TelegramSendService",
"TelegramControlChannel",
"RuntimeManager"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/architecture/",
"docs/"
],
"prefer_like_patterns": [],
"target_doc_hints": []
}
}
## Pipeline Steps
### router_resolved
```json
{
"input": {},
"output": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"subintent": "SUMMARY",
"anchor_type": "ARCHITECTURE",
"confidence": 1.0
}
}
```
### anchors_extracted
```json
{
"input": {},
"output": {
"anchor_type": "ARCHITECTURE",
"endpoint_paths": [],
"target_doc_hints": [],
"matched_aliases": []
}
}
```
### retrieval_profile_selected
```json
{
"input": {},
"output": {
"profile": "docs_summary_architecture",
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/architecture/",
"docs/"
],
"prefer_like_patterns": [],
"target_doc_hints": []
}
}
}
```
### retrieval_executed
```json
{
"input": {},
"output": {
"profile": "docs_summary_architecture",
"row_count": 8,
"top_files": [
"docs/architecture/telegram-notify-app-overview.md",
"docs/README.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md"
]
}
}
```
### evidence_assembled
```json
{
"input": {},
"output": {
"mode": "summary",
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
"document_count": 2
}
}
```
### evidence_gate_checked
```json
{
"input": {},
"output": {
"passed": true,
"reason": "target_doc_found",
"answer_mode": "grounded_summary"
}
}
```
### answer_generated
```json
{
"input": {},
"output": {
"answer_mode": "grounded_summary",
"answer_length": 753
}
}
```
## Diagnostics
{}
## Mismatches
- none
@@ -0,0 +1,129 @@
{
"meta": {
"case_id": "v2-docs-summary-07-worker",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"doc_scope should contain 'docs/logic/telegram-notification-loop.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']",
"llm: expected answer to contain 'worker'"
],
"actual": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"anchor_type": "LOGIC_FLOW",
"rag_count": 8,
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
"answer_mode": "insufficient_evidence",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramNotifyWorker",
"RuntimeManager",
"TelegramNotifyModule",
"TelegramSendService",
"TelegramControlChannel"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/logic/",
"docs/architecture/",
"docs/"
],
"prefer_like_patterns": [],
"target_doc_hints": []
}
}
},
"pipeline_steps": [
{
"step": "router_resolved",
"output": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"subintent": "SUMMARY",
"anchor_type": "LOGIC_FLOW",
"confidence": 1.0
}
},
{
"step": "anchors_extracted",
"output": {
"anchor_type": "LOGIC_FLOW",
"endpoint_paths": [],
"target_doc_hints": [],
"matched_aliases": []
}
},
{
"step": "retrieval_profile_selected",
"output": {
"profile": "docs_summary_logic_flow",
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/logic/",
"docs/architecture/",
"docs/"
],
"prefer_like_patterns": [],
"target_doc_hints": []
}
}
},
{
"step": "retrieval_executed",
"output": {
"profile": "docs_summary_logic_flow",
"row_count": 8,
"top_files": [
"docs/architecture/telegram-notify-app-overview.md",
"docs/README.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md"
]
}
},
{
"step": "evidence_assembled",
"output": {
"mode": "summary",
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
"document_count": 1
}
},
{
"step": "evidence_gate_checked",
"output": {
"passed": false,
"reason": "target_doc_missing",
"answer_mode": "insufficient_evidence"
}
},
{
"step": "answer_generated",
"output": {
"answer_mode": "insufficient_evidence",
"answer_length": 131
}
}
]
}
@@ -0,0 +1,162 @@
# v2-docs-summary-07-worker
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Что делает worker в этом приложении?
## Actual
{
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"anchor_type": "LOGIC_FLOW",
"rag_count": 8,
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
"answer_mode": "insufficient_evidence",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramNotifyWorker",
"RuntimeManager",
"TelegramNotifyModule",
"TelegramSendService",
"TelegramControlChannel"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/logic/",
"docs/architecture/",
"docs/"
],
"prefer_like_patterns": [],
"target_doc_hints": []
}
}
## Pipeline Steps
### router_resolved
```json
{
"input": {},
"output": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"subintent": "SUMMARY",
"anchor_type": "LOGIC_FLOW",
"confidence": 1.0
}
}
```
### anchors_extracted
```json
{
"input": {},
"output": {
"anchor_type": "LOGIC_FLOW",
"endpoint_paths": [],
"target_doc_hints": [],
"matched_aliases": []
}
}
```
### retrieval_profile_selected
```json
{
"input": {},
"output": {
"profile": "docs_summary_logic_flow",
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/logic/",
"docs/architecture/",
"docs/"
],
"prefer_like_patterns": [],
"target_doc_hints": []
}
}
}
```
### retrieval_executed
```json
{
"input": {},
"output": {
"profile": "docs_summary_logic_flow",
"row_count": 8,
"top_files": [
"docs/architecture/telegram-notify-app-overview.md",
"docs/README.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md"
]
}
}
```
### evidence_assembled
```json
{
"input": {},
"output": {
"mode": "summary",
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
"document_count": 1
}
}
```
### evidence_gate_checked
```json
{
"input": {},
"output": {
"passed": false,
"reason": "target_doc_missing",
"answer_mode": "insufficient_evidence"
}
}
```
### answer_generated
```json
{
"input": {},
"output": {
"answer_mode": "insufficient_evidence",
"answer_length": 131
}
}
```
## Diagnostics
{}
## Mismatches
- doc_scope should contain 'docs/logic/telegram-notification-loop.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
- llm: expected answer to contain 'worker'
@@ -0,0 +1,143 @@
{
"meta": {
"case_id": "v2-docs-summary-08-health-statuses",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"doc_scope should contain 'docs/domains/runtime-health-entity.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md']",
"doc_scope should contain 'docs/api/health-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md']",
"llm: expected answer to contain 'ok'",
"llm: expected answer to contain 'degraded'",
"llm: expected answer to contain 'unhealthy'",
"llm: expected answer to contain 'unknown'"
],
"actual": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"anchor_type": "DOMAIN_ENTITY",
"rag_count": 8,
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
"answer_mode": "insufficient_evidence",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"RuntimeManager",
"TelegramNotifyWorker",
"TelegramSendService",
"TelegramNotifyModule",
"TelegramControlChannel"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/domains/",
"docs/api/",
"docs/architecture/"
],
"prefer_like_patterns": [
"%runtime-health-entity.md%"
],
"target_doc_hints": [
"docs/domains/runtime-health-entity.md"
]
}
}
},
"pipeline_steps": [
{
"step": "router_resolved",
"output": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"subintent": "SUMMARY",
"anchor_type": "DOMAIN_ENTITY",
"confidence": 1.0
}
},
{
"step": "anchors_extracted",
"output": {
"anchor_type": "DOMAIN_ENTITY",
"endpoint_paths": [],
"target_doc_hints": [
"docs/domains/runtime-health-entity.md"
],
"matched_aliases": [
"runtime-health-entity"
]
}
},
{
"step": "retrieval_profile_selected",
"output": {
"profile": "docs_summary_domain_entity",
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/domains/",
"docs/api/",
"docs/architecture/"
],
"prefer_like_patterns": [
"%runtime-health-entity.md%"
],
"target_doc_hints": [
"docs/domains/runtime-health-entity.md"
]
}
}
},
{
"step": "retrieval_executed",
"output": {
"profile": "docs_summary_domain_entity",
"row_count": 8,
"top_files": [
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md"
]
}
},
{
"step": "evidence_assembled",
"output": {
"mode": "summary",
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
"document_count": 1
}
},
{
"step": "evidence_gate_checked",
"output": {
"passed": false,
"reason": "target_doc_missing",
"answer_mode": "insufficient_evidence"
}
},
{
"step": "answer_generated",
"output": {
"answer_mode": "insufficient_evidence",
"answer_length": 131
}
}
]
}
@@ -0,0 +1,176 @@
# v2-docs-summary-08-health-statuses
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Какие статусы здоровья есть у runtime?
## Actual
{
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"anchor_type": "DOMAIN_ENTITY",
"rag_count": 8,
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
"answer_mode": "insufficient_evidence",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"RuntimeManager",
"TelegramNotifyWorker",
"TelegramSendService",
"TelegramNotifyModule",
"TelegramControlChannel"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/domains/",
"docs/api/",
"docs/architecture/"
],
"prefer_like_patterns": [
"%runtime-health-entity.md%"
],
"target_doc_hints": [
"docs/domains/runtime-health-entity.md"
]
}
}
## Pipeline Steps
### router_resolved
```json
{
"input": {},
"output": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"subintent": "SUMMARY",
"anchor_type": "DOMAIN_ENTITY",
"confidence": 1.0
}
}
```
### anchors_extracted
```json
{
"input": {},
"output": {
"anchor_type": "DOMAIN_ENTITY",
"endpoint_paths": [],
"target_doc_hints": [
"docs/domains/runtime-health-entity.md"
],
"matched_aliases": [
"runtime-health-entity"
]
}
}
```
### retrieval_profile_selected
```json
{
"input": {},
"output": {
"profile": "docs_summary_domain_entity",
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/domains/",
"docs/api/",
"docs/architecture/"
],
"prefer_like_patterns": [
"%runtime-health-entity.md%"
],
"target_doc_hints": [
"docs/domains/runtime-health-entity.md"
]
}
}
}
```
### retrieval_executed
```json
{
"input": {},
"output": {
"profile": "docs_summary_domain_entity",
"row_count": 8,
"top_files": [
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md"
]
}
}
```
### evidence_assembled
```json
{
"input": {},
"output": {
"mode": "summary",
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
"document_count": 1
}
}
```
### evidence_gate_checked
```json
{
"input": {},
"output": {
"passed": false,
"reason": "target_doc_missing",
"answer_mode": "insufficient_evidence"
}
}
```
### answer_generated
```json
{
"input": {},
"output": {
"answer_mode": "insufficient_evidence",
"answer_length": 131
}
}
```
## Diagnostics
{}
## Mismatches
- doc_scope should contain 'docs/domains/runtime-health-entity.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md']
- doc_scope should contain 'docs/api/health-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md']
- llm: expected answer to contain 'ok'
- llm: expected answer to contain 'degraded'
- llm: expected answer to contain 'unhealthy'
- llm: expected answer to contain 'unknown'
@@ -0,0 +1,151 @@
{
"meta": {
"case_id": "v2-docs-summary-09-manual-send",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']",
"llm: expected answer to contain '/send'",
"llm: expected answer to contain 'message'",
"llm: expected answer to contain 'TelegramSendService'"
],
"actual": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"anchor_type": "API_ENDPOINT",
"rag_count": 8,
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
"answer_mode": "insufficient_evidence",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramSendService",
"TelegramNotifyWorker",
"RuntimeManager",
"TelegramNotifyModule",
"TelegramControlChannel"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/api/",
"docs/"
],
"prefer_like_patterns": [
"%send-message-endpoint.md%"
],
"target_doc_hints": [
"docs/api/send-message-endpoint.md"
],
"path_prefixes": [
"docs/api/",
"docs/architecture/",
"docs/"
]
}
}
},
"pipeline_steps": [
{
"step": "router_resolved",
"output": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"subintent": "SUMMARY",
"anchor_type": "API_ENDPOINT",
"confidence": 1.0
}
},
{
"step": "anchors_extracted",
"output": {
"anchor_type": "API_ENDPOINT",
"endpoint_paths": [],
"target_doc_hints": [
"docs/api/send-message-endpoint.md"
],
"matched_aliases": [
"/send"
]
}
},
{
"step": "retrieval_profile_selected",
"output": {
"profile": "docs_summary_api_endpoint",
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/api/",
"docs/"
],
"prefer_like_patterns": [
"%send-message-endpoint.md%"
],
"target_doc_hints": [
"docs/api/send-message-endpoint.md"
],
"path_prefixes": [
"docs/api/",
"docs/architecture/",
"docs/"
]
}
}
},
{
"step": "retrieval_executed",
"output": {
"profile": "docs_summary_api_endpoint",
"row_count": 8,
"top_files": [
"docs/architecture/telegram-notify-app-overview.md",
"docs/README.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md"
]
}
},
{
"step": "evidence_assembled",
"output": {
"mode": "summary",
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
"document_count": 1
}
},
{
"step": "evidence_gate_checked",
"output": {
"passed": false,
"reason": "target_doc_missing",
"answer_mode": "insufficient_evidence"
}
},
{
"step": "answer_generated",
"output": {
"answer_mode": "insufficient_evidence",
"answer_length": 131
}
}
]
}
@@ -0,0 +1,184 @@
# v2-docs-summary-09-manual-send
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Как в приложении устроена ручная отправка сообщения?
## Actual
{
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "SUMMARY",
"anchor_type": "API_ENDPOINT",
"rag_count": 8,
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
"answer_mode": "insufficient_evidence",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramSendService",
"TelegramNotifyWorker",
"RuntimeManager",
"TelegramNotifyModule",
"TelegramControlChannel"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/api/",
"docs/"
],
"prefer_like_patterns": [
"%send-message-endpoint.md%"
],
"target_doc_hints": [
"docs/api/send-message-endpoint.md"
],
"path_prefixes": [
"docs/api/",
"docs/architecture/",
"docs/"
]
}
}
## Pipeline Steps
### router_resolved
```json
{
"input": {},
"output": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"subintent": "SUMMARY",
"anchor_type": "API_ENDPOINT",
"confidence": 1.0
}
}
```
### anchors_extracted
```json
{
"input": {},
"output": {
"anchor_type": "API_ENDPOINT",
"endpoint_paths": [],
"target_doc_hints": [
"docs/api/send-message-endpoint.md"
],
"matched_aliases": [
"/send"
]
}
}
```
### retrieval_profile_selected
```json
{
"input": {},
"output": {
"profile": "docs_summary_api_endpoint",
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/api/",
"docs/"
],
"prefer_like_patterns": [
"%send-message-endpoint.md%"
],
"target_doc_hints": [
"docs/api/send-message-endpoint.md"
],
"path_prefixes": [
"docs/api/",
"docs/architecture/",
"docs/"
]
}
}
}
```
### retrieval_executed
```json
{
"input": {},
"output": {
"profile": "docs_summary_api_endpoint",
"row_count": 8,
"top_files": [
"docs/architecture/telegram-notify-app-overview.md",
"docs/README.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md"
]
}
}
```
### evidence_assembled
```json
{
"input": {},
"output": {
"mode": "summary",
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
"document_count": 1
}
}
```
### evidence_gate_checked
```json
{
"input": {},
"output": {
"passed": false,
"reason": "target_doc_missing",
"answer_mode": "insufficient_evidence"
}
}
```
### answer_generated
```json
{
"input": {},
"output": {
"answer_mode": "insufficient_evidence",
"answer_length": 131
}
}
```
## Diagnostics
{}
## Mismatches
- doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
- llm: expected answer to contain '/send'
- llm: expected answer to contain 'message'
- llm: expected answer to contain 'TelegramSendService'
@@ -0,0 +1,127 @@
{
"meta": {
"case_id": "v2-docs-summary-10-telegram-integration",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"domain: expected DOCS, got GENERAL",
"intent: expected DOC_EXPLAIN, got GENERAL_QA",
"anchor_type: expected API_ENDPOINT, got None",
"doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']"
],
"actual": {
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"anchor_type": null,
"rag_count": 8,
"llm_answer": "- Purpose: точка входа в техническую документацию сервиса `test_echo_app`.\n- Scope: архитектура, HTTP API control plane, цикл отправки уведомлений, health-модель и каталог ошибок.\n- Canonical structure: `docs/architecture`, `docs/api`, `docs/logic`, `docs/domains`, `docs/errors`.\n- Primary parent doc: [Архитектура Telegram Notify App](./architecture/telegram-notify-app-overview.md).\n- Navigation: документы связаны через `related_docs`, `parent`/`children` и markdown-ссылки без дублирования деталей.\n- Purpose: сервис поднимает HTTP control plane и фоновый worker для отправки уведомлений в Telegram.\n- Entry point: `src/telegram_notify_app/main.py`.\n- Main components: `RuntimeManager`, `TelegramControlChannel`, `TelegramNotifyModule`, `TelegramNotifyWorker`, `TelegramSendService`.\n- Configuration: `config/config.yaml` или путь из `CONFIG_PATH`.\n- Related API: [`/health`](../api/health-endpoint.md), [`/actions/{action}`](../api/control-actions-endpoint.md), [`/send`](../api/send-message-endpoint.md).\n- Related logic: [цикл отправки уведомлений](../logic/telegram-notification-loop.md).\n- Related domain: [runtime health](../domains/runtime-health-entity.md).",
"answer_mode": "grounded_summary",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/architecture/",
"docs/"
],
"prefer_like_patterns": [
"%README.md%",
"%overview%"
],
"target_doc_hints": []
}
}
},
"pipeline_steps": [
{
"step": "router_resolved",
"output": {
"domain": "GENERAL",
"intent": "GENERAL_QA",
"subintent": "SUMMARY",
"anchor_type": null,
"confidence": 1.0
}
},
{
"step": "anchors_extracted",
"output": {
"anchor_type": null,
"endpoint_paths": [],
"target_doc_hints": [],
"matched_aliases": []
}
},
{
"step": "retrieval_profile_selected",
"output": {
"profile": "general_qa_grounded_summary",
"layers": [
"D1_DOCUMENT_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/architecture/",
"docs/"
],
"prefer_like_patterns": [
"%README.md%",
"%overview%"
],
"target_doc_hints": []
}
}
},
{
"step": "retrieval_executed",
"output": {
"profile": "general_qa_grounded_summary",
"row_count": 8,
"top_files": [
"docs/architecture/telegram-notify-app-overview.md",
"docs/README.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md"
]
}
},
{
"step": "evidence_assembled",
"output": {
"mode": "summary",
"primary_doc": "docs/README.md",
"document_count": 2
}
},
{
"step": "evidence_gate_checked",
"output": {
"passed": true,
"reason": "general_docs_found",
"answer_mode": "grounded_summary"
}
},
{
"step": "answer_generated",
"output": {
"answer_mode": "grounded_summary",
"answer_length": 1170
}
}
]
}
@@ -0,0 +1,160 @@
# v2-docs-summary-10-telegram-integration
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
Как сервис взаимодействует с Telegram в целом?
## Actual
{
"domain": "GENERAL",
"intent": "GENERAL_QA",
"sub_intent": "SUMMARY",
"anchor_type": null,
"rag_count": 8,
"llm_answer": "- Purpose: точка входа в техническую документацию сервиса `test_echo_app`.\n- Scope: архитектура, HTTP API control plane, цикл отправки уведомлений, health-модель и каталог ошибок.\n- Canonical structure: `docs/architecture`, `docs/api`, `docs/logic`, `docs/domains`, `docs/errors`.\n- Primary parent doc: [Архитектура Telegram Notify App](./architecture/telegram-notify-app-overview.md).\n- Navigation: документы связаны через `related_docs`, `parent`/`children` и markdown-ссылки без дублирования деталей.\n- Purpose: сервис поднимает HTTP control plane и фоновый worker для отправки уведомлений в Telegram.\n- Entry point: `src/telegram_notify_app/main.py`.\n- Main components: `RuntimeManager`, `TelegramControlChannel`, `TelegramNotifyModule`, `TelegramNotifyWorker`, `TelegramSendService`.\n- Configuration: `config/config.yaml` или путь из `CONFIG_PATH`.\n- Related API: [`/health`](../api/health-endpoint.md), [`/actions/{action}`](../api/control-actions-endpoint.md), [`/send`](../api/send-message-endpoint.md).\n- Related logic: [цикл отправки уведомлений](../logic/telegram-notification-loop.md).\n- Related domain: [runtime health](../domains/runtime-health-entity.md).",
"answer_mode": "grounded_summary",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/architecture/",
"docs/"
],
"prefer_like_patterns": [
"%README.md%",
"%overview%"
],
"target_doc_hints": []
}
}
## Pipeline Steps
### router_resolved
```json
{
"input": {},
"output": {
"domain": "GENERAL",
"intent": "GENERAL_QA",
"subintent": "SUMMARY",
"anchor_type": null,
"confidence": 1.0
}
}
```
### anchors_extracted
```json
{
"input": {},
"output": {
"anchor_type": null,
"endpoint_paths": [],
"target_doc_hints": [],
"matched_aliases": []
}
}
```
### retrieval_profile_selected
```json
{
"input": {},
"output": {
"profile": "general_qa_grounded_summary",
"layers": [
"D1_DOCUMENT_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/architecture/",
"docs/"
],
"prefer_like_patterns": [
"%README.md%",
"%overview%"
],
"target_doc_hints": []
}
}
}
```
### retrieval_executed
```json
{
"input": {},
"output": {
"profile": "general_qa_grounded_summary",
"row_count": 8,
"top_files": [
"docs/architecture/telegram-notify-app-overview.md",
"docs/README.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md"
]
}
}
```
### evidence_assembled
```json
{
"input": {},
"output": {
"mode": "summary",
"primary_doc": "docs/README.md",
"document_count": 2
}
}
```
### evidence_gate_checked
```json
{
"input": {},
"output": {
"passed": true,
"reason": "general_docs_found",
"answer_mode": "grounded_summary"
}
}
```
### answer_generated
```json
{
"input": {},
"output": {
"answer_mode": "grounded_summary",
"answer_length": 1170
}
}
```
## Diagnostics
{}
## Mismatches
- domain: expected DOCS, got GENERAL
- intent: expected DOC_EXPLAIN, got GENERAL_QA
- anchor_type: expected API_ENDPOINT, got None
- doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
@@ -0,0 +1,137 @@
{
"meta": {
"case_id": "v2-find-files-01-health",
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
"runner": "process_v2",
"mode": "full_chain",
"passed": false,
"mismatches": [
"answer_mode: expected deterministic, got deterministic_file_list",
"llm: expected answer to contain 'docs/api/health-endpoint.md'"
],
"actual": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "FIND_FILES",
"anchor_type": "FIND_FILES",
"rag_count": 12,
"llm_answer": "docs/architecture/telegram-notify-app-overview.md\ndocs/README.md",
"answer_mode": "deterministic_file_list",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramNotifyModule",
"RuntimeManager",
"TelegramControlChannel",
"TelegramNotifyWorker",
"TelegramSendService"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/api/"
],
"prefer_like_patterns": [
"%health-endpoint.md%"
],
"target_doc_hints": [
"docs/api/health-endpoint.md"
]
}
}
},
"pipeline_steps": [
{
"step": "router_resolved",
"output": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"subintent": "FIND_FILES",
"anchor_type": "FIND_FILES",
"confidence": 1.0
}
},
{
"step": "anchors_extracted",
"output": {
"anchor_type": null,
"endpoint_paths": [
"/health"
],
"target_doc_hints": [
"docs/api/health-endpoint.md"
],
"matched_aliases": []
}
},
{
"step": "retrieval_profile_selected",
"output": {
"profile": "docs_find_files_catalog",
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/api/"
],
"prefer_like_patterns": [
"%health-endpoint.md%"
],
"target_doc_hints": [
"docs/api/health-endpoint.md"
]
}
}
},
{
"step": "retrieval_executed",
"output": {
"profile": "docs_find_files_catalog",
"row_count": 12,
"top_files": [
"docs/architecture/telegram-notify-app-overview.md",
"docs/README.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md"
]
}
},
{
"step": "evidence_assembled",
"output": {
"mode": "find_files",
"primary_file": "docs/architecture/telegram-notify-app-overview.md",
"file_count": 2
}
},
{
"step": "evidence_gate_checked",
"output": {
"passed": true,
"reason": "primary_file_confident",
"answer_mode": "deterministic_file_list"
}
},
{
"step": "answer_generated",
"output": {
"answer_mode": "deterministic_file_list",
"answer_length": 64
}
}
]
}
@@ -0,0 +1,170 @@
# v2-find-files-01-health
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
- runner: process_v2
- mode: full_chain
- passed: False
## Query
В каком файле описан `/health`?
## Actual
{
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"sub_intent": "FIND_FILES",
"anchor_type": "FIND_FILES",
"rag_count": 12,
"llm_answer": "docs/architecture/telegram-notify-app-overview.md\ndocs/README.md",
"answer_mode": "deterministic_file_list",
"path_scope": [],
"symbol_candidates": [],
"entity_candidates": [
"TelegramNotifyModule",
"RuntimeManager",
"TelegramControlChannel",
"TelegramNotifyWorker",
"TelegramSendService"
],
"doc_scope": [
"architecture.telegram_notify_app",
"docs/architecture/telegram-notify-app-overview.md",
"index.test_echo_app_docs",
"docs/README.md"
],
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/api/"
],
"prefer_like_patterns": [
"%health-endpoint.md%"
],
"target_doc_hints": [
"docs/api/health-endpoint.md"
]
}
}
## Pipeline Steps
### router_resolved
```json
{
"input": {},
"output": {
"domain": "DOCS",
"intent": "DOC_EXPLAIN",
"subintent": "FIND_FILES",
"anchor_type": "FIND_FILES",
"confidence": 1.0
}
}
```
### anchors_extracted
```json
{
"input": {},
"output": {
"anchor_type": null,
"endpoint_paths": [
"/health"
],
"target_doc_hints": [
"docs/api/health-endpoint.md"
],
"matched_aliases": []
}
}
```
### retrieval_profile_selected
```json
{
"input": {},
"output": {
"profile": "docs_find_files_catalog",
"layers": [
"D1_DOCUMENT_CATALOG",
"D3_ENTITY_CATALOG",
"D0_DOC_CHUNKS"
],
"filters": {
"prefer_path_prefixes": [
"docs/api/"
],
"prefer_like_patterns": [
"%health-endpoint.md%"
],
"target_doc_hints": [
"docs/api/health-endpoint.md"
]
}
}
}
```
### retrieval_executed
```json
{
"input": {},
"output": {
"profile": "docs_find_files_catalog",
"row_count": 12,
"top_files": [
"docs/architecture/telegram-notify-app-overview.md",
"docs/README.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md",
"docs/architecture/telegram-notify-app-overview.md"
]
}
}
```
### evidence_assembled
```json
{
"input": {},
"output": {
"mode": "find_files",
"primary_file": "docs/architecture/telegram-notify-app-overview.md",
"file_count": 2
}
}
```
### evidence_gate_checked
```json
{
"input": {},
"output": {
"passed": true,
"reason": "primary_file_confident",
"answer_mode": "deterministic_file_list"
}
}
```
### answer_generated
```json
{
"input": {},
"output": {
"answer_mode": "deterministic_file_list",
"answer_length": 64
}
}
```
## Diagnostics
{}
## Mismatches
- answer_mode: expected deterministic, got deterministic_file_list
- llm: expected answer to contain 'docs/api/health-endpoint.md'

Some files were not shown because too many files have changed in this diff Show More