Роутер работает нормально в process v2
This commit is contained in:
@@ -0,0 +1,421 @@
|
||||
defaults:
|
||||
runner: process_v2
|
||||
mode: full_chain
|
||||
input:
|
||||
rag_session_id: "fdf3ff03-81f0-4772-b68e-250147960243"
|
||||
|
||||
cases:
|
||||
- id: v2-general-01-what-is-service
|
||||
query: "Что это за сервис?"
|
||||
expected:
|
||||
router:
|
||||
domain: GENERAL
|
||||
intent: GENERAL_QA
|
||||
sub_intent: SUMMARY
|
||||
pipeline:
|
||||
answer_mode: grounded_summary
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["Telegram", "сервис"]
|
||||
|
||||
- id: v2-general-02-purpose
|
||||
query: "Для чего нужен test_echo_app?"
|
||||
expected:
|
||||
router:
|
||||
domain: GENERAL
|
||||
intent: GENERAL_QA
|
||||
sub_intent: SUMMARY
|
||||
pipeline:
|
||||
answer_mode: grounded_summary
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["test_echo_app"]
|
||||
|
||||
- id: v2-general-03-problem
|
||||
query: "Какую задачу решает это приложение?"
|
||||
expected:
|
||||
router:
|
||||
domain: GENERAL
|
||||
intent: GENERAL_QA
|
||||
sub_intent: SUMMARY
|
||||
pipeline:
|
||||
answer_mode: grounded_summary
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["Telegram"]
|
||||
|
||||
- id: v2-general-04-docs-contents
|
||||
query: "Что входит в документацию этого проекта?"
|
||||
expected:
|
||||
router:
|
||||
domain: GENERAL
|
||||
intent: GENERAL_QA
|
||||
sub_intent: SUMMARY
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["документац", "архитект"]
|
||||
|
||||
- id: v2-general-05-main-parts
|
||||
query: "Какие основные части есть у системы?"
|
||||
expected:
|
||||
router:
|
||||
domain: GENERAL
|
||||
intent: GENERAL_QA
|
||||
sub_intent: SUMMARY
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["систем", "Telegram"]
|
||||
|
||||
- id: v2-general-06-high-level-architecture
|
||||
query: "Как в целом устроено приложение?"
|
||||
expected:
|
||||
router:
|
||||
domain: GENERAL
|
||||
intent: GENERAL_QA
|
||||
sub_intent: SUMMARY
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["приложен"]
|
||||
|
||||
- id: v2-general-07-besides-api
|
||||
query: "Что тут есть кроме API?"
|
||||
expected:
|
||||
router:
|
||||
domain: GENERAL
|
||||
intent: GENERAL_QA
|
||||
sub_intent: SUMMARY
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["API"]
|
||||
|
||||
- id: v2-general-08-components
|
||||
query: "Из чего состоит Telegram Notify App?"
|
||||
expected:
|
||||
router:
|
||||
domain: GENERAL
|
||||
intent: GENERAL_QA
|
||||
sub_intent: SUMMARY
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["Telegram Notify App"]
|
||||
|
||||
- id: v2-general-09-reading-order
|
||||
query: "Какие документы стоит читать сначала?"
|
||||
expected:
|
||||
router:
|
||||
domain: GENERAL
|
||||
intent: GENERAL_QA
|
||||
sub_intent: SUMMARY
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["README", "архитект"]
|
||||
|
||||
- id: v2-general-10-short-summary
|
||||
query: "Дай короткое summary по проектной документации"
|
||||
expected:
|
||||
router:
|
||||
domain: GENERAL
|
||||
intent: GENERAL_QA
|
||||
sub_intent: SUMMARY
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["документац"]
|
||||
|
||||
- id: v2-docs-summary-01-health
|
||||
query: "Кратко объясни по документации, как работает `/health`"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: SUMMARY
|
||||
retrieval:
|
||||
non_empty: true
|
||||
layers_include: ["D1_DOCUMENT_CATALOG", "D3_ENTITY_CATALOG", "D0_DOC_CHUNKS"]
|
||||
doc_scope_contains: ["docs/README.md", "docs/architecture/telegram-notify-app-overview.md"]
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["/health", "status", "components"]
|
||||
pipeline:
|
||||
answer_mode: grounded_summary
|
||||
|
||||
- id: v2-docs-summary-02-send
|
||||
query: "Что делает endpoint `/send`?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: SUMMARY
|
||||
retrieval:
|
||||
non_empty: true
|
||||
layers_include: ["D1_DOCUMENT_CATALOG", "D3_ENTITY_CATALOG", "D0_DOC_CHUNKS"]
|
||||
doc_scope_contains: ["docs/api/send-message-endpoint.md"]
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["/send", "message", "Telegram"]
|
||||
|
||||
- id: v2-docs-summary-03-actions
|
||||
query: "Объясни назначение `/actions/{action}`"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: SUMMARY
|
||||
retrieval:
|
||||
non_empty: true
|
||||
doc_scope_contains: ["docs/api/control-actions-endpoint.md"]
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["/actions/{action}"]
|
||||
|
||||
- id: v2-docs-summary-04-runtime-health
|
||||
query: "Что такое runtime health в этой документации?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: SUMMARY
|
||||
retrieval:
|
||||
non_empty: true
|
||||
doc_scope_contains: ["docs/domains/runtime-health-entity.md"]
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["runtime health"]
|
||||
|
||||
- id: v2-docs-summary-05-notification-loop
|
||||
query: "Кратко опиши цикл отправки уведомлений"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: SUMMARY
|
||||
retrieval:
|
||||
non_empty: true
|
||||
doc_scope_contains: ["docs/logic/telegram-notification-loop.md"]
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["worker", "interval", "degraded"]
|
||||
|
||||
- id: v2-docs-summary-06-architecture
|
||||
query: "Как устроена архитектура Telegram Notify App?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: SUMMARY
|
||||
retrieval:
|
||||
non_empty: true
|
||||
doc_scope_contains: ["docs/architecture/telegram-notify-app-overview.md"]
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["RuntimeManager", "TelegramControlChannel"]
|
||||
|
||||
- id: v2-docs-summary-07-worker
|
||||
query: "Что делает worker в этом приложении?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: SUMMARY
|
||||
retrieval:
|
||||
non_empty: true
|
||||
doc_scope_contains: ["docs/logic/telegram-notification-loop.md", "docs/architecture/telegram-notify-app-overview.md"]
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["worker", "Telegram"]
|
||||
|
||||
- id: v2-docs-summary-08-health-statuses
|
||||
query: "Какие статусы здоровья есть у runtime?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: SUMMARY
|
||||
retrieval:
|
||||
non_empty: true
|
||||
doc_scope_contains: ["docs/domains/runtime-health-entity.md", "docs/api/health-endpoint.md"]
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["ok", "degraded", "unhealthy", "unknown"]
|
||||
|
||||
- id: v2-docs-summary-09-manual-send
|
||||
query: "Как в приложении устроена ручная отправка сообщения?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: SUMMARY
|
||||
retrieval:
|
||||
non_empty: true
|
||||
doc_scope_contains: ["docs/api/send-message-endpoint.md"]
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["/send", "message", "TelegramSendService"]
|
||||
|
||||
- id: v2-docs-summary-10-telegram-integration
|
||||
query: "Как сервис взаимодействует с Telegram в целом?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: SUMMARY
|
||||
retrieval:
|
||||
non_empty: true
|
||||
min_rows: 2
|
||||
doc_scope_contains: ["docs/architecture/telegram-notify-app-overview.md", "docs/api/send-message-endpoint.md"]
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["Telegram", "worker", "/send"]
|
||||
|
||||
- id: v2-find-files-01-health
|
||||
query: "В каком файле описан `/health`?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: FIND_FILES
|
||||
retrieval:
|
||||
non_empty: true
|
||||
pipeline:
|
||||
answer_mode: deterministic
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["docs/api/health-endpoint.md"]
|
||||
|
||||
- id: v2-find-files-02-send
|
||||
query: "Покажи файл с описанием endpoint `/send`"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: FIND_FILES
|
||||
retrieval:
|
||||
non_empty: true
|
||||
pipeline:
|
||||
answer_mode: deterministic
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["docs/api/send-message-endpoint.md"]
|
||||
|
||||
- id: v2-find-files-03-actions
|
||||
query: "Где находится документация по `/actions/{action}`?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: FIND_FILES
|
||||
retrieval:
|
||||
non_empty: true
|
||||
pipeline:
|
||||
answer_mode: deterministic
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["docs/api/control-actions-endpoint.md"]
|
||||
|
||||
- id: v2-find-files-04-runtime-health
|
||||
query: "В каком документе описан runtime health?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: FIND_FILES
|
||||
retrieval:
|
||||
non_empty: true
|
||||
pipeline:
|
||||
answer_mode: deterministic
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["docs/domains/runtime-health-entity.md"]
|
||||
|
||||
- id: v2-find-files-05-loop
|
||||
query: "Найди файл с логикой цикла отправки уведомлений"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: FIND_FILES
|
||||
retrieval:
|
||||
non_empty: true
|
||||
pipeline:
|
||||
answer_mode: deterministic
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["docs/logic/telegram-notification-loop.md"]
|
||||
|
||||
- id: v2-find-files-06-architecture
|
||||
query: "Где описана архитектура приложения?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: FIND_FILES
|
||||
retrieval:
|
||||
non_empty: true
|
||||
pipeline:
|
||||
answer_mode: deterministic
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["docs/architecture/telegram-notify-app-overview.md"]
|
||||
|
||||
- id: v2-find-files-07-errors-catalog
|
||||
query: "В каком файле лежит каталог ошибок?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: FIND_FILES
|
||||
retrieval:
|
||||
non_empty: true
|
||||
pipeline:
|
||||
answer_mode: deterministic
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["docs/errors/catalog.yaml"]
|
||||
|
||||
- id: v2-find-files-08-docs-index
|
||||
query: "Покажи файл-индекс документации проекта"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: FIND_FILES
|
||||
retrieval:
|
||||
non_empty: true
|
||||
pipeline:
|
||||
answer_mode: deterministic
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all: ["docs/README.md"]
|
||||
|
||||
- id: v2-find-files-09-telegram-bot-api
|
||||
query: "Какие файлы относятся к Telegram Bot API интеграции?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: FIND_FILES
|
||||
retrieval:
|
||||
non_empty: true
|
||||
min_rows: 2
|
||||
pipeline:
|
||||
answer_mode: deterministic
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all:
|
||||
["docs/architecture/telegram-notify-app-overview.md", "docs/api/send-message-endpoint.md", "docs/logic/telegram-notification-loop.md"]
|
||||
|
||||
- id: v2-find-files-10-telegram-notify-worker
|
||||
query: "В каких файлах описан worker `telegram_notify`?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: FIND_FILES
|
||||
retrieval:
|
||||
non_empty: true
|
||||
min_rows: 2
|
||||
pipeline:
|
||||
answer_mode: deterministic
|
||||
llm:
|
||||
non_empty: true
|
||||
contains_all:
|
||||
["docs/architecture/telegram-notify-app-overview.md", "docs/logic/telegram-notification-loop.md", "docs/domains/runtime-health-entity.md"]
|
||||
+244
@@ -0,0 +1,244 @@
|
||||
defaults:
|
||||
runner: process_v2
|
||||
mode: router_only
|
||||
|
||||
cases:
|
||||
- id: v2-router-general-01-what-is-service
|
||||
query: "Что это за сервис?"
|
||||
expected:
|
||||
router:
|
||||
domain: GENERAL
|
||||
intent: GENERAL_QA
|
||||
sub_intent: SUMMARY
|
||||
|
||||
- id: v2-router-general-02-purpose
|
||||
query: "Для чего нужен test_echo_app?"
|
||||
expected:
|
||||
router:
|
||||
domain: GENERAL
|
||||
intent: GENERAL_QA
|
||||
sub_intent: SUMMARY
|
||||
|
||||
- id: v2-router-general-03-problem
|
||||
query: "Какую задачу решает это приложение?"
|
||||
expected:
|
||||
router:
|
||||
domain: GENERAL
|
||||
intent: GENERAL_QA
|
||||
sub_intent: SUMMARY
|
||||
|
||||
- id: v2-router-general-04-docs-contents
|
||||
query: "Что входит в документацию этого проекта?"
|
||||
expected:
|
||||
router:
|
||||
domain: GENERAL
|
||||
intent: GENERAL_QA
|
||||
sub_intent: SUMMARY
|
||||
|
||||
- id: v2-router-general-05-main-parts
|
||||
query: "Какие основные части есть у системы?"
|
||||
expected:
|
||||
router:
|
||||
domain: GENERAL
|
||||
intent: GENERAL_QA
|
||||
sub_intent: SUMMARY
|
||||
|
||||
- id: v2-router-general-06-high-level-architecture
|
||||
query: "Как в целом устроено приложение?"
|
||||
expected:
|
||||
router:
|
||||
domain: GENERAL
|
||||
intent: GENERAL_QA
|
||||
sub_intent: SUMMARY
|
||||
|
||||
- id: v2-router-general-07-besides-api
|
||||
query: "Что тут есть кроме API?"
|
||||
expected:
|
||||
router:
|
||||
domain: GENERAL
|
||||
intent: GENERAL_QA
|
||||
sub_intent: SUMMARY
|
||||
|
||||
- id: v2-router-general-08-components
|
||||
query: "Из чего состоит Telegram Notify App?"
|
||||
expected:
|
||||
router:
|
||||
domain: GENERAL
|
||||
intent: GENERAL_QA
|
||||
sub_intent: SUMMARY
|
||||
|
||||
- id: v2-router-general-09-reading-order
|
||||
query: "Какие документы стоит читать сначала?"
|
||||
expected:
|
||||
router:
|
||||
domain: GENERAL
|
||||
intent: GENERAL_QA
|
||||
sub_intent: SUMMARY
|
||||
|
||||
- id: v2-router-general-10-short-summary
|
||||
query: "Дай короткое summary по проектной документации"
|
||||
expected:
|
||||
router:
|
||||
domain: GENERAL
|
||||
intent: GENERAL_QA
|
||||
sub_intent: SUMMARY
|
||||
|
||||
- id: v2-router-docs-summary-01-health
|
||||
query: "Кратко объясни по документации, как работает `/health`"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: SUMMARY
|
||||
|
||||
- id: v2-router-docs-summary-02-send
|
||||
query: "Что делает endpoint `/send`?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: SUMMARY
|
||||
|
||||
- id: v2-router-docs-summary-03-actions
|
||||
query: "Объясни назначение `/actions/{action}`"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: SUMMARY
|
||||
|
||||
- id: v2-router-docs-summary-04-runtime-health
|
||||
query: "Что такое runtime health в этой документации?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: SUMMARY
|
||||
|
||||
- id: v2-router-docs-summary-05-notification-loop
|
||||
query: "Кратко опиши цикл отправки уведомлений"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: SUMMARY
|
||||
|
||||
- id: v2-router-docs-summary-06-architecture
|
||||
query: "Как устроена архитектура Telegram Notify App?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: SUMMARY
|
||||
|
||||
- id: v2-router-docs-summary-07-worker
|
||||
query: "Что делает worker в этом приложении?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: SUMMARY
|
||||
|
||||
- id: v2-router-docs-summary-08-health-statuses
|
||||
query: "Какие статусы здоровья есть у runtime?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: SUMMARY
|
||||
|
||||
- id: v2-router-docs-summary-09-manual-send
|
||||
query: "Как в приложении устроена ручная отправка сообщения?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: SUMMARY
|
||||
|
||||
- id: v2-router-docs-summary-10-telegram-integration
|
||||
query: "Как сервис взаимодействует с Telegram в целом?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: SUMMARY
|
||||
|
||||
- id: v2-router-find-files-01-health
|
||||
query: "В каком файле описан `/health`?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: FIND_FILES
|
||||
|
||||
- id: v2-router-find-files-02-send
|
||||
query: "Покажи файл с описанием endpoint `/send`"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: FIND_FILES
|
||||
|
||||
- id: v2-router-find-files-03-actions
|
||||
query: "Где находится документация по `/actions/{action}`?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: FIND_FILES
|
||||
|
||||
- id: v2-router-find-files-04-runtime-health
|
||||
query: "В каком документе описан runtime health?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: FIND_FILES
|
||||
|
||||
- id: v2-router-find-files-05-loop
|
||||
query: "Найди файл с логикой цикла отправки уведомлений"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: FIND_FILES
|
||||
|
||||
- id: v2-router-find-files-06-architecture
|
||||
query: "Где описана архитектура приложения?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: FIND_FILES
|
||||
|
||||
- id: v2-router-find-files-07-errors-catalog
|
||||
query: "В каком файле лежит каталог ошибок?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: FIND_FILES
|
||||
|
||||
- id: v2-router-find-files-08-docs-index
|
||||
query: "Покажи файл-индекс документации проекта"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: FIND_FILES
|
||||
|
||||
- id: v2-router-find-files-09-telegram-bot-api
|
||||
query: "Какие файлы относятся к Telegram Bot API интеграции?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: FIND_FILES
|
||||
|
||||
- id: v2-router-find-files-10-telegram-notify-worker
|
||||
query: "В каких файлах описан worker `telegram_notify`?"
|
||||
expected:
|
||||
router:
|
||||
domain: DOCS
|
||||
intent: DOC_EXPLAIN
|
||||
sub_intent: FIND_FILES
|
||||
@@ -10,7 +10,7 @@ _src = _agent_root / "src"
|
||||
if _src.exists() and str(_src) not in sys.path:
|
||||
sys.path.insert(0, str(_src))
|
||||
|
||||
from app.modules.rag.persistence.repository import RagRepository
|
||||
from app.core.rag.persistence.repository import RagRepository
|
||||
from tests.pipeline_setup_v3.shared.env_loader import load_pipeline_setup_env
|
||||
from tests.pipeline_setup_v3.shared.rag_indexer import RagSessionIndexer
|
||||
|
||||
|
||||
@@ -80,6 +80,7 @@ class CaseDirectoryLoader:
|
||||
pipeline = dict(raw.get("pipeline") or {})
|
||||
return CaseExpectations(
|
||||
router=RouterExpectation(
|
||||
domain=str(router.get("domain") or "").strip() or None,
|
||||
intent=str(router.get("intent") or "").strip() or None,
|
||||
sub_intent=str(router.get("sub_intent") or "").strip() or None,
|
||||
graph_id=str(router.get("graph_id") or "").strip() or None,
|
||||
@@ -114,4 +115,6 @@ class CaseDirectoryLoader:
|
||||
def _normalize_runner(self, value: str) -> str:
|
||||
if value in {"agent_runtime", "runtime", "code_qa_eval"}:
|
||||
return "agent_runtime"
|
||||
if value in {"process_v2", "v2_process", "v2"}:
|
||||
return "process_v2"
|
||||
return value
|
||||
|
||||
@@ -5,7 +5,7 @@ from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
|
||||
RunnerKind = Literal["agent_runtime"]
|
||||
RunnerKind = Literal["agent_runtime", "process_v2"]
|
||||
ModeKind = Literal["router_only", "router_rag", "full_chain"]
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ class CaseInput:
|
||||
|
||||
@dataclass(slots=True, frozen=True)
|
||||
class RouterExpectation:
|
||||
domain: str | None = None
|
||||
intent: str | None = None
|
||||
sub_intent: str | None = None
|
||||
graph_id: str | None = None
|
||||
|
||||
@@ -19,10 +19,12 @@ class V3Runner:
|
||||
*,
|
||||
pipeline_mode: str = "full",
|
||||
router_llm_mode: str = "deterministic",
|
||||
workflow_llm_enabled: bool = True,
|
||||
) -> None:
|
||||
self._cases_dir = cases_dir
|
||||
self._pipeline_mode = pipeline_mode
|
||||
self._router_llm_mode = router_llm_mode
|
||||
self._workflow_llm_enabled = workflow_llm_enabled
|
||||
self._validator = CaseValidator()
|
||||
self._sessions = RagSessionProvider()
|
||||
self._agent_runtime = None
|
||||
@@ -50,9 +52,11 @@ class V3Runner:
|
||||
return results, self._writer.write_summary(results)
|
||||
|
||||
def _execute(self, case, rag_session_id):
|
||||
if case.runner != "agent_runtime":
|
||||
raise ValueError(f"Unsupported runner: {case.runner}")
|
||||
return self._agent_runtime_adapter().execute(case, rag_session_id)
|
||||
if case.runner == "agent_runtime":
|
||||
return self._agent_runtime_adapter().execute(case, rag_session_id)
|
||||
if case.runner == "process_v2":
|
||||
return self._v2_process_adapter().execute(case, rag_session_id)
|
||||
raise ValueError(f"Unsupported runner: {case.runner}")
|
||||
|
||||
def _agent_runtime_adapter(self):
|
||||
if self._agent_runtime is None:
|
||||
@@ -63,3 +67,10 @@ class V3Runner:
|
||||
router_llm_mode=self._router_llm_mode,
|
||||
)
|
||||
return self._agent_runtime
|
||||
|
||||
def _v2_process_adapter(self):
|
||||
if not hasattr(self, "_v2_process"):
|
||||
from tests.pipeline_setup_v3.runtime.v2_process_adapter import V2ProcessAdapter
|
||||
|
||||
self._v2_process = V2ProcessAdapter(workflow_llm_enabled=self._workflow_llm_enabled)
|
||||
return self._v2_process
|
||||
|
||||
@@ -24,7 +24,7 @@ class RagSessionProvider:
|
||||
return self._cache[key]
|
||||
|
||||
def _build_indexer(self):
|
||||
from app.modules.rag.persistence.repository import RagRepository
|
||||
from app.core.rag.persistence.repository import RagRepository
|
||||
from tests.pipeline_setup_v3.shared.rag_indexer import RagSessionIndexer
|
||||
|
||||
if self._repository is None:
|
||||
|
||||
@@ -6,6 +6,7 @@ from tests.pipeline_setup_v3.core.models import V3Case
|
||||
class CaseValidator:
|
||||
def validate(self, case: V3Case, actual: dict, details: dict | None = None) -> list[str]:
|
||||
mismatches: list[str] = []
|
||||
self._check(case.expectations.router.domain, actual.get("domain"), "domain", mismatches)
|
||||
self._check(case.expectations.router.intent, actual.get("intent"), "intent", mismatches)
|
||||
self._check(case.expectations.router.sub_intent, actual.get("sub_intent"), "sub_intent", mismatches)
|
||||
self._check(case.expectations.router.graph_id, actual.get("graph_id"), "graph_id", mismatches)
|
||||
|
||||
@@ -21,6 +21,12 @@ def main(argv: list[str] | None = None) -> int:
|
||||
parser.add_argument("--run-name", default="manual_run", help="Output directory prefix")
|
||||
parser.add_argument("--results-dir", default=None, help="Override results root directory")
|
||||
parser.add_argument("--pipeline-mode", default="full", choices=["full", "pre_llm_only"], help="Pipeline execution mode")
|
||||
parser.add_argument(
|
||||
"--workflow-llm-enabled",
|
||||
default="true",
|
||||
choices=["true", "false"],
|
||||
help="Enable or disable final workflow LLM calls where supported",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--router-llm-mode",
|
||||
default="deterministic",
|
||||
@@ -39,6 +45,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
run_name=str(ns.run_name).strip() or "manual_run",
|
||||
pipeline_mode=str(ns.pipeline_mode).strip() or "full",
|
||||
router_llm_mode=str(ns.router_llm_mode).strip() or "deterministic",
|
||||
workflow_llm_enabled=str(ns.workflow_llm_enabled).strip().lower() == "true",
|
||||
)
|
||||
print(f"Cases dir: {cases_dir}")
|
||||
print(f"Run dir: {runner.run_dir}")
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
from tests.pipeline_setup_v3.runtime.agent_runtime_adapter import AgentRuntimeAdapter
|
||||
|
||||
__all__ = ["AgentRuntimeAdapter"]
|
||||
__all__ = [
|
||||
"agent_runtime_adapter",
|
||||
"v2_process_adapter",
|
||||
]
|
||||
|
||||
@@ -2,20 +2,20 @@ from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
from app.modules.agent.runtime import (
|
||||
from app.core.agent.runtime import (
|
||||
AgentRuntimeExecutor,
|
||||
DocsQAPipelineRunner,
|
||||
RuntimeRepoContextFactory,
|
||||
RuntimeRetrievalAdapter,
|
||||
)
|
||||
from app.modules.agent.llm import AgentLlmService
|
||||
from app.modules.agent.llm.prompt_loader import PromptLoader
|
||||
from app.modules.agent.runtime.steps.context import build_retrieval_request, build_retrieval_result
|
||||
from app.modules.agent.intent_router_v2 import ConversationState, IntentRouterV2
|
||||
from app.modules.agent.intent_router_v2.factory import GigaChatIntentRouterFactory
|
||||
from app.modules.shared.gigachat.client import GigaChatClient
|
||||
from app.modules.shared.gigachat.settings import GigaChatSettings
|
||||
from app.modules.shared.gigachat.token_provider import GigaChatTokenProvider
|
||||
from app.core.agent.llm import AgentLlmService
|
||||
from app.core.agent.llm.prompt_loader import PromptLoader
|
||||
from app.core.agent.runtime.steps.context import build_retrieval_request, build_retrieval_result
|
||||
from app.core.agent.intent_router import ConversationState, IntentRouterV2
|
||||
from app.core.agent.intent_router.factory import GigaChatIntentRouterFactory
|
||||
from app.core.shared.gigachat.client import GigaChatClient
|
||||
from app.core.shared.gigachat.settings import GigaChatSettings
|
||||
from app.core.shared.gigachat.token_provider import GigaChatTokenProvider
|
||||
from tests.pipeline_setup_v3.core.models import ExecutionPayload, V3Case
|
||||
|
||||
|
||||
@@ -190,6 +190,7 @@ class AgentRuntimeAdapter:
|
||||
) -> dict:
|
||||
route_dump = route.model_dump(mode="json") if route else {}
|
||||
return {
|
||||
"domain": route_dump.get("routing_domain") or route_dump.get("domain") or _infer_domain(route_dump.get("intent")),
|
||||
"intent": route_dump.get("intent"),
|
||||
"sub_intent": dict(route_dump.get("query_plan") or {}).get("sub_intent"),
|
||||
"graph_id": route_dump.get("graph_id"),
|
||||
@@ -262,6 +263,7 @@ class AgentRuntimeAdapter:
|
||||
entity_candidates = tuple(diagnostics.query_entity_candidates or self._docs_entity_candidates(result.raw_rows))
|
||||
doc_scope = tuple(diagnostics.doc_ids or self._docs_scope(result.raw_rows))
|
||||
return {
|
||||
"domain": getattr(route, "routing_domain", None) or getattr(route, "domain", None) or _infer_domain(route.intent),
|
||||
"intent": route.intent,
|
||||
"sub_intent": route.query_plan.sub_intent,
|
||||
"graph_id": route.graph_id,
|
||||
@@ -287,6 +289,7 @@ class AgentRuntimeAdapter:
|
||||
|
||||
def _stubbed_code_payload(self, case: V3Case, route) -> ExecutionPayload:
|
||||
actual = {
|
||||
"domain": _infer_domain(route.intent),
|
||||
"intent": route.intent,
|
||||
"sub_intent": route.query_plan.sub_intent,
|
||||
"graph_id": route.graph_id,
|
||||
@@ -493,3 +496,12 @@ def _input_step(query: str) -> dict:
|
||||
"input": {"query": query},
|
||||
"output": {"query": query},
|
||||
}
|
||||
|
||||
|
||||
def _infer_domain(intent: str | None) -> str | None:
|
||||
value = str(intent or "").strip()
|
||||
if value == "GENERAL_QA":
|
||||
return "GENERAL"
|
||||
if value in {"DOCUMENTATION_EXPLAIN", "DOC_EXPLAIN"}:
|
||||
return "DOCS"
|
||||
return None
|
||||
|
||||
@@ -0,0 +1,420 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from app.core.agent.processes.v2.anchor_signals import route_anchor_summary
|
||||
from app.core.agent.processes.v2 import V2IntentRouter
|
||||
from app.core.agent.processes.v2.evidence.assembler import DocsEvidenceAssembler
|
||||
from app.core.agent.processes.v2.evidence.gate import DocsEvidenceGate
|
||||
from app.core.agent.processes.v2.models import RetrievedFile, RetrievedSummary, V2Intent, V2Subintent
|
||||
from app.core.agent.processes.v2.retrieval import DocsMetadataLookupIndex
|
||||
from app.core.agent.processes.v2.retrieval.policy_resolver import V2RetrievalPolicyResolver
|
||||
from app.core.agent.processes.v2.retrieval.v2_rag_adapter import V2RagRetrievalAdapter
|
||||
from app.core.agent.processes.v2.workflows.docs_explain_find_files.context import DocsExplainFindFilesContext
|
||||
from app.core.agent.processes.v2.workflows.docs_explain_find_files.graph import DocsExplainFindFilesGraph
|
||||
from app.core.agent.processes.v2.workflows.docs_explain_summary.context import DocsExplainSummaryContext
|
||||
from app.core.agent.processes.v2.workflows.docs_explain_summary.graph import DocsExplainSummaryGraph
|
||||
from app.core.agent.processes.v2.workflows.general_summary.context import GeneralSummaryContext
|
||||
from app.core.agent.processes.v2.workflows.general_summary.graph import GeneralSummaryGraph
|
||||
from app.core.agent.utils.llm import AgentLlmService, PromptLoader
|
||||
from app.core.rag.embedding.gigachat_embedder import GigaChatEmbedder
|
||||
from app.core.rag.persistence import RagRepository
|
||||
from app.core.rag.retrieval.session_retriever import RagSessionRetriever
|
||||
from app.core.shared.gigachat.client import GigaChatClient
|
||||
from app.core.shared.gigachat.settings import GigaChatSettings
|
||||
from app.core.shared.gigachat.token_provider import GigaChatTokenProvider
|
||||
from app.infra.observability.module_trace import RequestTraceContext
|
||||
from tests.pipeline_setup_v3.core.models import ExecutionPayload, V3Case
|
||||
|
||||
|
||||
class V2ProcessAdapter:
|
||||
def __init__(self, *, workflow_llm_enabled: bool = True) -> None:
|
||||
self._workflow_llm_enabled = workflow_llm_enabled
|
||||
self._router = V2IntentRouter(llm=_build_v2_llm())
|
||||
self._policy = V2RetrievalPolicyResolver()
|
||||
retriever = RagSessionRetriever(repository=RagRepository(), embedder=GigaChatEmbedder(_build_client()))
|
||||
self._retrieval = V2RagRetrievalAdapter(retriever)
|
||||
self._evidence = DocsEvidenceAssembler()
|
||||
self._gate = DocsEvidenceGate()
|
||||
self._summary_graph = DocsExplainSummaryGraph(_build_v2_llm())
|
||||
self._find_files_graph = DocsExplainFindFilesGraph()
|
||||
self._general_graph = GeneralSummaryGraph(_build_v2_llm())
|
||||
|
||||
def execute(self, case: V3Case, rag_session_id: str | None) -> ExecutionPayload:
|
||||
return asyncio.run(self._execute_async(case, rag_session_id))
|
||||
|
||||
async def _execute_async(self, case: V3Case, rag_session_id: str | None) -> ExecutionPayload:
|
||||
runtime = _RuntimeStub(query=case.query)
|
||||
route = self._router.route(case.query)
|
||||
_log_pipeline_step(
|
||||
runtime,
|
||||
"router_resolved",
|
||||
{
|
||||
"domain": route.routing_domain,
|
||||
"intent": route.intent,
|
||||
"subintent": route.subintent,
|
||||
"confidence": route.confidence,
|
||||
},
|
||||
)
|
||||
_log_pipeline_step(
|
||||
runtime,
|
||||
"anchors_extracted",
|
||||
{
|
||||
"signal_types": route_anchor_summary(route)["signal_types"],
|
||||
"endpoint_paths": route.anchors.endpoint_paths,
|
||||
"target_doc_hints": route.anchors.target_doc_hints,
|
||||
"matched_aliases": route.anchors.matched_aliases,
|
||||
},
|
||||
)
|
||||
_log_pipeline_step(
|
||||
runtime,
|
||||
"alias_resolution",
|
||||
{
|
||||
"resolved_aliases": route.anchors.matched_aliases,
|
||||
"target_doc_hints": route.anchors.target_doc_hints,
|
||||
},
|
||||
)
|
||||
if case.mode == "router_only":
|
||||
return ExecutionPayload(
|
||||
actual=_actual_from_v2(route),
|
||||
details=_details(case.query, route=route, pipeline_steps=_build_pipeline_steps(runtime.logs)),
|
||||
)
|
||||
plan = self._policy.resolve(route)
|
||||
_log_pipeline_step(
|
||||
runtime,
|
||||
"retrieval_profile_selected",
|
||||
{"profile": plan.profile, "layers": plan.layers, "filters": plan.filters},
|
||||
)
|
||||
semantic_rows = await self._retrieve_rows(route, rag_session_id, plan)
|
||||
seeded_rows = await self._seed_candidates_from_target_hints(route, rag_session_id, plan)
|
||||
metadata_rows = self._metadata_lookup_candidates([*seeded_rows, *semantic_rows], route)
|
||||
rows = self._merge_candidate_rows(seeded_rows, metadata_rows, semantic_rows)
|
||||
_log_pipeline_step(
|
||||
runtime,
|
||||
"candidate_generation",
|
||||
{
|
||||
"resolved_aliases": route.anchors.matched_aliases,
|
||||
"target_doc_hints": route.anchors.target_doc_hints,
|
||||
"candidate_docs_before_ranking": [self._trace_row(row) for row in rows[:8]],
|
||||
"sources": {
|
||||
"seeded": [self._trace_row(row) for row in seeded_rows[:5]],
|
||||
"metadata_lookup": [self._trace_row(row) for row in metadata_rows[:5]],
|
||||
"semantic": [self._trace_row(row) for row in semantic_rows[:5]],
|
||||
},
|
||||
},
|
||||
)
|
||||
_log_pipeline_step(
|
||||
runtime,
|
||||
"retrieval_executed",
|
||||
{
|
||||
"query": case.query,
|
||||
"profile": plan.profile,
|
||||
"row_count": len(rows),
|
||||
"target_doc_hints": route.anchors.target_doc_hints,
|
||||
"top_results": [self._trace_row(row) for row in rows[:5]],
|
||||
},
|
||||
)
|
||||
if case.mode == "router_rag":
|
||||
return ExecutionPayload(
|
||||
actual=_actual_from_v2(route, rows=rows, plan=plan, answer_mode="partial"),
|
||||
details=_details(case.query, route=route, plan=plan, rows=rows, pipeline_steps=_build_pipeline_steps(runtime.logs)),
|
||||
)
|
||||
answer, evidence, gate = await self._run_workflow(runtime, route, rag_session_id, rows)
|
||||
answer_mode = gate.answer_mode
|
||||
_log_pipeline_step(
|
||||
runtime,
|
||||
"answer_generated",
|
||||
{"answer_mode": answer_mode, "answer_length": len(answer)},
|
||||
)
|
||||
return ExecutionPayload(
|
||||
actual=_actual_from_v2(route, rows=rows, plan=plan, answer=answer, answer_mode=answer_mode),
|
||||
details=_details(
|
||||
case.query,
|
||||
route=route,
|
||||
plan=plan,
|
||||
rows=rows,
|
||||
evidence=evidence,
|
||||
answer=answer,
|
||||
logs=runtime.logs,
|
||||
pipeline_steps=_build_pipeline_steps(runtime.logs),
|
||||
),
|
||||
)
|
||||
|
||||
async def _retrieve_rows(self, route, rag_session_id: str | None, plan) -> list[dict]:
|
||||
if not rag_session_id:
|
||||
if route.intent == V2Intent.GENERAL_QA:
|
||||
return []
|
||||
raise ValueError("process_v2 cases with DOCS intent require rag_session_id")
|
||||
return await self._retrieval.fetch_rows(rag_session_id, route.normalized_query, plan)
|
||||
|
||||
async def _seed_candidates_from_target_hints(self, route, rag_session_id: str | None, plan) -> list[dict]:
|
||||
if not rag_session_id or not route.anchors.target_doc_hints:
|
||||
return []
|
||||
return await self._retrieval.fetch_exact_paths(rag_session_id, paths=route.anchors.target_doc_hints, layers=plan.layers)
|
||||
|
||||
def _metadata_lookup_candidates(self, rows: list[dict], route) -> list[dict]:
|
||||
return DocsMetadataLookupIndex(rows).lookup(route)
|
||||
|
||||
def _merge_candidate_rows(self, *groups: list[dict]) -> list[dict]:
|
||||
merged: list[dict] = []
|
||||
seen: set[tuple[str, str, str]] = set()
|
||||
for rows in groups:
|
||||
for row in rows:
|
||||
key = (
|
||||
str(row.get("path") or ""),
|
||||
str(row.get("layer") or ""),
|
||||
str(dict(row.get("metadata") or {}).get("section_path") or ""),
|
||||
)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
merged.append(row)
|
||||
return merged
|
||||
|
||||
async def _run_workflow(
|
||||
self,
|
||||
runtime: "_RuntimeStub",
|
||||
route,
|
||||
rag_session_id: str | None,
|
||||
rows: list[dict],
|
||||
) -> tuple[str, dict, object]:
|
||||
if route.intent == V2Intent.GENERAL_QA:
|
||||
documents = self._evidence.assemble_summaries(rows, route)
|
||||
gate = self._gate.check_summaries(route, documents)
|
||||
_log_pipeline_step(
|
||||
runtime,
|
||||
"evidence_assembled",
|
||||
{"mode": "summary", "primary_doc": documents[0].path if documents else None, "document_count": len(documents)},
|
||||
)
|
||||
self._log_ranking(runtime, documents)
|
||||
_log_pipeline_step(
|
||||
runtime,
|
||||
"evidence_gate_checked",
|
||||
{"passed": gate.passed, "reason": gate.reason, "answer_mode": gate.answer_mode},
|
||||
)
|
||||
context = GeneralSummaryContext(runtime=runtime, route=route, prompt_name="v2_general.summary_answer")
|
||||
context.workflow_llm_enabled = self._workflow_llm_enabled
|
||||
context.documents = documents
|
||||
context.gate_decision = gate
|
||||
final = await self._general_graph.run(context)
|
||||
return final.answer, {"documents": [_serialize_summary(item) for item in documents], "files": []}, gate
|
||||
if route.subintent == V2Subintent.FIND_FILES:
|
||||
files = self._evidence.assemble_files(rows, route)
|
||||
gate = self._gate.check_files(route, files)
|
||||
_log_pipeline_step(
|
||||
runtime,
|
||||
"evidence_assembled",
|
||||
{"mode": "find_files", "primary_file": files[0].path if files else None, "file_count": len(files)},
|
||||
)
|
||||
self._log_ranking(runtime, files)
|
||||
_log_pipeline_step(
|
||||
runtime,
|
||||
"evidence_gate_checked",
|
||||
{"passed": gate.passed, "reason": gate.reason, "answer_mode": gate.answer_mode},
|
||||
)
|
||||
context = DocsExplainFindFilesContext(
|
||||
runtime=runtime,
|
||||
route=route,
|
||||
rag_session_id=rag_session_id or "",
|
||||
files=files,
|
||||
gate_decision=gate,
|
||||
)
|
||||
final = await self._find_files_graph.run(context)
|
||||
return final.answer, {"documents": [], "files": [_serialize_file(item) for item in files]}, gate
|
||||
documents = self._evidence.assemble_summaries(rows, route)
|
||||
gate = self._gate.check_summaries(route, documents)
|
||||
_log_pipeline_step(
|
||||
runtime,
|
||||
"evidence_assembled",
|
||||
{"mode": "summary", "primary_doc": documents[0].path if documents else None, "document_count": len(documents)},
|
||||
)
|
||||
self._log_ranking(runtime, documents)
|
||||
_log_pipeline_step(
|
||||
runtime,
|
||||
"evidence_gate_checked",
|
||||
{"passed": gate.passed, "reason": gate.reason, "answer_mode": gate.answer_mode},
|
||||
)
|
||||
context = DocsExplainSummaryContext(
|
||||
runtime=runtime,
|
||||
route=route,
|
||||
rag_session_id=rag_session_id or "",
|
||||
prompt_name="v2_docs_explain.summary_answer",
|
||||
workflow_llm_enabled=self._workflow_llm_enabled,
|
||||
documents=documents,
|
||||
gate_decision=gate,
|
||||
)
|
||||
final = await self._summary_graph.run(context)
|
||||
return final.answer, {"documents": [_serialize_summary(item) for item in documents], "files": []}, gate
|
||||
|
||||
def _trace_row(self, row: dict) -> dict[str, object]:
|
||||
metadata = dict(row.get("metadata") or {})
|
||||
return {
|
||||
"path": str(row.get("path") or ""),
|
||||
"layer": str(row.get("layer") or ""),
|
||||
"title": str(row.get("title") or ""),
|
||||
"document_id": str(metadata.get("document_id") or metadata.get("doc_id") or ""),
|
||||
}
|
||||
|
||||
def _log_ranking(self, runtime: "_RuntimeStub", items: list) -> None:
|
||||
top_docs: list[dict[str, object]] = []
|
||||
for item in items[:4]:
|
||||
top_docs.append(
|
||||
{
|
||||
"doc": getattr(item, "path", ""),
|
||||
"score": getattr(item, "score", 0),
|
||||
"match_reason": getattr(item, "match_reason", ""),
|
||||
}
|
||||
)
|
||||
_log_pipeline_step(
|
||||
runtime,
|
||||
"ranking_explained",
|
||||
{
|
||||
"doc": getattr(item, "path", ""),
|
||||
"score": getattr(item, "score", 0),
|
||||
"score_breakdown": getattr(item, "score_breakdown", {}),
|
||||
"match_reason": getattr(item, "match_reason", ""),
|
||||
},
|
||||
)
|
||||
_log_pipeline_step(
|
||||
runtime,
|
||||
"ranking_explained",
|
||||
{
|
||||
"top_docs_after_ranking": top_docs,
|
||||
"ranking_score_breakdown": [
|
||||
{
|
||||
"doc": getattr(item, "path", ""),
|
||||
"score_breakdown": getattr(item, "score_breakdown", {}),
|
||||
}
|
||||
for item in items[:4]
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _RequestStub:
|
||||
request_id: str
|
||||
message: str
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _SessionStub:
|
||||
active_rag_session_id: str | None = None
|
||||
|
||||
|
||||
class _PublisherStub:
|
||||
async def publish_status(self, request_id: str, source: str, message: str, payload: dict | None = None) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class _TraceLoggerStub:
|
||||
def __init__(self, store: list[dict]) -> None:
|
||||
self._store = store
|
||||
|
||||
def log_module(self, request_id: str, module: str, title: str, payload: dict | None = None) -> None:
|
||||
self._store.append(
|
||||
{"request_id": request_id, "module": module, "event": title, "payload": dict(payload or {})}
|
||||
)
|
||||
|
||||
|
||||
class _RuntimeStub:
|
||||
def __init__(self, *, query: str) -> None:
|
||||
self.logs: list[dict] = []
|
||||
self.request = _RequestStub(request_id="pipeline_setup_v3", message=query)
|
||||
self.session = _SessionStub()
|
||||
self.publisher = _PublisherStub()
|
||||
self.trace = RequestTraceContext(request_id=self.request.request_id, logger=_TraceLoggerStub(self.logs))
|
||||
|
||||
|
||||
def _build_client() -> GigaChatClient:
|
||||
settings = GigaChatSettings.from_env()
|
||||
return GigaChatClient(settings, GigaChatTokenProvider(settings))
|
||||
|
||||
|
||||
def _build_v2_llm() -> AgentLlmService:
|
||||
prompt_paths = [
|
||||
Path(__file__).resolve().parents[3] / "src/app/core/agent/processes/v2/prompts.yml",
|
||||
Path(__file__).resolve().parents[3] / "src/app/core/agent/processes/v2/general_prompts.yml",
|
||||
Path(__file__).resolve().parents[3] / "src/app/core/agent/processes/v2/intent_router/routers/prompts.yml",
|
||||
]
|
||||
return AgentLlmService(client=_build_client(), prompts=PromptLoader(prompt_paths))
|
||||
|
||||
|
||||
def _actual_from_v2(route, *, rows: list[dict] | None = None, plan=None, answer: str = "", answer_mode: str = "partial") -> dict:
|
||||
return {
|
||||
"domain": route.routing_domain,
|
||||
"intent": route.intent,
|
||||
"sub_intent": route.subintent,
|
||||
"rag_count": len(rows or []),
|
||||
"llm_answer": answer,
|
||||
"answer_mode": answer_mode,
|
||||
"path_scope": tuple(),
|
||||
"symbol_candidates": tuple(),
|
||||
"entity_candidates": tuple(_entity_candidates(rows or [])),
|
||||
"doc_scope": tuple(_doc_scope(rows or [])),
|
||||
"layers": tuple(getattr(plan, "layers", []) or []),
|
||||
"filters": dict(getattr(plan, "filters", {}) or {}),
|
||||
}
|
||||
|
||||
|
||||
def _details(query: str, **payload) -> dict:
|
||||
details = {"query": query}
|
||||
for key, value in payload.items():
|
||||
if key == "route":
|
||||
details["router_result"] = asdict(value)
|
||||
elif key == "plan":
|
||||
details["retrieval_plan"] = asdict(value)
|
||||
else:
|
||||
details[key] = value
|
||||
return details
|
||||
|
||||
|
||||
def _doc_scope(rows: list[dict]) -> list[str]:
|
||||
values: list[str] = []
|
||||
for row in rows:
|
||||
metadata = dict(row.get("metadata") or {})
|
||||
for candidate in (metadata.get("document_id"), metadata.get("doc_id"), row.get("path")):
|
||||
value = str(candidate or "").strip()
|
||||
if value and value not in values:
|
||||
values.append(value)
|
||||
return values
|
||||
|
||||
|
||||
def _entity_candidates(rows: list[dict]) -> list[str]:
|
||||
values: list[str] = []
|
||||
for row in rows:
|
||||
metadata = dict(row.get("metadata") or {})
|
||||
value = str(metadata.get("entity_name") or row.get("title") or "").strip()
|
||||
if value and value not in values and str(row.get("layer") or "") == "D3_ENTITY_CATALOG":
|
||||
values.append(value)
|
||||
return values
|
||||
|
||||
|
||||
def _serialize_summary(item: RetrievedSummary) -> dict:
|
||||
return asdict(item)
|
||||
|
||||
|
||||
def _serialize_file(item: RetrievedFile) -> dict:
|
||||
return asdict(item)
|
||||
|
||||
|
||||
def _build_pipeline_steps(logs: list[dict]) -> list[dict]:
|
||||
steps: list[dict] = []
|
||||
for item in logs:
|
||||
if item.get("module") != "process.v2.pipeline":
|
||||
continue
|
||||
steps.append({"step": item.get("event"), "output": item.get("payload") or {}})
|
||||
return steps
|
||||
|
||||
|
||||
def _log_pipeline_step(runtime: _RuntimeStub, step: str, payload: dict[str, object]) -> None:
|
||||
runtime.logs.append(
|
||||
{
|
||||
"request_id": runtime.request.request_id,
|
||||
"module": "process.v2.pipeline",
|
||||
"event": step,
|
||||
"payload": payload,
|
||||
}
|
||||
)
|
||||
@@ -3,7 +3,7 @@ from __future__ import annotations
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from app.modules.shared.env_loader import load_workspace_env
|
||||
from app.core.shared.config import load_workspace_env
|
||||
|
||||
|
||||
def load_pipeline_setup_env(start_dir: str | Path | None = None) -> list[Path]:
|
||||
|
||||
@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING
|
||||
from uuid import uuid4
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.modules.rag.persistence.repository import RagRepository
|
||||
from app.core.rag.persistence.repository import RagRepository
|
||||
|
||||
|
||||
class LocalRepoFileCollector:
|
||||
@@ -77,7 +77,7 @@ class DeterministicEmbedder:
|
||||
|
||||
class RagSessionIndexer:
|
||||
def __init__(self, repository: "RagRepository") -> None:
|
||||
from app.modules.rag.services.rag_service import RagService
|
||||
from app.core.rag.indexing.service import RagService
|
||||
|
||||
self._repository = repository
|
||||
self._rag = RagService(embedder=DeterministicEmbedder(), repository=repository)
|
||||
@@ -90,7 +90,7 @@ class RagSessionIndexer:
|
||||
files = LocalRepoFileCollector(repo_path).collect()
|
||||
if not files:
|
||||
raise ValueError(f"No indexable text files found under: {repo_path}")
|
||||
logger = logging.getLogger("app.modules.rag.services.rag_service")
|
||||
logger = logging.getLogger("app.core.rag.indexing.service")
|
||||
previous_level = logger.level
|
||||
logger.setLevel(logging.ERROR)
|
||||
try:
|
||||
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-01-health",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"llm: expected answer to contain 'status'",
|
||||
"llm: expected answer to contain 'components'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "Информация о том, как именно работает `/health`, не представлена в предоставленных SUMMARY-блоках. \n\nФайлы, на которые я опирался:\n- docs/architecture/telegram-notify-app-overview.md",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
# v2-docs-summary-01-health
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Кратко объясни по документации, как работает `/health`
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "Информация о том, как именно работает `/health`, не представлена в предоставленных SUMMARY-блоках. \n\nФайлы, на которые я опирался:\n- docs/architecture/telegram-notify-app-overview.md",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- llm: expected answer to contain 'status'
|
||||
- llm: expected answer to contain 'components'
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-02-send",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']",
|
||||
"llm: expected answer to contain 'message'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "На основе представленного SUMMARY-блока, документация о том, что именно делает endpoint `/send`, отсутствует. В контексте указано, что он связан с отправкой сообщений, но подробностей его работы не приведено.\n\nФайлы, на которые опирался ответ:\n- docs/architecture/telegram-notify-app-overview.md",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
# v2-docs-summary-02-send
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Что делает endpoint `/send`?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "На основе представленного SUMMARY-блока, документация о том, что именно делает endpoint `/send`, отсутствует. В контексте указано, что он связан с отправкой сообщений, но подробностей его работы не приведено.\n\nФайлы, на которые опирался ответ:\n- docs/architecture/telegram-notify-app-overview.md",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
|
||||
- llm: expected answer to contain 'message'
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-03-actions",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"domain: expected DOCS, got GENERAL",
|
||||
"intent: expected DOC_EXPLAIN, got GENERAL_QA",
|
||||
"retrieval: expected non-empty rag, got 0 rows",
|
||||
"doc_scope should contain 'docs/api/control-actions-endpoint.md', got []"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "`/actions/{action}` обычно используется в API для выполнения конкретных действий или операций, где `{action}` заменяется названием нужного действия. Это позволяет организовывать и вызывать различные функции через единый интерфейс.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
# v2-docs-summary-03-actions
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Объясни назначение `/actions/{action}`
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "`/actions/{action}` обычно используется в API для выполнения конкретных действий или операций, где `{action}` заменяется названием нужного действия. Это позволяет организовывать и вызывать различные функции через единый интерфейс.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- domain: expected DOCS, got GENERAL
|
||||
- intent: expected DOC_EXPLAIN, got GENERAL_QA
|
||||
- retrieval: expected non-empty rag, got 0 rows
|
||||
- doc_scope should contain 'docs/api/control-actions-endpoint.md', got []
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-04-runtime-health",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"domain: expected DOCS, got GENERAL",
|
||||
"intent: expected DOC_EXPLAIN, got GENERAL_QA",
|
||||
"retrieval: expected non-empty rag, got 0 rows",
|
||||
"doc_scope should contain 'docs/domains/runtime-health-entity.md', got []"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Runtime health — это процесс мониторинга и обеспечения стабильности работы системы или приложения во время их выполнения (runtime), позволяющий своевременно обнаруживать и устранять возникающие проблемы.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
# v2-docs-summary-04-runtime-health
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Что такое runtime health в этой документации?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Runtime health — это процесс мониторинга и обеспечения стабильности работы системы или приложения во время их выполнения (runtime), позволяющий своевременно обнаруживать и устранять возникающие проблемы.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- domain: expected DOCS, got GENERAL
|
||||
- intent: expected DOC_EXPLAIN, got GENERAL_QA
|
||||
- retrieval: expected non-empty rag, got 0 rows
|
||||
- doc_scope should contain 'docs/domains/runtime-health-entity.md', got []
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-05-notification-loop",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"domain: expected DOCS, got GENERAL",
|
||||
"intent: expected DOC_EXPLAIN, got GENERAL_QA",
|
||||
"retrieval: expected non-empty rag, got 0 rows",
|
||||
"doc_scope should contain 'docs/logic/telegram-notification-loop.md', got []",
|
||||
"llm: expected answer to contain 'worker'",
|
||||
"llm: expected answer to contain 'interval'",
|
||||
"llm: expected answer to contain 'degraded'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Цикл отправки уведомлений включает следующие шаги: отлавливание события, обработка данных, выбор получателей, формирование сообщения и его отправка выбранным пользователям.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
# v2-docs-summary-05-notification-loop
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Кратко опиши цикл отправки уведомлений
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Цикл отправки уведомлений включает следующие шаги: отлавливание события, обработка данных, выбор получателей, формирование сообщения и его отправка выбранным пользователям.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- domain: expected DOCS, got GENERAL
|
||||
- intent: expected DOC_EXPLAIN, got GENERAL_QA
|
||||
- retrieval: expected non-empty rag, got 0 rows
|
||||
- doc_scope should contain 'docs/logic/telegram-notification-loop.md', got []
|
||||
- llm: expected answer to contain 'worker'
|
||||
- llm: expected answer to contain 'interval'
|
||||
- llm: expected answer to contain 'degraded'
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-06-architecture",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"domain: expected DOCS, got GENERAL",
|
||||
"intent: expected DOC_EXPLAIN, got GENERAL_QA",
|
||||
"retrieval: expected non-empty rag, got 0 rows",
|
||||
"doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []",
|
||||
"llm: expected answer to contain 'RuntimeManager'",
|
||||
"llm: expected answer to contain 'TelegramControlChannel'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Архитектура Telegram Notify App основана на клиент-серверной модели, где серверное приложение обрабатывает входящие данные и отправляет уведомления через Telegram API. Клиентское приложение взаимодействует с сервером для отправки уведомлений. Пользователь настраивает параметры уведомлений в приложении, а сервер передает их в виде сообщений в указанный Telegram-канал или пользователю.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
# v2-docs-summary-06-architecture
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Как устроена архитектура Telegram Notify App?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Архитектура Telegram Notify App основана на клиент-серверной модели, где серверное приложение обрабатывает входящие данные и отправляет уведомления через Telegram API. Клиентское приложение взаимодействует с сервером для отправки уведомлений. Пользователь настраивает параметры уведомлений в приложении, а сервер передает их в виде сообщений в указанный Telegram-канал или пользователю.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- domain: expected DOCS, got GENERAL
|
||||
- intent: expected DOC_EXPLAIN, got GENERAL_QA
|
||||
- retrieval: expected non-empty rag, got 0 rows
|
||||
- doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []
|
||||
- llm: expected answer to contain 'RuntimeManager'
|
||||
- llm: expected answer to contain 'TelegramControlChannel'
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-07-worker",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"domain: expected DOCS, got GENERAL",
|
||||
"intent: expected DOC_EXPLAIN, got GENERAL_QA",
|
||||
"retrieval: expected non-empty rag, got 0 rows",
|
||||
"doc_scope should contain 'docs/logic/telegram-notification-loop.md', got []",
|
||||
"doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []",
|
||||
"llm: expected answer to contain 'Telegram'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Worker в этом приложении выполняет фоновые задачи или процессы, позволяя основному приложению работать быстро и плавно. Это может включать обработку данных, отправку уведомлений, загрузку файлов и другие ресурсоемкие операции.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
# v2-docs-summary-07-worker
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Что делает worker в этом приложении?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Worker в этом приложении выполняет фоновые задачи или процессы, позволяя основному приложению работать быстро и плавно. Это может включать обработку данных, отправку уведомлений, загрузку файлов и другие ресурсоемкие операции.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- domain: expected DOCS, got GENERAL
|
||||
- intent: expected DOC_EXPLAIN, got GENERAL_QA
|
||||
- retrieval: expected non-empty rag, got 0 rows
|
||||
- doc_scope should contain 'docs/logic/telegram-notification-loop.md', got []
|
||||
- doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []
|
||||
- llm: expected answer to contain 'Telegram'
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-08-health-statuses",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"doc_scope should contain 'docs/domains/runtime-health-entity.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']",
|
||||
"doc_scope should contain 'docs/api/health-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']",
|
||||
"llm: expected answer to contain 'ok'",
|
||||
"llm: expected answer to contain 'degraded'",
|
||||
"llm: expected answer to contain 'unhealthy'",
|
||||
"llm: expected answer to contain 'unknown'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "На основе представленного SUMMARY-блока конкретной информации о статусах здоровья runtime нет. Файл, связанный с доменом \"runtime health\" (../domains/runtime-health-entity.md), вероятно, содержит эту информацию, но она не включена в текущий контекст.\n\nФайлы, на которые опирался ответ:\n- docs/architecture/telegram-notify-app-overview.md",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
# v2-docs-summary-08-health-statuses
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Какие статусы здоровья есть у runtime?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "На основе представленного SUMMARY-блока конкретной информации о статусах здоровья runtime нет. Файл, связанный с доменом \"runtime health\" (../domains/runtime-health-entity.md), вероятно, содержит эту информацию, но она не включена в текущий контекст.\n\nФайлы, на которые опирался ответ:\n- docs/architecture/telegram-notify-app-overview.md",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- doc_scope should contain 'docs/domains/runtime-health-entity.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
|
||||
- doc_scope should contain 'docs/api/health-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
|
||||
- llm: expected answer to contain 'ok'
|
||||
- llm: expected answer to contain 'degraded'
|
||||
- llm: expected answer to contain 'unhealthy'
|
||||
- llm: expected answer to contain 'unknown'
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-09-manual-send",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"domain: expected DOCS, got GENERAL",
|
||||
"intent: expected DOC_EXPLAIN, got GENERAL_QA",
|
||||
"retrieval: expected non-empty rag, got 0 rows",
|
||||
"doc_scope should contain 'docs/api/send-message-endpoint.md', got []",
|
||||
"llm: expected answer to contain '/send'",
|
||||
"llm: expected answer to contain 'message'",
|
||||
"llm: expected answer to contain 'TelegramSendService'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Ручная отправка сообщения в приложении обычно осуществляется нажатием кнопки \"Отправить\" или аналогичного элемента интерфейса после того, как пользователь введет текст сообщения.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
# v2-docs-summary-09-manual-send
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Как в приложении устроена ручная отправка сообщения?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Ручная отправка сообщения в приложении обычно осуществляется нажатием кнопки \"Отправить\" или аналогичного элемента интерфейса после того, как пользователь введет текст сообщения.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- domain: expected DOCS, got GENERAL
|
||||
- intent: expected DOC_EXPLAIN, got GENERAL_QA
|
||||
- retrieval: expected non-empty rag, got 0 rows
|
||||
- doc_scope should contain 'docs/api/send-message-endpoint.md', got []
|
||||
- llm: expected answer to contain '/send'
|
||||
- llm: expected answer to contain 'message'
|
||||
- llm: expected answer to contain 'TelegramSendService'
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-10-telegram-integration",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"domain: expected DOCS, got GENERAL",
|
||||
"intent: expected DOC_EXPLAIN, got GENERAL_QA",
|
||||
"retrieval: expected non-empty rag, got 0 rows",
|
||||
"retrieval: expected at least 2 rows, got 0",
|
||||
"doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []",
|
||||
"doc_scope should contain 'docs/api/send-message-endpoint.md', got []",
|
||||
"llm: expected answer to contain 'worker'",
|
||||
"llm: expected answer to contain '/send'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Сервис может отправлять и получать сообщения, реагировать на команды и запросы пользователей через Telegram-бота. Взаимодействие происходит посредством отправки запросов и получения ответов от бота в чате.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
# v2-docs-summary-10-telegram-integration
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Как сервис взаимодействует с Telegram в целом?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Сервис может отправлять и получать сообщения, реагировать на команды и запросы пользователей через Telegram-бота. Взаимодействие происходит посредством отправки запросов и получения ответов от бота в чате.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- domain: expected DOCS, got GENERAL
|
||||
- intent: expected DOC_EXPLAIN, got GENERAL_QA
|
||||
- retrieval: expected non-empty rag, got 0 rows
|
||||
- retrieval: expected at least 2 rows, got 0
|
||||
- doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []
|
||||
- doc_scope should contain 'docs/api/send-message-endpoint.md', got []
|
||||
- llm: expected answer to contain 'worker'
|
||||
- llm: expected answer to contain '/send'
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-01-health",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"llm: expected answer to contain 'docs/api/health-endpoint.md'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"RuntimeManager",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
# v2-find-files-01-health
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
В каком файле описан `/health`?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"RuntimeManager",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- llm: expected answer to contain 'docs/api/health-endpoint.md'
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-02-send",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"llm: expected answer to contain 'docs/api/send-message-endpoint.md'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
# v2-find-files-02-send
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Покажи файл с описанием endpoint `/send`
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- llm: expected answer to contain 'docs/api/send-message-endpoint.md'
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-03-actions",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"llm: expected answer to contain 'docs/api/control-actions-endpoint.md'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
# v2-find-files-03-actions
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Где находится документация по `/actions/{action}`?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- llm: expected answer to contain 'docs/api/control-actions-endpoint.md'
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-04-runtime-health",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"llm: expected answer to contain 'docs/domains/runtime-health-entity.md'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/architecture/telegram-notify-app-overview.md`\n- `docs/README.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
# v2-find-files-04-runtime-health
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
В каком документе описан runtime health?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/architecture/telegram-notify-app-overview.md`\n- `docs/README.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- llm: expected answer to contain 'docs/domains/runtime-health-entity.md'
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-05-loop",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService",
|
||||
"RuntimeManager",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
# v2-find-files-05-loop
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Найди файл с логикой цикла отправки уведомлений
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService",
|
||||
"RuntimeManager",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-06-architecture",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": true,
|
||||
"mismatches": [],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
# v2-find-files-06-architecture
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: True
|
||||
|
||||
## Query
|
||||
Где описана архитектура приложения?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- none
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-07-errors-catalog",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"llm: expected answer to contain 'docs/errors/catalog.yaml'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyWorker"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
# v2-find-files-07-errors-catalog
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
В каком файле лежит каталог ошибок?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyWorker"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- llm: expected answer to contain 'docs/errors/catalog.yaml'
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-08-docs-index",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": true,
|
||||
"mismatches": [],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
# v2-find-files-08-docs-index
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: True
|
||||
|
||||
## Query
|
||||
Покажи файл-индекс документации проекта
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- none
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-09-telegram-bot-api",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"llm: expected answer to contain 'docs/api/send-message-endpoint.md'",
|
||||
"llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
# v2-find-files-09-telegram-bot-api
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Какие файлы относятся к Telegram Bot API интеграции?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- llm: expected answer to contain 'docs/api/send-message-endpoint.md'
|
||||
- llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-10-telegram-notify-worker",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'",
|
||||
"llm: expected answer to contain 'docs/domains/runtime-health-entity.md'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
# v2-find-files-10-telegram-notify-worker
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
В каких файлах описан worker `telegram_notify`?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'
|
||||
- llm: expected answer to contain 'docs/domains/runtime-health-entity.md'
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-general-01-what-is-service",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"llm: expected answer to contain 'Telegram'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Это зависит от контекста. Уточни, пожалуйста, о каком именно сервисе идет речь.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
# v2-general-01-what-is-service
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Что это за сервис?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Это зависит от контекста. Уточни, пожалуйста, о каком именно сервисе идет речь.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- llm: expected answer to contain 'Telegram'
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-general-02-purpose",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": true,
|
||||
"mismatches": [],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Test_echo_app нужен для тестирования и отладки приложений и систем, повторяя входящий запрос в качестве ответа, чтобы убедиться в правильной работе связи и функциональности.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
# v2-general-02-purpose
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: True
|
||||
|
||||
## Query
|
||||
Для чего нужен test_echo_app?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Test_echo_app нужен для тестирования и отладки приложений и систем, повторяя входящий запрос в качестве ответа, чтобы убедиться в правильной работе связи и функциональности.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- none
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-general-03-problem",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"llm: expected answer to contain 'Telegram'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Приложение решает задачу (указать конкретную задачу, например, управления контактами, планирования задач, учета финансов и т.д.) для пользователей путем предоставления удобного и интуитивно понятного интерфейса.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
# v2-general-03-problem
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Какую задачу решает это приложение?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Приложение решает задачу (указать конкретную задачу, например, управления контактами, планирования задач, учета финансов и т.д.) для пользователей путем предоставления удобного и интуитивно понятного интерфейса.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- llm: expected answer to contain 'Telegram'
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-general-04-docs-contents",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"domain: expected GENERAL, got DOCS",
|
||||
"intent: expected GENERAL_QA, got DOC_EXPLAIN"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "Документация проекта включает следующие компоненты:\n\n- Архитектура приложения `test_echo_app`\n- Описание HTTP API (control plane)\n- Цикл отправки уведомлений\n- Health-модель\n- Каталог ошибок\n\nСтруктура документации канонически разделена на несколько секций: `docs/architecture`, `docs/api`, `docs/logic`, `docs/domains`, `docs/errors`. Основная родительская документация — \"Архитектура Telegram Notify App\" (находится по пути ./architecture/telegram-notify-app-overview.md). Документы взаимосвязаны с помощью ссылок `related_docs`, `parent`/`children` и обычных markdown-ссылок без повторения одной и той же информации.\n\nФайлы, использованные для ответа: docs/README.md",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
# v2-general-04-docs-contents
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Что входит в документацию этого проекта?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "Документация проекта включает следующие компоненты:\n\n- Архитектура приложения `test_echo_app`\n- Описание HTTP API (control plane)\n- Цикл отправки уведомлений\n- Health-модель\n- Каталог ошибок\n\nСтруктура документации канонически разделена на несколько секций: `docs/architecture`, `docs/api`, `docs/logic`, `docs/domains`, `docs/errors`. Основная родительская документация — \"Архитектура Telegram Notify App\" (находится по пути ./architecture/telegram-notify-app-overview.md). Документы взаимосвязаны с помощью ссылок `related_docs`, `parent`/`children` и обычных markdown-ссылок без повторения одной и той же информации.\n\nФайлы, использованные для ответа: docs/README.md",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- domain: expected GENERAL, got DOCS
|
||||
- intent: expected GENERAL_QA, got DOC_EXPLAIN
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-general-05-main-parts",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"llm: expected answer to contain 'Telegram'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Основные части любой системы включают вход, процессоры (обработчики), память и выход.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
# v2-general-05-main-parts
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Какие основные части есть у системы?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Основные части любой системы включают вход, процессоры (обработчики), память и выход.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- llm: expected answer to contain 'Telegram'
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-general-06-high-level-architecture",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": true,
|
||||
"mismatches": [],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Приложение обычно состоит из пользовательского интерфейса (то, что видит и с чем взаимодействует пользователь), логики приложения (код, который обрабатывает действия пользователя) и базы данных (место хранения информации). Все эти части связаны между собой для обеспечения удобной работы пользователя.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
# v2-general-06-high-level-architecture
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: True
|
||||
|
||||
## Query
|
||||
Как в целом устроено приложение?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Приложение обычно состоит из пользовательского интерфейса (то, что видит и с чем взаимодействует пользователь), логики приложения (код, который обрабатывает действия пользователя) и базы данных (место хранения информации). Все эти части связаны между собой для обеспечения удобной работы пользователя.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- none
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-general-07-besides-api",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": true,
|
||||
"mismatches": [],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Кроме API, здесь есть дружелюбная атмосфера, готовность помочь и интересные беседы на различные темы.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
# v2-general-07-besides-api
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: True
|
||||
|
||||
## Query
|
||||
Что тут есть кроме API?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Кроме API, здесь есть дружелюбная атмосфера, готовность помочь и интересные беседы на различные темы.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- none
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-general-08-components",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": true,
|
||||
"mismatches": [],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Telegram Notify App состоит из простого пользовательского интерфейса для отправки уведомлений, механизма обработки сообщений и интеграции с API Telegram, позволяющей отправлять сообщения в выбранные чаты или каналы.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
# v2-general-08-components
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: True
|
||||
|
||||
## Query
|
||||
Из чего состоит Telegram Notify App?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Telegram Notify App состоит из простого пользовательского интерфейса для отправки уведомлений, механизма обработки сообщений и интеграции с API Telegram, позволяющей отправлять сообщения в выбранные чаты или каналы.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- none
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-general-09-reading-order",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"domain: expected GENERAL, got DOCS",
|
||||
"intent: expected GENERAL_QA, got DOC_EXPLAIN",
|
||||
"llm: expected answer to contain 'README'",
|
||||
"llm: expected answer to contain 'архитект'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "Не нашёл подходящих SUMMARY-блоков в документации по этому запросу.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
# v2-general-09-reading-order
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Какие документы стоит читать сначала?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "Не нашёл подходящих SUMMARY-блоков в документации по этому запросу.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- domain: expected GENERAL, got DOCS
|
||||
- intent: expected GENERAL_QA, got DOC_EXPLAIN
|
||||
- llm: expected answer to contain 'README'
|
||||
- llm: expected answer to contain 'архитект'
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-general-10-short-summary",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": true,
|
||||
"mismatches": [],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Проектная документация включает описание целей, задач, методов, графика и бюджета проекта, а также список необходимых ресурсов и команду исполнителей.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
# v2-general-10-short-summary
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: True
|
||||
|
||||
## Query
|
||||
Дай короткое summary по проектной документации
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Проектная документация включает описание целей, задач, методов, графика и бюджета проекта, а также список необходимых ресурсов и команду исполнителей.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- none
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
# pipeline_setup_v3 summary
|
||||
|
||||
Passed: 7/30
|
||||
|
||||
| File | Case | Mode | Query | Actual sub-intent | RAG layers | Tokens | Pass |
|
||||
|------|------|------|-------|-------------------|------------|--------|------|
|
||||
| full_chain_docs_v2_matrix.yaml | v2-general-01-what-is-service | full_chain | Что это за сервис? | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-general-02-purpose | full_chain | Для чего нужен test_echo_app? | SUMMARY | — | — | ✓ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-general-03-problem | full_chain | Какую задачу решает это приложение? | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-general-04-docs-contents | full_chain | Что входит в документацию этого проекта? | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-general-05-main-parts | full_chain | Какие основные части есть у системы? | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-general-06-high-level-architecture | full_chain | Как в целом устроено приложение? | SUMMARY | — | — | ✓ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-general-07-besides-api | full_chain | Что тут есть кроме API? | SUMMARY | — | — | ✓ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-general-08-components | full_chain | Из чего состоит Telegram Notify App? | SUMMARY | — | — | ✓ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-general-09-reading-order | full_chain | Какие документы стоит читать сначала? | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-general-10-short-summary | full_chain | Дай короткое summary по проектной документации | SUMMARY | — | — | ✓ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-01-health | full_chain | Кратко объясни по документации, как работает `/health` | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-02-send | full_chain | Что делает endpoint `/send`? | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-03-actions | full_chain | Объясни назначение `/actions/{action}` | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-04-runtime-health | full_chain | Что такое runtime health в этой документации? | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-05-notification-loop | full_chain | Кратко опиши цикл отправки уведомлений | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-06-architecture | full_chain | Как устроена архитектура Telegram Notify App? | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-07-worker | full_chain | Что делает worker в этом приложении? | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-08-health-statuses | full_chain | Какие статусы здоровья есть у runtime? | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-09-manual-send | full_chain | Как в приложении устроена ручная отправка сообщения? | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-10-telegram-integration | full_chain | Как сервис взаимодействует с Telegram в целом? | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-find-files-01-health | full_chain | В каком файле описан `/health`? | FIND_FILES | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-find-files-02-send | full_chain | Покажи файл с описанием endpoint `/send` | FIND_FILES | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-find-files-03-actions | full_chain | Где находится документация по `/actions/{action}`? | FIND_FILES | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-find-files-04-runtime-health | full_chain | В каком документе описан runtime health? | FIND_FILES | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-find-files-05-loop | full_chain | Найди файл с логикой цикла отправки уведомлений | FIND_FILES | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-find-files-06-architecture | full_chain | Где описана архитектура приложения? | FIND_FILES | — | — | ✓ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-find-files-07-errors-catalog | full_chain | В каком файле лежит каталог ошибок? | FIND_FILES | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-find-files-08-docs-index | full_chain | Покажи файл-индекс документации проекта | FIND_FILES | — | — | ✓ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-find-files-09-telegram-bot-api | full_chain | Какие файлы относятся к Telegram Bot API интеграции? | FIND_FILES | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-find-files-10-telegram-notify-worker | full_chain | В каких файлах описан worker `telegram_notify`? | FIND_FILES | — | — | ✗ |
|
||||
|
||||
## Failures
|
||||
- **v2-general-01-what-is-service**: llm: expected answer to contain 'Telegram'
|
||||
- **v2-general-03-problem**: llm: expected answer to contain 'Telegram'
|
||||
- **v2-general-04-docs-contents**: domain: expected GENERAL, got DOCS; intent: expected GENERAL_QA, got DOC_EXPLAIN
|
||||
- **v2-general-05-main-parts**: llm: expected answer to contain 'Telegram'
|
||||
- **v2-general-09-reading-order**: domain: expected GENERAL, got DOCS; intent: expected GENERAL_QA, got DOC_EXPLAIN; llm: expected answer to contain 'README'; llm: expected answer to contain 'архитект'
|
||||
- **v2-docs-summary-01-health**: llm: expected answer to contain 'status'; llm: expected answer to contain 'components'
|
||||
- **v2-docs-summary-02-send**: doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']; llm: expected answer to contain 'message'
|
||||
- **v2-docs-summary-03-actions**: domain: expected DOCS, got GENERAL; intent: expected DOC_EXPLAIN, got GENERAL_QA; retrieval: expected non-empty rag, got 0 rows; doc_scope should contain 'docs/api/control-actions-endpoint.md', got []
|
||||
- **v2-docs-summary-04-runtime-health**: domain: expected DOCS, got GENERAL; intent: expected DOC_EXPLAIN, got GENERAL_QA; retrieval: expected non-empty rag, got 0 rows; doc_scope should contain 'docs/domains/runtime-health-entity.md', got []
|
||||
- **v2-docs-summary-05-notification-loop**: domain: expected DOCS, got GENERAL; intent: expected DOC_EXPLAIN, got GENERAL_QA; retrieval: expected non-empty rag, got 0 rows; doc_scope should contain 'docs/logic/telegram-notification-loop.md', got []; llm: expected answer to contain 'worker'; llm: expected answer to contain 'interval'; llm: expected answer to contain 'degraded'
|
||||
- **v2-docs-summary-06-architecture**: domain: expected DOCS, got GENERAL; intent: expected DOC_EXPLAIN, got GENERAL_QA; retrieval: expected non-empty rag, got 0 rows; doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []; llm: expected answer to contain 'RuntimeManager'; llm: expected answer to contain 'TelegramControlChannel'
|
||||
- **v2-docs-summary-07-worker**: domain: expected DOCS, got GENERAL; intent: expected DOC_EXPLAIN, got GENERAL_QA; retrieval: expected non-empty rag, got 0 rows; doc_scope should contain 'docs/logic/telegram-notification-loop.md', got []; doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []; llm: expected answer to contain 'Telegram'
|
||||
- **v2-docs-summary-08-health-statuses**: doc_scope should contain 'docs/domains/runtime-health-entity.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']; doc_scope should contain 'docs/api/health-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']; llm: expected answer to contain 'ok'; llm: expected answer to contain 'degraded'; llm: expected answer to contain 'unhealthy'; llm: expected answer to contain 'unknown'
|
||||
- **v2-docs-summary-09-manual-send**: domain: expected DOCS, got GENERAL; intent: expected DOC_EXPLAIN, got GENERAL_QA; retrieval: expected non-empty rag, got 0 rows; doc_scope should contain 'docs/api/send-message-endpoint.md', got []; llm: expected answer to contain '/send'; llm: expected answer to contain 'message'; llm: expected answer to contain 'TelegramSendService'
|
||||
- **v2-docs-summary-10-telegram-integration**: domain: expected DOCS, got GENERAL; intent: expected DOC_EXPLAIN, got GENERAL_QA; retrieval: expected non-empty rag, got 0 rows; retrieval: expected at least 2 rows, got 0; doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []; doc_scope should contain 'docs/api/send-message-endpoint.md', got []; llm: expected answer to contain 'worker'; llm: expected answer to contain '/send'
|
||||
- **v2-find-files-01-health**: llm: expected answer to contain 'docs/api/health-endpoint.md'
|
||||
- **v2-find-files-02-send**: llm: expected answer to contain 'docs/api/send-message-endpoint.md'
|
||||
- **v2-find-files-03-actions**: llm: expected answer to contain 'docs/api/control-actions-endpoint.md'
|
||||
- **v2-find-files-04-runtime-health**: llm: expected answer to contain 'docs/domains/runtime-health-entity.md'
|
||||
- **v2-find-files-05-loop**: llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'
|
||||
- **v2-find-files-07-errors-catalog**: llm: expected answer to contain 'docs/errors/catalog.yaml'
|
||||
- **v2-find-files-09-telegram-bot-api**: llm: expected answer to contain 'docs/api/send-message-endpoint.md'; llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'
|
||||
- **v2-find-files-10-telegram-notify-worker**: llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'; llm: expected answer to contain 'docs/domains/runtime-health-entity.md'
|
||||
|
||||
## LLM Answers
|
||||
- **v2-general-01-what-is-service**
|
||||
Query: Что это за сервис?
|
||||
> Это зависит от контекста. Уточни, пожалуйста, о каком именно сервисе идет речь.
|
||||
- **v2-general-02-purpose**
|
||||
Query: Для чего нужен test_echo_app?
|
||||
> Test\_echo\_app нужен для тестирования и отладки приложений и систем, повторяя входящий запрос в качестве ответа, чтобы убедиться в правильной работе связи и функциональности.
|
||||
- **v2-general-03-problem**
|
||||
Query: Какую задачу решает это приложение?
|
||||
> Приложение решает задачу \(указать конкретную задачу, например, управления контактами, планирования задач, учета финансов и т.д.\) для пользователей путем предоставления удобного и интуитивно понятного интерфейса.
|
||||
- **v2-general-04-docs-contents**
|
||||
Query: Что входит в документацию этого проекта?
|
||||
> Документация проекта включает следующие компоненты: \- Архитектура приложения \`test\_echo\_app\` \- Описание HTTP API \(control plane\) \- Цикл отправки уведомлений \- Health\-модель \- Каталог ошибок Структура документации канонически разделена на несколько секций: \`docs/architecture\`, \`docs/api\`, \`docs/logic\`, \`docs/domains\`, \`docs/errors\`. Основная родительская документация — "Архитектура Telegram Notify App" \(находится по пути ./architecture/telegram\-notify\-app\-overview.md\). Документы взаимосвязаны с помощью ссылок \`related\_docs\`, \`parent\`/\`children\` и обычных markdown\-ссылок без повторения одной и той же информации. Файлы, использованные для ответа: docs/README.md
|
||||
- **v2-general-05-main-parts**
|
||||
Query: Какие основные части есть у системы?
|
||||
> Основные части любой системы включают вход, процессоры \(обработчики\), память и выход.
|
||||
- **v2-general-06-high-level-architecture**
|
||||
Query: Как в целом устроено приложение?
|
||||
> Приложение обычно состоит из пользовательского интерфейса \(то, что видит и с чем взаимодействует пользователь\), логики приложения \(код, который обрабатывает действия пользователя\) и базы данных \(место хранения информации\). Все эти части связаны между собой для обеспечения удобной работы пользователя.
|
||||
- **v2-general-07-besides-api**
|
||||
Query: Что тут есть кроме API?
|
||||
> Кроме API, здесь есть дружелюбная атмосфера, готовность помочь и интересные беседы на различные темы.
|
||||
- **v2-general-08-components**
|
||||
Query: Из чего состоит Telegram Notify App?
|
||||
> Telegram Notify App состоит из простого пользовательского интерфейса для отправки уведомлений, механизма обработки сообщений и интеграции с API Telegram, позволяющей отправлять сообщения в выбранные чаты или каналы.
|
||||
- **v2-general-09-reading-order**
|
||||
Query: Какие документы стоит читать сначала?
|
||||
> Не нашёл подходящих SUMMARY\-блоков в документации по этому запросу.
|
||||
- **v2-general-10-short-summary**
|
||||
Query: Дай короткое summary по проектной документации
|
||||
> Проектная документация включает описание целей, задач, методов, графика и бюджета проекта, а также список необходимых ресурсов и команду исполнителей.
|
||||
- **v2-docs-summary-01-health**
|
||||
Query: Кратко объясни по документации, как работает `/health`
|
||||
> Информация о том, как именно работает \`/health\`, не представлена в предоставленных SUMMARY\-блоках. Файлы, на которые я опирался: \- docs/architecture/telegram\-notify\-app\-overview.md
|
||||
- **v2-docs-summary-02-send**
|
||||
Query: Что делает endpoint `/send`?
|
||||
> На основе представленного SUMMARY\-блока, документация о том, что именно делает endpoint \`/send\`, отсутствует. В контексте указано, что он связан с отправкой сообщений, но подробностей его работы не приведено. Файлы, на которые опирался ответ: \- docs/architecture/telegram\-notify\-app\-overview.md
|
||||
- **v2-docs-summary-03-actions**
|
||||
Query: Объясни назначение `/actions/{action}`
|
||||
> \`/actions/\{action\}\` обычно используется в API для выполнения конкретных действий или операций, где \`\{action\}\` заменяется названием нужного действия. Это позволяет организовывать и вызывать различные функции через единый интерфейс.
|
||||
- **v2-docs-summary-04-runtime-health**
|
||||
Query: Что такое runtime health в этой документации?
|
||||
> Runtime health — это процесс мониторинга и обеспечения стабильности работы системы или приложения во время их выполнения \(runtime\), позволяющий своевременно обнаруживать и устранять возникающие проблемы.
|
||||
- **v2-docs-summary-05-notification-loop**
|
||||
Query: Кратко опиши цикл отправки уведомлений
|
||||
> Цикл отправки уведомлений включает следующие шаги: отлавливание события, обработка данных, выбор получателей, формирование сообщения и его отправка выбранным пользователям.
|
||||
- **v2-docs-summary-06-architecture**
|
||||
Query: Как устроена архитектура Telegram Notify App?
|
||||
> Архитектура Telegram Notify App основана на клиент\-серверной модели, где серверное приложение обрабатывает входящие данные и отправляет уведомления через Telegram API. Клиентское приложение взаимодействует с сервером для отправки уведомлений. Пользователь настраивает параметры уведомлений в приложении, а сервер передает их в виде сообщений в указанный Telegram\-канал или пользователю.
|
||||
- **v2-docs-summary-07-worker**
|
||||
Query: Что делает worker в этом приложении?
|
||||
> Worker в этом приложении выполняет фоновые задачи или процессы, позволяя основному приложению работать быстро и плавно. Это может включать обработку данных, отправку уведомлений, загрузку файлов и другие ресурсоемкие операции.
|
||||
- **v2-docs-summary-08-health-statuses**
|
||||
Query: Какие статусы здоровья есть у runtime?
|
||||
> На основе представленного SUMMARY\-блока конкретной информации о статусах здоровья runtime нет. Файл, связанный с доменом "runtime health" \(../domains/runtime\-health\-entity.md\), вероятно, содержит эту информацию, но она не включена в текущий контекст. Файлы, на которые опирался ответ: \- docs/architecture/telegram\-notify\-app\-overview.md
|
||||
- **v2-docs-summary-09-manual-send**
|
||||
Query: Как в приложении устроена ручная отправка сообщения?
|
||||
> Ручная отправка сообщения в приложении обычно осуществляется нажатием кнопки "Отправить" или аналогичного элемента интерфейса после того, как пользователь введет текст сообщения.
|
||||
- **v2-docs-summary-10-telegram-integration**
|
||||
Query: Как сервис взаимодействует с Telegram в целом?
|
||||
> Сервис может отправлять и получать сообщения, реагировать на команды и запросы пользователей через Telegram\-бота. Взаимодействие происходит посредством отправки запросов и получения ответов от бота в чате.
|
||||
- **v2-find-files-01-health**
|
||||
Query: В каком файле описан `/health`?
|
||||
> Нашёл релевантные файлы документации: \- \`docs/README.md\` \- \`docs/architecture/telegram\-notify\-app\-overview.md\`
|
||||
- **v2-find-files-02-send**
|
||||
Query: Покажи файл с описанием endpoint `/send`
|
||||
> Нашёл релевантные файлы документации: \- \`docs/README.md\` \- \`docs/architecture/telegram\-notify\-app\-overview.md\`
|
||||
- **v2-find-files-03-actions**
|
||||
Query: Где находится документация по `/actions/{action}`?
|
||||
> Нашёл релевантные файлы документации: \- \`docs/README.md\` \- \`docs/architecture/telegram\-notify\-app\-overview.md\`
|
||||
- **v2-find-files-04-runtime-health**
|
||||
Query: В каком документе описан runtime health?
|
||||
> Нашёл релевантные файлы документации: \- \`docs/architecture/telegram\-notify\-app\-overview.md\` \- \`docs/README.md\`
|
||||
- **v2-find-files-05-loop**
|
||||
Query: Найди файл с логикой цикла отправки уведомлений
|
||||
> Нашёл релевантные файлы документации: \- \`docs/README.md\` \- \`docs/architecture/telegram\-notify\-app\-overview.md\`
|
||||
- **v2-find-files-06-architecture**
|
||||
Query: Где описана архитектура приложения?
|
||||
> Нашёл релевантные файлы документации: \- \`docs/README.md\` \- \`docs/architecture/telegram\-notify\-app\-overview.md\`
|
||||
- **v2-find-files-07-errors-catalog**
|
||||
Query: В каком файле лежит каталог ошибок?
|
||||
> Нашёл релевантные файлы документации: \- \`docs/README.md\`
|
||||
- **v2-find-files-08-docs-index**
|
||||
Query: Покажи файл-индекс документации проекта
|
||||
> Нашёл релевантные файлы документации: \- \`docs/README.md\`
|
||||
- **v2-find-files-09-telegram-bot-api**
|
||||
Query: Какие файлы относятся к Telegram Bot API интеграции?
|
||||
> Нашёл релевантные файлы документации: \- \`docs/README.md\` \- \`docs/architecture/telegram\-notify\-app\-overview.md\`
|
||||
- **v2-find-files-10-telegram-notify-worker**
|
||||
Query: В каких файлах описан worker `telegram_notify`?
|
||||
> Нашёл релевантные файлы документации: \- \`docs/architecture/telegram\-notify\-app\-overview.md\`
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-01-health",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"answer_mode: expected grounded_summary, got insufficient_evidence",
|
||||
"llm: expected answer to contain '/health'",
|
||||
"llm: expected answer to contain 'status'",
|
||||
"llm: expected answer to contain 'components'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%health-endpoint.md%",
|
||||
"%/health%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/health-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"endpoint_paths": [
|
||||
"/health"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/health-endpoint.md"
|
||||
],
|
||||
"matched_aliases": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%health-endpoint.md%",
|
||||
"%/health%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/health-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+186
@@ -0,0 +1,186 @@
|
||||
# v2-docs-summary-01-health
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Кратко объясни по документации, как работает `/health`
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%health-endpoint.md%",
|
||||
"%/health%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/health-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"endpoint_paths": [
|
||||
"/health"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/health-endpoint.md"
|
||||
],
|
||||
"matched_aliases": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%health-endpoint.md%",
|
||||
"%/health%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/health-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- answer_mode: expected grounded_summary, got insufficient_evidence
|
||||
- llm: expected answer to contain '/health'
|
||||
- llm: expected answer to contain 'status'
|
||||
- llm: expected answer to contain 'components'
|
||||
+152
@@ -0,0 +1,152 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-02-send",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']",
|
||||
"llm: expected answer to contain '/send'",
|
||||
"llm: expected answer to contain 'message'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%send-message-endpoint.md%",
|
||||
"%/send%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"endpoint_paths": [
|
||||
"/send"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"matched_aliases": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%send-message-endpoint.md%",
|
||||
"%/send%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+185
@@ -0,0 +1,185 @@
|
||||
# v2-docs-summary-02-send
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Что делает endpoint `/send`?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%send-message-endpoint.md%",
|
||||
"%/send%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"endpoint_paths": [
|
||||
"/send"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"matched_aliases": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%send-message-endpoint.md%",
|
||||
"%/send%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
|
||||
- llm: expected answer to contain '/send'
|
||||
- llm: expected answer to contain 'message'
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-03-actions",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"doc_scope should contain 'docs/api/control-actions-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']",
|
||||
"llm: expected answer to contain '/actions/{action}'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramControlChannel",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%control-actions-endpoint.md%",
|
||||
"%/actions/{action}%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/control-actions-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"endpoint_paths": [
|
||||
"/actions/{action}"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/control-actions-endpoint.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"/actions/{action}"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%control-actions-endpoint.md%",
|
||||
"%/actions/{action}%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/control-actions-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+186
@@ -0,0 +1,186 @@
|
||||
# v2-docs-summary-03-actions
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Объясни назначение `/actions/{action}`
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramControlChannel",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%control-actions-endpoint.md%",
|
||||
"%/actions/{action}%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/control-actions-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"endpoint_paths": [
|
||||
"/actions/{action}"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/control-actions-endpoint.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"/actions/{action}"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%control-actions-endpoint.md%",
|
||||
"%/actions/{action}%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/control-actions-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- doc_scope should contain 'docs/api/control-actions-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
|
||||
- llm: expected answer to contain '/actions/{action}'
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-04-runtime-health",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"doc_scope should contain 'docs/domains/runtime-health-entity.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md']",
|
||||
"llm: expected answer to contain 'runtime health'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/domains/",
|
||||
"docs/api/",
|
||||
"docs/architecture/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%runtime-health-entity.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"runtime-health-entity"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_summary_domain_entity",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/domains/",
|
||||
"docs/api/",
|
||||
"docs/architecture/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%runtime-health-entity.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_summary_domain_entity",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
# v2-docs-summary-04-runtime-health
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Что такое runtime health в этой документации?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/domains/",
|
||||
"docs/api/",
|
||||
"docs/architecture/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%runtime-health-entity.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"runtime-health-entity"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_domain_entity",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/domains/",
|
||||
"docs/api/",
|
||||
"docs/architecture/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%runtime-health-entity.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_domain_entity",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- doc_scope should contain 'docs/domains/runtime-health-entity.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md']
|
||||
- llm: expected answer to contain 'runtime health'
|
||||
+143
@@ -0,0 +1,143 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-05-notification-loop",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"doc_scope should contain 'docs/logic/telegram-notification-loop.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']",
|
||||
"llm: expected answer to contain 'worker'",
|
||||
"llm: expected answer to contain 'interval'",
|
||||
"llm: expected answer to contain 'degraded'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "LOGIC_FLOW",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md, docs/README.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/logic/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%telegram-notification-loop.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/logic/telegram-notification-loop.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "LOGIC_FLOW",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": "LOGIC_FLOW",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [
|
||||
"docs/logic/telegram-notification-loop.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"telegram-notification-loop"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_summary_logic_flow",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/logic/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%telegram-notification-loop.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/logic/telegram-notification-loop.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_summary_logic_flow",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 147
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+176
@@ -0,0 +1,176 @@
|
||||
# v2-docs-summary-05-notification-loop
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Кратко опиши цикл отправки уведомлений
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "LOGIC_FLOW",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md, docs/README.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/logic/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%telegram-notification-loop.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/logic/telegram-notification-loop.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "LOGIC_FLOW",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": "LOGIC_FLOW",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [
|
||||
"docs/logic/telegram-notification-loop.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"telegram-notification-loop"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_logic_flow",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/logic/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%telegram-notification-loop.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/logic/telegram-notification-loop.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_logic_flow",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 2
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 147
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- doc_scope should contain 'docs/logic/telegram-notification-loop.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
|
||||
- llm: expected answer to contain 'worker'
|
||||
- llm: expected answer to contain 'interval'
|
||||
- llm: expected answer to contain 'degraded'
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-06-architecture",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": true,
|
||||
"mismatches": [],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "ARCHITECTURE",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "- Purpose: сервис поднимает HTTP control plane и фоновый worker для отправки уведомлений в Telegram.\n- Entry point: `src/telegram_notify_app/main.py`.\n- Main components: `RuntimeManager`, `TelegramControlChannel`, `TelegramNotifyModule`, `TelegramNotifyWorker`, `TelegramSendService`.\n- Configuration: `config/config.yaml` или путь из `CONFIG_PATH`.\n- Related API: [`/health`](../api/health-endpoint.md), [`/actions/{action}`](../api/control-actions-endpoint.md), [`/send`](../api/send-message-endpoint.md).\n- Related logic: [цикл отправки уведомлений](../logic/telegram-notification-loop.md).\n- Related domain: [runtime health](../domains/runtime-health-entity.md).\n\nФайлы-источники:\n- docs/architecture/telegram-notify-app-overview.md\n- docs/README.md",
|
||||
"answer_mode": "grounded_summary",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "ARCHITECTURE",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": "ARCHITECTURE",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [],
|
||||
"matched_aliases": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_summary_architecture",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_summary_architecture",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": true,
|
||||
"reason": "target_doc_found",
|
||||
"answer_mode": "grounded_summary"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "grounded_summary",
|
||||
"answer_length": 753
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+159
@@ -0,0 +1,159 @@
|
||||
# v2-docs-summary-06-architecture
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: True
|
||||
|
||||
## Query
|
||||
Как устроена архитектура Telegram Notify App?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "ARCHITECTURE",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "- Purpose: сервис поднимает HTTP control plane и фоновый worker для отправки уведомлений в Telegram.\n- Entry point: `src/telegram_notify_app/main.py`.\n- Main components: `RuntimeManager`, `TelegramControlChannel`, `TelegramNotifyModule`, `TelegramNotifyWorker`, `TelegramSendService`.\n- Configuration: `config/config.yaml` или путь из `CONFIG_PATH`.\n- Related API: [`/health`](../api/health-endpoint.md), [`/actions/{action}`](../api/control-actions-endpoint.md), [`/send`](../api/send-message-endpoint.md).\n- Related logic: [цикл отправки уведомлений](../logic/telegram-notification-loop.md).\n- Related domain: [runtime health](../domains/runtime-health-entity.md).\n\nФайлы-источники:\n- docs/architecture/telegram-notify-app-overview.md\n- docs/README.md",
|
||||
"answer_mode": "grounded_summary",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "ARCHITECTURE",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": "ARCHITECTURE",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [],
|
||||
"matched_aliases": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_architecture",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_architecture",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 2
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": true,
|
||||
"reason": "target_doc_found",
|
||||
"answer_mode": "grounded_summary"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "grounded_summary",
|
||||
"answer_length": 753
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- none
|
||||
+129
@@ -0,0 +1,129 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-07-worker",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"doc_scope should contain 'docs/logic/telegram-notification-loop.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']",
|
||||
"llm: expected answer to contain 'worker'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "LOGIC_FLOW",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/logic/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "LOGIC_FLOW",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": "LOGIC_FLOW",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [],
|
||||
"matched_aliases": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_summary_logic_flow",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/logic/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_summary_logic_flow",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
# v2-docs-summary-07-worker
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Что делает worker в этом приложении?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "LOGIC_FLOW",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/logic/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "LOGIC_FLOW",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": "LOGIC_FLOW",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [],
|
||||
"matched_aliases": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_logic_flow",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/logic/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_logic_flow",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- doc_scope should contain 'docs/logic/telegram-notification-loop.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
|
||||
- llm: expected answer to contain 'worker'
|
||||
+143
@@ -0,0 +1,143 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-08-health-statuses",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"doc_scope should contain 'docs/domains/runtime-health-entity.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md']",
|
||||
"doc_scope should contain 'docs/api/health-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md']",
|
||||
"llm: expected answer to contain 'ok'",
|
||||
"llm: expected answer to contain 'degraded'",
|
||||
"llm: expected answer to contain 'unhealthy'",
|
||||
"llm: expected answer to contain 'unknown'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/domains/",
|
||||
"docs/api/",
|
||||
"docs/architecture/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%runtime-health-entity.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"runtime-health-entity"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_summary_domain_entity",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/domains/",
|
||||
"docs/api/",
|
||||
"docs/architecture/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%runtime-health-entity.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_summary_domain_entity",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+176
@@ -0,0 +1,176 @@
|
||||
# v2-docs-summary-08-health-statuses
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Какие статусы здоровья есть у runtime?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/domains/",
|
||||
"docs/api/",
|
||||
"docs/architecture/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%runtime-health-entity.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"runtime-health-entity"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_domain_entity",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/domains/",
|
||||
"docs/api/",
|
||||
"docs/architecture/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%runtime-health-entity.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_domain_entity",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- doc_scope should contain 'docs/domains/runtime-health-entity.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md']
|
||||
- doc_scope should contain 'docs/api/health-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md']
|
||||
- llm: expected answer to contain 'ok'
|
||||
- llm: expected answer to contain 'degraded'
|
||||
- llm: expected answer to contain 'unhealthy'
|
||||
- llm: expected answer to contain 'unknown'
|
||||
+151
@@ -0,0 +1,151 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-09-manual-send",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']",
|
||||
"llm: expected answer to contain '/send'",
|
||||
"llm: expected answer to contain 'message'",
|
||||
"llm: expected answer to contain 'TelegramSendService'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%send-message-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"/send"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%send-message-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+184
@@ -0,0 +1,184 @@
|
||||
# v2-docs-summary-09-manual-send
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Как в приложении устроена ручная отправка сообщения?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%send-message-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"/send"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%send-message-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
|
||||
- llm: expected answer to contain '/send'
|
||||
- llm: expected answer to contain 'message'
|
||||
- llm: expected answer to contain 'TelegramSendService'
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-10-telegram-integration",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"domain: expected DOCS, got GENERAL",
|
||||
"intent: expected DOC_EXPLAIN, got GENERAL_QA",
|
||||
"anchor_type: expected API_ENDPOINT, got None",
|
||||
"doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": null,
|
||||
"rag_count": 8,
|
||||
"llm_answer": "- Purpose: точка входа в техническую документацию сервиса `test_echo_app`.\n- Scope: архитектура, HTTP API control plane, цикл отправки уведомлений, health-модель и каталог ошибок.\n- Canonical structure: `docs/architecture`, `docs/api`, `docs/logic`, `docs/domains`, `docs/errors`.\n- Primary parent doc: [Архитектура Telegram Notify App](./architecture/telegram-notify-app-overview.md).\n- Navigation: документы связаны через `related_docs`, `parent`/`children` и markdown-ссылки без дублирования деталей.\n- Purpose: сервис поднимает HTTP control plane и фоновый worker для отправки уведомлений в Telegram.\n- Entry point: `src/telegram_notify_app/main.py`.\n- Main components: `RuntimeManager`, `TelegramControlChannel`, `TelegramNotifyModule`, `TelegramNotifyWorker`, `TelegramSendService`.\n- Configuration: `config/config.yaml` или путь из `CONFIG_PATH`.\n- Related API: [`/health`](../api/health-endpoint.md), [`/actions/{action}`](../api/control-actions-endpoint.md), [`/send`](../api/send-message-endpoint.md).\n- Related logic: [цикл отправки уведомлений](../logic/telegram-notification-loop.md).\n- Related domain: [runtime health](../domains/runtime-health-entity.md).",
|
||||
"answer_mode": "grounded_summary",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%README.md%",
|
||||
"%overview%"
|
||||
],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": null,
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": null,
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [],
|
||||
"matched_aliases": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "general_qa_grounded_summary",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%README.md%",
|
||||
"%overview%"
|
||||
],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "general_qa_grounded_summary",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/README.md",
|
||||
"document_count": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": true,
|
||||
"reason": "general_docs_found",
|
||||
"answer_mode": "grounded_summary"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "grounded_summary",
|
||||
"answer_length": 1170
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
# v2-docs-summary-10-telegram-integration
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Как сервис взаимодействует с Telegram в целом?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": null,
|
||||
"rag_count": 8,
|
||||
"llm_answer": "- Purpose: точка входа в техническую документацию сервиса `test_echo_app`.\n- Scope: архитектура, HTTP API control plane, цикл отправки уведомлений, health-модель и каталог ошибок.\n- Canonical structure: `docs/architecture`, `docs/api`, `docs/logic`, `docs/domains`, `docs/errors`.\n- Primary parent doc: [Архитектура Telegram Notify App](./architecture/telegram-notify-app-overview.md).\n- Navigation: документы связаны через `related_docs`, `parent`/`children` и markdown-ссылки без дублирования деталей.\n- Purpose: сервис поднимает HTTP control plane и фоновый worker для отправки уведомлений в Telegram.\n- Entry point: `src/telegram_notify_app/main.py`.\n- Main components: `RuntimeManager`, `TelegramControlChannel`, `TelegramNotifyModule`, `TelegramNotifyWorker`, `TelegramSendService`.\n- Configuration: `config/config.yaml` или путь из `CONFIG_PATH`.\n- Related API: [`/health`](../api/health-endpoint.md), [`/actions/{action}`](../api/control-actions-endpoint.md), [`/send`](../api/send-message-endpoint.md).\n- Related logic: [цикл отправки уведомлений](../logic/telegram-notification-loop.md).\n- Related domain: [runtime health](../domains/runtime-health-entity.md).",
|
||||
"answer_mode": "grounded_summary",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%README.md%",
|
||||
"%overview%"
|
||||
],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": null,
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": null,
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [],
|
||||
"matched_aliases": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "general_qa_grounded_summary",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%README.md%",
|
||||
"%overview%"
|
||||
],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "general_qa_grounded_summary",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/README.md",
|
||||
"document_count": 2
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": true,
|
||||
"reason": "general_docs_found",
|
||||
"answer_mode": "grounded_summary"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "grounded_summary",
|
||||
"answer_length": 1170
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- domain: expected DOCS, got GENERAL
|
||||
- intent: expected DOC_EXPLAIN, got GENERAL_QA
|
||||
- anchor_type: expected API_ENDPOINT, got None
|
||||
- doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
|
||||
+137
@@ -0,0 +1,137 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-01-health",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"answer_mode: expected deterministic, got deterministic_file_list",
|
||||
"llm: expected answer to contain 'docs/api/health-endpoint.md'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "docs/architecture/telegram-notify-app-overview.md\ndocs/README.md",
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"RuntimeManager",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%health-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/health-endpoint.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": null,
|
||||
"endpoint_paths": [
|
||||
"/health"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/health-endpoint.md"
|
||||
],
|
||||
"matched_aliases": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%health-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/health-endpoint.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"row_count": 12,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "find_files",
|
||||
"primary_file": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"file_count": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": true,
|
||||
"reason": "primary_file_confident",
|
||||
"answer_mode": "deterministic_file_list"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"answer_length": 64
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+170
@@ -0,0 +1,170 @@
|
||||
# v2-find-files-01-health
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
В каком файле описан `/health`?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "docs/architecture/telegram-notify-app-overview.md\ndocs/README.md",
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"RuntimeManager",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%health-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/health-endpoint.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": null,
|
||||
"endpoint_paths": [
|
||||
"/health"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/health-endpoint.md"
|
||||
],
|
||||
"matched_aliases": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%health-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/health-endpoint.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"row_count": 12,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "find_files",
|
||||
"primary_file": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"file_count": 2
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": true,
|
||||
"reason": "primary_file_confident",
|
||||
"answer_mode": "deterministic_file_list"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"answer_length": 64
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- answer_mode: expected deterministic, got deterministic_file_list
|
||||
- llm: expected answer to contain 'docs/api/health-endpoint.md'
|
||||
+137
@@ -0,0 +1,137 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-02-send",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"answer_mode: expected deterministic, got deterministic_file_list",
|
||||
"llm: expected answer to contain 'docs/api/send-message-endpoint.md'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "docs/architecture/telegram-notify-app-overview.md\ndocs/README.md",
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%send-message-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": null,
|
||||
"endpoint_paths": [
|
||||
"/send"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"matched_aliases": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%send-message-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"row_count": 12,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "find_files",
|
||||
"primary_file": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"file_count": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": true,
|
||||
"reason": "primary_file_confident",
|
||||
"answer_mode": "deterministic_file_list"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"answer_length": 64
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+170
@@ -0,0 +1,170 @@
|
||||
# v2-find-files-02-send
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Покажи файл с описанием endpoint `/send`
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "docs/architecture/telegram-notify-app-overview.md\ndocs/README.md",
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%send-message-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": null,
|
||||
"endpoint_paths": [
|
||||
"/send"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"matched_aliases": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%send-message-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"row_count": 12,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "find_files",
|
||||
"primary_file": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"file_count": 2
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": true,
|
||||
"reason": "primary_file_confident",
|
||||
"answer_mode": "deterministic_file_list"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"answer_length": 64
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- answer_mode: expected deterministic, got deterministic_file_list
|
||||
- llm: expected answer to contain 'docs/api/send-message-endpoint.md'
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-03-actions",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"answer_mode: expected deterministic, got deterministic_file_list",
|
||||
"llm: expected answer to contain 'docs/api/control-actions-endpoint.md'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "docs/architecture/telegram-notify-app-overview.md\ndocs/README.md",
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%control-actions-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/control-actions-endpoint.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": null,
|
||||
"endpoint_paths": [
|
||||
"/actions/{action}"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/control-actions-endpoint.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"/actions/{action}"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%control-actions-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/control-actions-endpoint.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"row_count": 12,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "find_files",
|
||||
"primary_file": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"file_count": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": true,
|
||||
"reason": "primary_file_confident",
|
||||
"answer_mode": "deterministic_file_list"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"answer_length": 64
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user