Роутер работает нормально в process v2
This commit is contained in:
+43
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-01-health",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"llm: expected answer to contain 'status'",
|
||||
"llm: expected answer to contain 'components'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "Информация о том, как именно работает `/health`, не представлена в предоставленных SUMMARY-блоках. \n\nФайлы, на которые я опирался:\n- docs/architecture/telegram-notify-app-overview.md",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
# v2-docs-summary-01-health
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Кратко объясни по документации, как работает `/health`
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "Информация о том, как именно работает `/health`, не представлена в предоставленных SUMMARY-блоках. \n\nФайлы, на которые я опирался:\n- docs/architecture/telegram-notify-app-overview.md",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- llm: expected answer to contain 'status'
|
||||
- llm: expected answer to contain 'components'
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-02-send",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']",
|
||||
"llm: expected answer to contain 'message'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "На основе представленного SUMMARY-блока, документация о том, что именно делает endpoint `/send`, отсутствует. В контексте указано, что он связан с отправкой сообщений, но подробностей его работы не приведено.\n\nФайлы, на которые опирался ответ:\n- docs/architecture/telegram-notify-app-overview.md",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
# v2-docs-summary-02-send
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Что делает endpoint `/send`?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "На основе представленного SUMMARY-блока, документация о том, что именно делает endpoint `/send`, отсутствует. В контексте указано, что он связан с отправкой сообщений, но подробностей его работы не приведено.\n\nФайлы, на которые опирался ответ:\n- docs/architecture/telegram-notify-app-overview.md",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
|
||||
- llm: expected answer to contain 'message'
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-03-actions",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"domain: expected DOCS, got GENERAL",
|
||||
"intent: expected DOC_EXPLAIN, got GENERAL_QA",
|
||||
"retrieval: expected non-empty rag, got 0 rows",
|
||||
"doc_scope should contain 'docs/api/control-actions-endpoint.md', got []"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "`/actions/{action}` обычно используется в API для выполнения конкретных действий или операций, где `{action}` заменяется названием нужного действия. Это позволяет организовывать и вызывать различные функции через единый интерфейс.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
# v2-docs-summary-03-actions
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Объясни назначение `/actions/{action}`
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "`/actions/{action}` обычно используется в API для выполнения конкретных действий или операций, где `{action}` заменяется названием нужного действия. Это позволяет организовывать и вызывать различные функции через единый интерфейс.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- domain: expected DOCS, got GENERAL
|
||||
- intent: expected DOC_EXPLAIN, got GENERAL_QA
|
||||
- retrieval: expected non-empty rag, got 0 rows
|
||||
- doc_scope should contain 'docs/api/control-actions-endpoint.md', got []
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-04-runtime-health",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"domain: expected DOCS, got GENERAL",
|
||||
"intent: expected DOC_EXPLAIN, got GENERAL_QA",
|
||||
"retrieval: expected non-empty rag, got 0 rows",
|
||||
"doc_scope should contain 'docs/domains/runtime-health-entity.md', got []"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Runtime health — это процесс мониторинга и обеспечения стабильности работы системы или приложения во время их выполнения (runtime), позволяющий своевременно обнаруживать и устранять возникающие проблемы.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
# v2-docs-summary-04-runtime-health
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Что такое runtime health в этой документации?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Runtime health — это процесс мониторинга и обеспечения стабильности работы системы или приложения во время их выполнения (runtime), позволяющий своевременно обнаруживать и устранять возникающие проблемы.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- domain: expected DOCS, got GENERAL
|
||||
- intent: expected DOC_EXPLAIN, got GENERAL_QA
|
||||
- retrieval: expected non-empty rag, got 0 rows
|
||||
- doc_scope should contain 'docs/domains/runtime-health-entity.md', got []
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-05-notification-loop",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"domain: expected DOCS, got GENERAL",
|
||||
"intent: expected DOC_EXPLAIN, got GENERAL_QA",
|
||||
"retrieval: expected non-empty rag, got 0 rows",
|
||||
"doc_scope should contain 'docs/logic/telegram-notification-loop.md', got []",
|
||||
"llm: expected answer to contain 'worker'",
|
||||
"llm: expected answer to contain 'interval'",
|
||||
"llm: expected answer to contain 'degraded'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Цикл отправки уведомлений включает следующие шаги: отлавливание события, обработка данных, выбор получателей, формирование сообщения и его отправка выбранным пользователям.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
# v2-docs-summary-05-notification-loop
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Кратко опиши цикл отправки уведомлений
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Цикл отправки уведомлений включает следующие шаги: отлавливание события, обработка данных, выбор получателей, формирование сообщения и его отправка выбранным пользователям.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- domain: expected DOCS, got GENERAL
|
||||
- intent: expected DOC_EXPLAIN, got GENERAL_QA
|
||||
- retrieval: expected non-empty rag, got 0 rows
|
||||
- doc_scope should contain 'docs/logic/telegram-notification-loop.md', got []
|
||||
- llm: expected answer to contain 'worker'
|
||||
- llm: expected answer to contain 'interval'
|
||||
- llm: expected answer to contain 'degraded'
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-06-architecture",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"domain: expected DOCS, got GENERAL",
|
||||
"intent: expected DOC_EXPLAIN, got GENERAL_QA",
|
||||
"retrieval: expected non-empty rag, got 0 rows",
|
||||
"doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []",
|
||||
"llm: expected answer to contain 'RuntimeManager'",
|
||||
"llm: expected answer to contain 'TelegramControlChannel'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Архитектура Telegram Notify App основана на клиент-серверной модели, где серверное приложение обрабатывает входящие данные и отправляет уведомления через Telegram API. Клиентское приложение взаимодействует с сервером для отправки уведомлений. Пользователь настраивает параметры уведомлений в приложении, а сервер передает их в виде сообщений в указанный Telegram-канал или пользователю.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
# v2-docs-summary-06-architecture
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Как устроена архитектура Telegram Notify App?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Архитектура Telegram Notify App основана на клиент-серверной модели, где серверное приложение обрабатывает входящие данные и отправляет уведомления через Telegram API. Клиентское приложение взаимодействует с сервером для отправки уведомлений. Пользователь настраивает параметры уведомлений в приложении, а сервер передает их в виде сообщений в указанный Telegram-канал или пользователю.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- domain: expected DOCS, got GENERAL
|
||||
- intent: expected DOC_EXPLAIN, got GENERAL_QA
|
||||
- retrieval: expected non-empty rag, got 0 rows
|
||||
- doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []
|
||||
- llm: expected answer to contain 'RuntimeManager'
|
||||
- llm: expected answer to contain 'TelegramControlChannel'
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-07-worker",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"domain: expected DOCS, got GENERAL",
|
||||
"intent: expected DOC_EXPLAIN, got GENERAL_QA",
|
||||
"retrieval: expected non-empty rag, got 0 rows",
|
||||
"doc_scope should contain 'docs/logic/telegram-notification-loop.md', got []",
|
||||
"doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []",
|
||||
"llm: expected answer to contain 'Telegram'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Worker в этом приложении выполняет фоновые задачи или процессы, позволяя основному приложению работать быстро и плавно. Это может включать обработку данных, отправку уведомлений, загрузку файлов и другие ресурсоемкие операции.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
# v2-docs-summary-07-worker
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Что делает worker в этом приложении?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Worker в этом приложении выполняет фоновые задачи или процессы, позволяя основному приложению работать быстро и плавно. Это может включать обработку данных, отправку уведомлений, загрузку файлов и другие ресурсоемкие операции.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- domain: expected DOCS, got GENERAL
|
||||
- intent: expected DOC_EXPLAIN, got GENERAL_QA
|
||||
- retrieval: expected non-empty rag, got 0 rows
|
||||
- doc_scope should contain 'docs/logic/telegram-notification-loop.md', got []
|
||||
- doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []
|
||||
- llm: expected answer to contain 'Telegram'
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-08-health-statuses",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"doc_scope should contain 'docs/domains/runtime-health-entity.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']",
|
||||
"doc_scope should contain 'docs/api/health-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']",
|
||||
"llm: expected answer to contain 'ok'",
|
||||
"llm: expected answer to contain 'degraded'",
|
||||
"llm: expected answer to contain 'unhealthy'",
|
||||
"llm: expected answer to contain 'unknown'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "На основе представленного SUMMARY-блока конкретной информации о статусах здоровья runtime нет. Файл, связанный с доменом \"runtime health\" (../domains/runtime-health-entity.md), вероятно, содержит эту информацию, но она не включена в текущий контекст.\n\nФайлы, на которые опирался ответ:\n- docs/architecture/telegram-notify-app-overview.md",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+55
@@ -0,0 +1,55 @@
|
||||
# v2-docs-summary-08-health-statuses
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Какие статусы здоровья есть у runtime?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "На основе представленного SUMMARY-блока конкретной информации о статусах здоровья runtime нет. Файл, связанный с доменом \"runtime health\" (../domains/runtime-health-entity.md), вероятно, содержит эту информацию, но она не включена в текущий контекст.\n\nФайлы, на которые опирался ответ:\n- docs/architecture/telegram-notify-app-overview.md",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- doc_scope should contain 'docs/domains/runtime-health-entity.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
|
||||
- doc_scope should contain 'docs/api/health-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
|
||||
- llm: expected answer to contain 'ok'
|
||||
- llm: expected answer to contain 'degraded'
|
||||
- llm: expected answer to contain 'unhealthy'
|
||||
- llm: expected answer to contain 'unknown'
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-09-manual-send",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"domain: expected DOCS, got GENERAL",
|
||||
"intent: expected DOC_EXPLAIN, got GENERAL_QA",
|
||||
"retrieval: expected non-empty rag, got 0 rows",
|
||||
"doc_scope should contain 'docs/api/send-message-endpoint.md', got []",
|
||||
"llm: expected answer to contain '/send'",
|
||||
"llm: expected answer to contain 'message'",
|
||||
"llm: expected answer to contain 'TelegramSendService'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Ручная отправка сообщения в приложении обычно осуществляется нажатием кнопки \"Отправить\" или аналогичного элемента интерфейса после того, как пользователь введет текст сообщения.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
# v2-docs-summary-09-manual-send
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Как в приложении устроена ручная отправка сообщения?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Ручная отправка сообщения в приложении обычно осуществляется нажатием кнопки \"Отправить\" или аналогичного элемента интерфейса после того, как пользователь введет текст сообщения.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- domain: expected DOCS, got GENERAL
|
||||
- intent: expected DOC_EXPLAIN, got GENERAL_QA
|
||||
- retrieval: expected non-empty rag, got 0 rows
|
||||
- doc_scope should contain 'docs/api/send-message-endpoint.md', got []
|
||||
- llm: expected answer to contain '/send'
|
||||
- llm: expected answer to contain 'message'
|
||||
- llm: expected answer to contain 'TelegramSendService'
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-10-telegram-integration",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"domain: expected DOCS, got GENERAL",
|
||||
"intent: expected DOC_EXPLAIN, got GENERAL_QA",
|
||||
"retrieval: expected non-empty rag, got 0 rows",
|
||||
"retrieval: expected at least 2 rows, got 0",
|
||||
"doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []",
|
||||
"doc_scope should contain 'docs/api/send-message-endpoint.md', got []",
|
||||
"llm: expected answer to contain 'worker'",
|
||||
"llm: expected answer to contain '/send'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Сервис может отправлять и получать сообщения, реагировать на команды и запросы пользователей через Telegram-бота. Взаимодействие происходит посредством отправки запросов и получения ответов от бота в чате.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
# v2-docs-summary-10-telegram-integration
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Как сервис взаимодействует с Telegram в целом?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Сервис может отправлять и получать сообщения, реагировать на команды и запросы пользователей через Telegram-бота. Взаимодействие происходит посредством отправки запросов и получения ответов от бота в чате.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- domain: expected DOCS, got GENERAL
|
||||
- intent: expected DOC_EXPLAIN, got GENERAL_QA
|
||||
- retrieval: expected non-empty rag, got 0 rows
|
||||
- retrieval: expected at least 2 rows, got 0
|
||||
- doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []
|
||||
- doc_scope should contain 'docs/api/send-message-endpoint.md', got []
|
||||
- llm: expected answer to contain 'worker'
|
||||
- llm: expected answer to contain '/send'
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-01-health",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"llm: expected answer to contain 'docs/api/health-endpoint.md'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"RuntimeManager",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
# v2-find-files-01-health
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
В каком файле описан `/health`?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"RuntimeManager",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- llm: expected answer to contain 'docs/api/health-endpoint.md'
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-02-send",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"llm: expected answer to contain 'docs/api/send-message-endpoint.md'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
# v2-find-files-02-send
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Покажи файл с описанием endpoint `/send`
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- llm: expected answer to contain 'docs/api/send-message-endpoint.md'
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-03-actions",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"llm: expected answer to contain 'docs/api/control-actions-endpoint.md'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
# v2-find-files-03-actions
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Где находится документация по `/actions/{action}`?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- llm: expected answer to contain 'docs/api/control-actions-endpoint.md'
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-04-runtime-health",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"llm: expected answer to contain 'docs/domains/runtime-health-entity.md'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/architecture/telegram-notify-app-overview.md`\n- `docs/README.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
# v2-find-files-04-runtime-health
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
В каком документе описан runtime health?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/architecture/telegram-notify-app-overview.md`\n- `docs/README.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- llm: expected answer to contain 'docs/domains/runtime-health-entity.md'
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-05-loop",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService",
|
||||
"RuntimeManager",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
# v2-find-files-05-loop
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Найди файл с логикой цикла отправки уведомлений
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService",
|
||||
"RuntimeManager",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-06-architecture",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": true,
|
||||
"mismatches": [],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
# v2-find-files-06-architecture
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: True
|
||||
|
||||
## Query
|
||||
Где описана архитектура приложения?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- none
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-07-errors-catalog",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"llm: expected answer to contain 'docs/errors/catalog.yaml'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyWorker"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
# v2-find-files-07-errors-catalog
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
В каком файле лежит каталог ошибок?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyWorker"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- llm: expected answer to contain 'docs/errors/catalog.yaml'
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-08-docs-index",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": true,
|
||||
"mismatches": [],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
# v2-find-files-08-docs-index
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: True
|
||||
|
||||
## Query
|
||||
Покажи файл-индекс документации проекта
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- none
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-09-telegram-bot-api",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"llm: expected answer to contain 'docs/api/send-message-endpoint.md'",
|
||||
"llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
# v2-find-files-09-telegram-bot-api
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Какие файлы относятся к Telegram Bot API интеграции?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/README.md`\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- llm: expected answer to contain 'docs/api/send-message-endpoint.md'
|
||||
- llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-10-telegram-notify-worker",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'",
|
||||
"llm: expected answer to contain 'docs/domains/runtime-health-entity.md'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
# v2-find-files-10-telegram-notify-worker
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
В каких файлах описан worker `telegram_notify`?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"rag_count": 20,
|
||||
"llm_answer": "Нашёл релевантные файлы документации:\n- `docs/architecture/telegram-notify-app-overview.md`",
|
||||
"answer_mode": "deterministic",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'
|
||||
- llm: expected answer to contain 'docs/domains/runtime-health-entity.md'
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-general-01-what-is-service",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"llm: expected answer to contain 'Telegram'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Это зависит от контекста. Уточни, пожалуйста, о каком именно сервисе идет речь.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
# v2-general-01-what-is-service
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Что это за сервис?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Это зависит от контекста. Уточни, пожалуйста, о каком именно сервисе идет речь.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- llm: expected answer to contain 'Telegram'
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-general-02-purpose",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": true,
|
||||
"mismatches": [],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Test_echo_app нужен для тестирования и отладки приложений и систем, повторяя входящий запрос в качестве ответа, чтобы убедиться в правильной работе связи и функциональности.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
# v2-general-02-purpose
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: True
|
||||
|
||||
## Query
|
||||
Для чего нужен test_echo_app?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Test_echo_app нужен для тестирования и отладки приложений и систем, повторяя входящий запрос в качестве ответа, чтобы убедиться в правильной работе связи и функциональности.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- none
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-general-03-problem",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"llm: expected answer to contain 'Telegram'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Приложение решает задачу (указать конкретную задачу, например, управления контактами, планирования задач, учета финансов и т.д.) для пользователей путем предоставления удобного и интуитивно понятного интерфейса.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
# v2-general-03-problem
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Какую задачу решает это приложение?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Приложение решает задачу (указать конкретную задачу, например, управления контактами, планирования задач, учета финансов и т.д.) для пользователей путем предоставления удобного и интуитивно понятного интерфейса.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- llm: expected answer to contain 'Telegram'
|
||||
+43
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-general-04-docs-contents",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"domain: expected GENERAL, got DOCS",
|
||||
"intent: expected GENERAL_QA, got DOC_EXPLAIN"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "Документация проекта включает следующие компоненты:\n\n- Архитектура приложения `test_echo_app`\n- Описание HTTP API (control plane)\n- Цикл отправки уведомлений\n- Health-модель\n- Каталог ошибок\n\nСтруктура документации канонически разделена на несколько секций: `docs/architecture`, `docs/api`, `docs/logic`, `docs/domains`, `docs/errors`. Основная родительская документация — \"Архитектура Telegram Notify App\" (находится по пути ./architecture/telegram-notify-app-overview.md). Документы взаимосвязаны с помощью ссылок `related_docs`, `parent`/`children` и обычных markdown-ссылок без повторения одной и той же информации.\n\nФайлы, использованные для ответа: docs/README.md",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
# v2-general-04-docs-contents
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Что входит в документацию этого проекта?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "Документация проекта включает следующие компоненты:\n\n- Архитектура приложения `test_echo_app`\n- Описание HTTP API (control plane)\n- Цикл отправки уведомлений\n- Health-модель\n- Каталог ошибок\n\nСтруктура документации канонически разделена на несколько секций: `docs/architecture`, `docs/api`, `docs/logic`, `docs/domains`, `docs/errors`. Основная родительская документация — \"Архитектура Telegram Notify App\" (находится по пути ./architecture/telegram-notify-app-overview.md). Документы взаимосвязаны с помощью ссылок `related_docs`, `parent`/`children` и обычных markdown-ссылок без повторения одной и той же информации.\n\nФайлы, использованные для ответа: docs/README.md",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- domain: expected GENERAL, got DOCS
|
||||
- intent: expected GENERAL_QA, got DOC_EXPLAIN
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-general-05-main-parts",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"llm: expected answer to contain 'Telegram'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Основные части любой системы включают вход, процессоры (обработчики), память и выход.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
# v2-general-05-main-parts
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Какие основные части есть у системы?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Основные части любой системы включают вход, процессоры (обработчики), память и выход.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- llm: expected answer to contain 'Telegram'
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-general-06-high-level-architecture",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": true,
|
||||
"mismatches": [],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Приложение обычно состоит из пользовательского интерфейса (то, что видит и с чем взаимодействует пользователь), логики приложения (код, который обрабатывает действия пользователя) и базы данных (место хранения информации). Все эти части связаны между собой для обеспечения удобной работы пользователя.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
# v2-general-06-high-level-architecture
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: True
|
||||
|
||||
## Query
|
||||
Как в целом устроено приложение?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Приложение обычно состоит из пользовательского интерфейса (то, что видит и с чем взаимодействует пользователь), логики приложения (код, который обрабатывает действия пользователя) и базы данных (место хранения информации). Все эти части связаны между собой для обеспечения удобной работы пользователя.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- none
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-general-07-besides-api",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": true,
|
||||
"mismatches": [],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Кроме API, здесь есть дружелюбная атмосфера, готовность помочь и интересные беседы на различные темы.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
# v2-general-07-besides-api
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: True
|
||||
|
||||
## Query
|
||||
Что тут есть кроме API?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Кроме API, здесь есть дружелюбная атмосфера, готовность помочь и интересные беседы на различные темы.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- none
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-general-08-components",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": true,
|
||||
"mismatches": [],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Telegram Notify App состоит из простого пользовательского интерфейса для отправки уведомлений, механизма обработки сообщений и интеграции с API Telegram, позволяющей отправлять сообщения в выбранные чаты или каналы.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
# v2-general-08-components
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: True
|
||||
|
||||
## Query
|
||||
Из чего состоит Telegram Notify App?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Telegram Notify App состоит из простого пользовательского интерфейса для отправки уведомлений, механизма обработки сообщений и интеграции с API Telegram, позволяющей отправлять сообщения в выбранные чаты или каналы.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- none
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-general-09-reading-order",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"domain: expected GENERAL, got DOCS",
|
||||
"intent: expected GENERAL_QA, got DOC_EXPLAIN",
|
||||
"llm: expected answer to contain 'README'",
|
||||
"llm: expected answer to contain 'архитект'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "Не нашёл подходящих SUMMARY-блоков в документации по этому запросу.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
# v2-general-09-reading-order
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Какие документы стоит читать сначала?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "Не нашёл подходящих SUMMARY-блоков в документации по этому запросу.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- domain: expected GENERAL, got DOCS
|
||||
- intent: expected GENERAL_QA, got DOC_EXPLAIN
|
||||
- llm: expected answer to contain 'README'
|
||||
- llm: expected answer to contain 'архитект'
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-general-10-short-summary",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": true,
|
||||
"mismatches": [],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Проектная документация включает описание целей, задач, методов, графика и бюджета проекта, а также список необходимых ресурсов и команду исполнителей.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": []
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
# v2-general-10-short-summary
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: True
|
||||
|
||||
## Query
|
||||
Дай короткое summary по проектной документации
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"rag_count": 0,
|
||||
"llm_answer": "Проектная документация включает описание целей, задач, методов, графика и бюджета проекта, а также список необходимых ресурсов и команду исполнителей.",
|
||||
"answer_mode": "answered",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [],
|
||||
"layers": [],
|
||||
"filters": {}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
- none
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- none
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
# pipeline_setup_v3 summary
|
||||
|
||||
Passed: 7/30
|
||||
|
||||
| File | Case | Mode | Query | Actual sub-intent | RAG layers | Tokens | Pass |
|
||||
|------|------|------|-------|-------------------|------------|--------|------|
|
||||
| full_chain_docs_v2_matrix.yaml | v2-general-01-what-is-service | full_chain | Что это за сервис? | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-general-02-purpose | full_chain | Для чего нужен test_echo_app? | SUMMARY | — | — | ✓ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-general-03-problem | full_chain | Какую задачу решает это приложение? | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-general-04-docs-contents | full_chain | Что входит в документацию этого проекта? | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-general-05-main-parts | full_chain | Какие основные части есть у системы? | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-general-06-high-level-architecture | full_chain | Как в целом устроено приложение? | SUMMARY | — | — | ✓ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-general-07-besides-api | full_chain | Что тут есть кроме API? | SUMMARY | — | — | ✓ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-general-08-components | full_chain | Из чего состоит Telegram Notify App? | SUMMARY | — | — | ✓ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-general-09-reading-order | full_chain | Какие документы стоит читать сначала? | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-general-10-short-summary | full_chain | Дай короткое summary по проектной документации | SUMMARY | — | — | ✓ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-01-health | full_chain | Кратко объясни по документации, как работает `/health` | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-02-send | full_chain | Что делает endpoint `/send`? | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-03-actions | full_chain | Объясни назначение `/actions/{action}` | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-04-runtime-health | full_chain | Что такое runtime health в этой документации? | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-05-notification-loop | full_chain | Кратко опиши цикл отправки уведомлений | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-06-architecture | full_chain | Как устроена архитектура Telegram Notify App? | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-07-worker | full_chain | Что делает worker в этом приложении? | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-08-health-statuses | full_chain | Какие статусы здоровья есть у runtime? | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-09-manual-send | full_chain | Как в приложении устроена ручная отправка сообщения? | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-docs-summary-10-telegram-integration | full_chain | Как сервис взаимодействует с Telegram в целом? | SUMMARY | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-find-files-01-health | full_chain | В каком файле описан `/health`? | FIND_FILES | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-find-files-02-send | full_chain | Покажи файл с описанием endpoint `/send` | FIND_FILES | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-find-files-03-actions | full_chain | Где находится документация по `/actions/{action}`? | FIND_FILES | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-find-files-04-runtime-health | full_chain | В каком документе описан runtime health? | FIND_FILES | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-find-files-05-loop | full_chain | Найди файл с логикой цикла отправки уведомлений | FIND_FILES | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-find-files-06-architecture | full_chain | Где описана архитектура приложения? | FIND_FILES | — | — | ✓ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-find-files-07-errors-catalog | full_chain | В каком файле лежит каталог ошибок? | FIND_FILES | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-find-files-08-docs-index | full_chain | Покажи файл-индекс документации проекта | FIND_FILES | — | — | ✓ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-find-files-09-telegram-bot-api | full_chain | Какие файлы относятся к Telegram Bot API интеграции? | FIND_FILES | — | — | ✗ |
|
||||
| full_chain_docs_v2_matrix.yaml | v2-find-files-10-telegram-notify-worker | full_chain | В каких файлах описан worker `telegram_notify`? | FIND_FILES | — | — | ✗ |
|
||||
|
||||
## Failures
|
||||
- **v2-general-01-what-is-service**: llm: expected answer to contain 'Telegram'
|
||||
- **v2-general-03-problem**: llm: expected answer to contain 'Telegram'
|
||||
- **v2-general-04-docs-contents**: domain: expected GENERAL, got DOCS; intent: expected GENERAL_QA, got DOC_EXPLAIN
|
||||
- **v2-general-05-main-parts**: llm: expected answer to contain 'Telegram'
|
||||
- **v2-general-09-reading-order**: domain: expected GENERAL, got DOCS; intent: expected GENERAL_QA, got DOC_EXPLAIN; llm: expected answer to contain 'README'; llm: expected answer to contain 'архитект'
|
||||
- **v2-docs-summary-01-health**: llm: expected answer to contain 'status'; llm: expected answer to contain 'components'
|
||||
- **v2-docs-summary-02-send**: doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']; llm: expected answer to contain 'message'
|
||||
- **v2-docs-summary-03-actions**: domain: expected DOCS, got GENERAL; intent: expected DOC_EXPLAIN, got GENERAL_QA; retrieval: expected non-empty rag, got 0 rows; doc_scope should contain 'docs/api/control-actions-endpoint.md', got []
|
||||
- **v2-docs-summary-04-runtime-health**: domain: expected DOCS, got GENERAL; intent: expected DOC_EXPLAIN, got GENERAL_QA; retrieval: expected non-empty rag, got 0 rows; doc_scope should contain 'docs/domains/runtime-health-entity.md', got []
|
||||
- **v2-docs-summary-05-notification-loop**: domain: expected DOCS, got GENERAL; intent: expected DOC_EXPLAIN, got GENERAL_QA; retrieval: expected non-empty rag, got 0 rows; doc_scope should contain 'docs/logic/telegram-notification-loop.md', got []; llm: expected answer to contain 'worker'; llm: expected answer to contain 'interval'; llm: expected answer to contain 'degraded'
|
||||
- **v2-docs-summary-06-architecture**: domain: expected DOCS, got GENERAL; intent: expected DOC_EXPLAIN, got GENERAL_QA; retrieval: expected non-empty rag, got 0 rows; doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []; llm: expected answer to contain 'RuntimeManager'; llm: expected answer to contain 'TelegramControlChannel'
|
||||
- **v2-docs-summary-07-worker**: domain: expected DOCS, got GENERAL; intent: expected DOC_EXPLAIN, got GENERAL_QA; retrieval: expected non-empty rag, got 0 rows; doc_scope should contain 'docs/logic/telegram-notification-loop.md', got []; doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []; llm: expected answer to contain 'Telegram'
|
||||
- **v2-docs-summary-08-health-statuses**: doc_scope should contain 'docs/domains/runtime-health-entity.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']; doc_scope should contain 'docs/api/health-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']; llm: expected answer to contain 'ok'; llm: expected answer to contain 'degraded'; llm: expected answer to contain 'unhealthy'; llm: expected answer to contain 'unknown'
|
||||
- **v2-docs-summary-09-manual-send**: domain: expected DOCS, got GENERAL; intent: expected DOC_EXPLAIN, got GENERAL_QA; retrieval: expected non-empty rag, got 0 rows; doc_scope should contain 'docs/api/send-message-endpoint.md', got []; llm: expected answer to contain '/send'; llm: expected answer to contain 'message'; llm: expected answer to contain 'TelegramSendService'
|
||||
- **v2-docs-summary-10-telegram-integration**: domain: expected DOCS, got GENERAL; intent: expected DOC_EXPLAIN, got GENERAL_QA; retrieval: expected non-empty rag, got 0 rows; retrieval: expected at least 2 rows, got 0; doc_scope should contain 'docs/architecture/telegram-notify-app-overview.md', got []; doc_scope should contain 'docs/api/send-message-endpoint.md', got []; llm: expected answer to contain 'worker'; llm: expected answer to contain '/send'
|
||||
- **v2-find-files-01-health**: llm: expected answer to contain 'docs/api/health-endpoint.md'
|
||||
- **v2-find-files-02-send**: llm: expected answer to contain 'docs/api/send-message-endpoint.md'
|
||||
- **v2-find-files-03-actions**: llm: expected answer to contain 'docs/api/control-actions-endpoint.md'
|
||||
- **v2-find-files-04-runtime-health**: llm: expected answer to contain 'docs/domains/runtime-health-entity.md'
|
||||
- **v2-find-files-05-loop**: llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'
|
||||
- **v2-find-files-07-errors-catalog**: llm: expected answer to contain 'docs/errors/catalog.yaml'
|
||||
- **v2-find-files-09-telegram-bot-api**: llm: expected answer to contain 'docs/api/send-message-endpoint.md'; llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'
|
||||
- **v2-find-files-10-telegram-notify-worker**: llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'; llm: expected answer to contain 'docs/domains/runtime-health-entity.md'
|
||||
|
||||
## LLM Answers
|
||||
- **v2-general-01-what-is-service**
|
||||
Query: Что это за сервис?
|
||||
> Это зависит от контекста. Уточни, пожалуйста, о каком именно сервисе идет речь.
|
||||
- **v2-general-02-purpose**
|
||||
Query: Для чего нужен test_echo_app?
|
||||
> Test\_echo\_app нужен для тестирования и отладки приложений и систем, повторяя входящий запрос в качестве ответа, чтобы убедиться в правильной работе связи и функциональности.
|
||||
- **v2-general-03-problem**
|
||||
Query: Какую задачу решает это приложение?
|
||||
> Приложение решает задачу \(указать конкретную задачу, например, управления контактами, планирования задач, учета финансов и т.д.\) для пользователей путем предоставления удобного и интуитивно понятного интерфейса.
|
||||
- **v2-general-04-docs-contents**
|
||||
Query: Что входит в документацию этого проекта?
|
||||
> Документация проекта включает следующие компоненты: \- Архитектура приложения \`test\_echo\_app\` \- Описание HTTP API \(control plane\) \- Цикл отправки уведомлений \- Health\-модель \- Каталог ошибок Структура документации канонически разделена на несколько секций: \`docs/architecture\`, \`docs/api\`, \`docs/logic\`, \`docs/domains\`, \`docs/errors\`. Основная родительская документация — "Архитектура Telegram Notify App" \(находится по пути ./architecture/telegram\-notify\-app\-overview.md\). Документы взаимосвязаны с помощью ссылок \`related\_docs\`, \`parent\`/\`children\` и обычных markdown\-ссылок без повторения одной и той же информации. Файлы, использованные для ответа: docs/README.md
|
||||
- **v2-general-05-main-parts**
|
||||
Query: Какие основные части есть у системы?
|
||||
> Основные части любой системы включают вход, процессоры \(обработчики\), память и выход.
|
||||
- **v2-general-06-high-level-architecture**
|
||||
Query: Как в целом устроено приложение?
|
||||
> Приложение обычно состоит из пользовательского интерфейса \(то, что видит и с чем взаимодействует пользователь\), логики приложения \(код, который обрабатывает действия пользователя\) и базы данных \(место хранения информации\). Все эти части связаны между собой для обеспечения удобной работы пользователя.
|
||||
- **v2-general-07-besides-api**
|
||||
Query: Что тут есть кроме API?
|
||||
> Кроме API, здесь есть дружелюбная атмосфера, готовность помочь и интересные беседы на различные темы.
|
||||
- **v2-general-08-components**
|
||||
Query: Из чего состоит Telegram Notify App?
|
||||
> Telegram Notify App состоит из простого пользовательского интерфейса для отправки уведомлений, механизма обработки сообщений и интеграции с API Telegram, позволяющей отправлять сообщения в выбранные чаты или каналы.
|
||||
- **v2-general-09-reading-order**
|
||||
Query: Какие документы стоит читать сначала?
|
||||
> Не нашёл подходящих SUMMARY\-блоков в документации по этому запросу.
|
||||
- **v2-general-10-short-summary**
|
||||
Query: Дай короткое summary по проектной документации
|
||||
> Проектная документация включает описание целей, задач, методов, графика и бюджета проекта, а также список необходимых ресурсов и команду исполнителей.
|
||||
- **v2-docs-summary-01-health**
|
||||
Query: Кратко объясни по документации, как работает `/health`
|
||||
> Информация о том, как именно работает \`/health\`, не представлена в предоставленных SUMMARY\-блоках. Файлы, на которые я опирался: \- docs/architecture/telegram\-notify\-app\-overview.md
|
||||
- **v2-docs-summary-02-send**
|
||||
Query: Что делает endpoint `/send`?
|
||||
> На основе представленного SUMMARY\-блока, документация о том, что именно делает endpoint \`/send\`, отсутствует. В контексте указано, что он связан с отправкой сообщений, но подробностей его работы не приведено. Файлы, на которые опирался ответ: \- docs/architecture/telegram\-notify\-app\-overview.md
|
||||
- **v2-docs-summary-03-actions**
|
||||
Query: Объясни назначение `/actions/{action}`
|
||||
> \`/actions/\{action\}\` обычно используется в API для выполнения конкретных действий или операций, где \`\{action\}\` заменяется названием нужного действия. Это позволяет организовывать и вызывать различные функции через единый интерфейс.
|
||||
- **v2-docs-summary-04-runtime-health**
|
||||
Query: Что такое runtime health в этой документации?
|
||||
> Runtime health — это процесс мониторинга и обеспечения стабильности работы системы или приложения во время их выполнения \(runtime\), позволяющий своевременно обнаруживать и устранять возникающие проблемы.
|
||||
- **v2-docs-summary-05-notification-loop**
|
||||
Query: Кратко опиши цикл отправки уведомлений
|
||||
> Цикл отправки уведомлений включает следующие шаги: отлавливание события, обработка данных, выбор получателей, формирование сообщения и его отправка выбранным пользователям.
|
||||
- **v2-docs-summary-06-architecture**
|
||||
Query: Как устроена архитектура Telegram Notify App?
|
||||
> Архитектура Telegram Notify App основана на клиент\-серверной модели, где серверное приложение обрабатывает входящие данные и отправляет уведомления через Telegram API. Клиентское приложение взаимодействует с сервером для отправки уведомлений. Пользователь настраивает параметры уведомлений в приложении, а сервер передает их в виде сообщений в указанный Telegram\-канал или пользователю.
|
||||
- **v2-docs-summary-07-worker**
|
||||
Query: Что делает worker в этом приложении?
|
||||
> Worker в этом приложении выполняет фоновые задачи или процессы, позволяя основному приложению работать быстро и плавно. Это может включать обработку данных, отправку уведомлений, загрузку файлов и другие ресурсоемкие операции.
|
||||
- **v2-docs-summary-08-health-statuses**
|
||||
Query: Какие статусы здоровья есть у runtime?
|
||||
> На основе представленного SUMMARY\-блока конкретной информации о статусах здоровья runtime нет. Файл, связанный с доменом "runtime health" \(../domains/runtime\-health\-entity.md\), вероятно, содержит эту информацию, но она не включена в текущий контекст. Файлы, на которые опирался ответ: \- docs/architecture/telegram\-notify\-app\-overview.md
|
||||
- **v2-docs-summary-09-manual-send**
|
||||
Query: Как в приложении устроена ручная отправка сообщения?
|
||||
> Ручная отправка сообщения в приложении обычно осуществляется нажатием кнопки "Отправить" или аналогичного элемента интерфейса после того, как пользователь введет текст сообщения.
|
||||
- **v2-docs-summary-10-telegram-integration**
|
||||
Query: Как сервис взаимодействует с Telegram в целом?
|
||||
> Сервис может отправлять и получать сообщения, реагировать на команды и запросы пользователей через Telegram\-бота. Взаимодействие происходит посредством отправки запросов и получения ответов от бота в чате.
|
||||
- **v2-find-files-01-health**
|
||||
Query: В каком файле описан `/health`?
|
||||
> Нашёл релевантные файлы документации: \- \`docs/README.md\` \- \`docs/architecture/telegram\-notify\-app\-overview.md\`
|
||||
- **v2-find-files-02-send**
|
||||
Query: Покажи файл с описанием endpoint `/send`
|
||||
> Нашёл релевантные файлы документации: \- \`docs/README.md\` \- \`docs/architecture/telegram\-notify\-app\-overview.md\`
|
||||
- **v2-find-files-03-actions**
|
||||
Query: Где находится документация по `/actions/{action}`?
|
||||
> Нашёл релевантные файлы документации: \- \`docs/README.md\` \- \`docs/architecture/telegram\-notify\-app\-overview.md\`
|
||||
- **v2-find-files-04-runtime-health**
|
||||
Query: В каком документе описан runtime health?
|
||||
> Нашёл релевантные файлы документации: \- \`docs/architecture/telegram\-notify\-app\-overview.md\` \- \`docs/README.md\`
|
||||
- **v2-find-files-05-loop**
|
||||
Query: Найди файл с логикой цикла отправки уведомлений
|
||||
> Нашёл релевантные файлы документации: \- \`docs/README.md\` \- \`docs/architecture/telegram\-notify\-app\-overview.md\`
|
||||
- **v2-find-files-06-architecture**
|
||||
Query: Где описана архитектура приложения?
|
||||
> Нашёл релевантные файлы документации: \- \`docs/README.md\` \- \`docs/architecture/telegram\-notify\-app\-overview.md\`
|
||||
- **v2-find-files-07-errors-catalog**
|
||||
Query: В каком файле лежит каталог ошибок?
|
||||
> Нашёл релевантные файлы документации: \- \`docs/README.md\`
|
||||
- **v2-find-files-08-docs-index**
|
||||
Query: Покажи файл-индекс документации проекта
|
||||
> Нашёл релевантные файлы документации: \- \`docs/README.md\`
|
||||
- **v2-find-files-09-telegram-bot-api**
|
||||
Query: Какие файлы относятся к Telegram Bot API интеграции?
|
||||
> Нашёл релевантные файлы документации: \- \`docs/README.md\` \- \`docs/architecture/telegram\-notify\-app\-overview.md\`
|
||||
- **v2-find-files-10-telegram-notify-worker**
|
||||
Query: В каких файлах описан worker `telegram_notify`?
|
||||
> Нашёл релевантные файлы документации: \- \`docs/architecture/telegram\-notify\-app\-overview.md\`
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-01-health",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"answer_mode: expected grounded_summary, got insufficient_evidence",
|
||||
"llm: expected answer to contain '/health'",
|
||||
"llm: expected answer to contain 'status'",
|
||||
"llm: expected answer to contain 'components'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%health-endpoint.md%",
|
||||
"%/health%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/health-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"endpoint_paths": [
|
||||
"/health"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/health-endpoint.md"
|
||||
],
|
||||
"matched_aliases": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%health-endpoint.md%",
|
||||
"%/health%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/health-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+186
@@ -0,0 +1,186 @@
|
||||
# v2-docs-summary-01-health
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Кратко объясни по документации, как работает `/health`
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%health-endpoint.md%",
|
||||
"%/health%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/health-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"endpoint_paths": [
|
||||
"/health"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/health-endpoint.md"
|
||||
],
|
||||
"matched_aliases": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%health-endpoint.md%",
|
||||
"%/health%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/health-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- answer_mode: expected grounded_summary, got insufficient_evidence
|
||||
- llm: expected answer to contain '/health'
|
||||
- llm: expected answer to contain 'status'
|
||||
- llm: expected answer to contain 'components'
|
||||
+152
@@ -0,0 +1,152 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-02-send",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']",
|
||||
"llm: expected answer to contain '/send'",
|
||||
"llm: expected answer to contain 'message'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%send-message-endpoint.md%",
|
||||
"%/send%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"endpoint_paths": [
|
||||
"/send"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"matched_aliases": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%send-message-endpoint.md%",
|
||||
"%/send%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+185
@@ -0,0 +1,185 @@
|
||||
# v2-docs-summary-02-send
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Что делает endpoint `/send`?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%send-message-endpoint.md%",
|
||||
"%/send%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"endpoint_paths": [
|
||||
"/send"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"matched_aliases": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%send-message-endpoint.md%",
|
||||
"%/send%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
|
||||
- llm: expected answer to contain '/send'
|
||||
- llm: expected answer to contain 'message'
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-03-actions",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"doc_scope should contain 'docs/api/control-actions-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']",
|
||||
"llm: expected answer to contain '/actions/{action}'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramControlChannel",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%control-actions-endpoint.md%",
|
||||
"%/actions/{action}%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/control-actions-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"endpoint_paths": [
|
||||
"/actions/{action}"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/control-actions-endpoint.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"/actions/{action}"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%control-actions-endpoint.md%",
|
||||
"%/actions/{action}%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/control-actions-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+186
@@ -0,0 +1,186 @@
|
||||
# v2-docs-summary-03-actions
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Объясни назначение `/actions/{action}`
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramControlChannel",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%control-actions-endpoint.md%",
|
||||
"%/actions/{action}%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/control-actions-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"endpoint_paths": [
|
||||
"/actions/{action}"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/control-actions-endpoint.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"/actions/{action}"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%control-actions-endpoint.md%",
|
||||
"%/actions/{action}%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/control-actions-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- doc_scope should contain 'docs/api/control-actions-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
|
||||
- llm: expected answer to contain '/actions/{action}'
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-04-runtime-health",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"doc_scope should contain 'docs/domains/runtime-health-entity.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md']",
|
||||
"llm: expected answer to contain 'runtime health'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/domains/",
|
||||
"docs/api/",
|
||||
"docs/architecture/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%runtime-health-entity.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"runtime-health-entity"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_summary_domain_entity",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/domains/",
|
||||
"docs/api/",
|
||||
"docs/architecture/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%runtime-health-entity.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_summary_domain_entity",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
# v2-docs-summary-04-runtime-health
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Что такое runtime health в этой документации?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/domains/",
|
||||
"docs/api/",
|
||||
"docs/architecture/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%runtime-health-entity.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"runtime-health-entity"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_domain_entity",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/domains/",
|
||||
"docs/api/",
|
||||
"docs/architecture/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%runtime-health-entity.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_domain_entity",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- doc_scope should contain 'docs/domains/runtime-health-entity.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md']
|
||||
- llm: expected answer to contain 'runtime health'
|
||||
+143
@@ -0,0 +1,143 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-05-notification-loop",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"doc_scope should contain 'docs/logic/telegram-notification-loop.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']",
|
||||
"llm: expected answer to contain 'worker'",
|
||||
"llm: expected answer to contain 'interval'",
|
||||
"llm: expected answer to contain 'degraded'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "LOGIC_FLOW",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md, docs/README.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/logic/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%telegram-notification-loop.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/logic/telegram-notification-loop.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "LOGIC_FLOW",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": "LOGIC_FLOW",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [
|
||||
"docs/logic/telegram-notification-loop.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"telegram-notification-loop"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_summary_logic_flow",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/logic/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%telegram-notification-loop.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/logic/telegram-notification-loop.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_summary_logic_flow",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 147
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+176
@@ -0,0 +1,176 @@
|
||||
# v2-docs-summary-05-notification-loop
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Кратко опиши цикл отправки уведомлений
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "LOGIC_FLOW",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md, docs/README.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/logic/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%telegram-notification-loop.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/logic/telegram-notification-loop.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "LOGIC_FLOW",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": "LOGIC_FLOW",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [
|
||||
"docs/logic/telegram-notification-loop.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"telegram-notification-loop"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_logic_flow",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/logic/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%telegram-notification-loop.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/logic/telegram-notification-loop.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_logic_flow",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 2
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 147
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- doc_scope should contain 'docs/logic/telegram-notification-loop.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
|
||||
- llm: expected answer to contain 'worker'
|
||||
- llm: expected answer to contain 'interval'
|
||||
- llm: expected answer to contain 'degraded'
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-06-architecture",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": true,
|
||||
"mismatches": [],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "ARCHITECTURE",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "- Purpose: сервис поднимает HTTP control plane и фоновый worker для отправки уведомлений в Telegram.\n- Entry point: `src/telegram_notify_app/main.py`.\n- Main components: `RuntimeManager`, `TelegramControlChannel`, `TelegramNotifyModule`, `TelegramNotifyWorker`, `TelegramSendService`.\n- Configuration: `config/config.yaml` или путь из `CONFIG_PATH`.\n- Related API: [`/health`](../api/health-endpoint.md), [`/actions/{action}`](../api/control-actions-endpoint.md), [`/send`](../api/send-message-endpoint.md).\n- Related logic: [цикл отправки уведомлений](../logic/telegram-notification-loop.md).\n- Related domain: [runtime health](../domains/runtime-health-entity.md).\n\nФайлы-источники:\n- docs/architecture/telegram-notify-app-overview.md\n- docs/README.md",
|
||||
"answer_mode": "grounded_summary",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "ARCHITECTURE",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": "ARCHITECTURE",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [],
|
||||
"matched_aliases": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_summary_architecture",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_summary_architecture",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": true,
|
||||
"reason": "target_doc_found",
|
||||
"answer_mode": "grounded_summary"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "grounded_summary",
|
||||
"answer_length": 753
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+159
@@ -0,0 +1,159 @@
|
||||
# v2-docs-summary-06-architecture
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: True
|
||||
|
||||
## Query
|
||||
Как устроена архитектура Telegram Notify App?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "ARCHITECTURE",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "- Purpose: сервис поднимает HTTP control plane и фоновый worker для отправки уведомлений в Telegram.\n- Entry point: `src/telegram_notify_app/main.py`.\n- Main components: `RuntimeManager`, `TelegramControlChannel`, `TelegramNotifyModule`, `TelegramNotifyWorker`, `TelegramSendService`.\n- Configuration: `config/config.yaml` или путь из `CONFIG_PATH`.\n- Related API: [`/health`](../api/health-endpoint.md), [`/actions/{action}`](../api/control-actions-endpoint.md), [`/send`](../api/send-message-endpoint.md).\n- Related logic: [цикл отправки уведомлений](../logic/telegram-notification-loop.md).\n- Related domain: [runtime health](../domains/runtime-health-entity.md).\n\nФайлы-источники:\n- docs/architecture/telegram-notify-app-overview.md\n- docs/README.md",
|
||||
"answer_mode": "grounded_summary",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "ARCHITECTURE",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": "ARCHITECTURE",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [],
|
||||
"matched_aliases": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_architecture",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_architecture",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 2
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": true,
|
||||
"reason": "target_doc_found",
|
||||
"answer_mode": "grounded_summary"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "grounded_summary",
|
||||
"answer_length": 753
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- none
|
||||
+129
@@ -0,0 +1,129 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-07-worker",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"doc_scope should contain 'docs/logic/telegram-notification-loop.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']",
|
||||
"llm: expected answer to contain 'worker'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "LOGIC_FLOW",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/logic/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "LOGIC_FLOW",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": "LOGIC_FLOW",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [],
|
||||
"matched_aliases": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_summary_logic_flow",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/logic/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_summary_logic_flow",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
# v2-docs-summary-07-worker
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Что делает worker в этом приложении?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "LOGIC_FLOW",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/logic/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "LOGIC_FLOW",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": "LOGIC_FLOW",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [],
|
||||
"matched_aliases": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_logic_flow",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/logic/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_logic_flow",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- doc_scope should contain 'docs/logic/telegram-notification-loop.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
|
||||
- llm: expected answer to contain 'worker'
|
||||
+143
@@ -0,0 +1,143 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-08-health-statuses",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"doc_scope should contain 'docs/domains/runtime-health-entity.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md']",
|
||||
"doc_scope should contain 'docs/api/health-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md']",
|
||||
"llm: expected answer to contain 'ok'",
|
||||
"llm: expected answer to contain 'degraded'",
|
||||
"llm: expected answer to contain 'unhealthy'",
|
||||
"llm: expected answer to contain 'unknown'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/domains/",
|
||||
"docs/api/",
|
||||
"docs/architecture/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%runtime-health-entity.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"runtime-health-entity"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_summary_domain_entity",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/domains/",
|
||||
"docs/api/",
|
||||
"docs/architecture/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%runtime-health-entity.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_summary_domain_entity",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+176
@@ -0,0 +1,176 @@
|
||||
# v2-docs-summary-08-health-statuses
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Какие статусы здоровья есть у runtime?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/domains/",
|
||||
"docs/api/",
|
||||
"docs/architecture/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%runtime-health-entity.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"runtime-health-entity"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_domain_entity",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/domains/",
|
||||
"docs/api/",
|
||||
"docs/architecture/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%runtime-health-entity.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_domain_entity",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- doc_scope should contain 'docs/domains/runtime-health-entity.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md']
|
||||
- doc_scope should contain 'docs/api/health-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md']
|
||||
- llm: expected answer to contain 'ok'
|
||||
- llm: expected answer to contain 'degraded'
|
||||
- llm: expected answer to contain 'unhealthy'
|
||||
- llm: expected answer to contain 'unknown'
|
||||
+151
@@ -0,0 +1,151 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-09-manual-send",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']",
|
||||
"llm: expected answer to contain '/send'",
|
||||
"llm: expected answer to contain 'message'",
|
||||
"llm: expected answer to contain 'TelegramSendService'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%send-message-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"/send"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%send-message-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+184
@@ -0,0 +1,184 @@
|
||||
# v2-docs-summary-09-manual-send
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Как в приложении устроена ручная отправка сообщения?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%send-message-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": "API_ENDPOINT",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"/send"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%send-message-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"path_prefixes": [
|
||||
"docs/api/",
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_api_endpoint",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
|
||||
- llm: expected answer to contain '/send'
|
||||
- llm: expected answer to contain 'message'
|
||||
- llm: expected answer to contain 'TelegramSendService'
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-docs-summary-10-telegram-integration",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"domain: expected DOCS, got GENERAL",
|
||||
"intent: expected DOC_EXPLAIN, got GENERAL_QA",
|
||||
"anchor_type: expected API_ENDPOINT, got None",
|
||||
"doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": null,
|
||||
"rag_count": 8,
|
||||
"llm_answer": "- Purpose: точка входа в техническую документацию сервиса `test_echo_app`.\n- Scope: архитектура, HTTP API control plane, цикл отправки уведомлений, health-модель и каталог ошибок.\n- Canonical structure: `docs/architecture`, `docs/api`, `docs/logic`, `docs/domains`, `docs/errors`.\n- Primary parent doc: [Архитектура Telegram Notify App](./architecture/telegram-notify-app-overview.md).\n- Navigation: документы связаны через `related_docs`, `parent`/`children` и markdown-ссылки без дублирования деталей.\n- Purpose: сервис поднимает HTTP control plane и фоновый worker для отправки уведомлений в Telegram.\n- Entry point: `src/telegram_notify_app/main.py`.\n- Main components: `RuntimeManager`, `TelegramControlChannel`, `TelegramNotifyModule`, `TelegramNotifyWorker`, `TelegramSendService`.\n- Configuration: `config/config.yaml` или путь из `CONFIG_PATH`.\n- Related API: [`/health`](../api/health-endpoint.md), [`/actions/{action}`](../api/control-actions-endpoint.md), [`/send`](../api/send-message-endpoint.md).\n- Related logic: [цикл отправки уведомлений](../logic/telegram-notification-loop.md).\n- Related domain: [runtime health](../domains/runtime-health-entity.md).",
|
||||
"answer_mode": "grounded_summary",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%README.md%",
|
||||
"%overview%"
|
||||
],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": null,
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": null,
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [],
|
||||
"matched_aliases": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "general_qa_grounded_summary",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%README.md%",
|
||||
"%overview%"
|
||||
],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "general_qa_grounded_summary",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/README.md",
|
||||
"document_count": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": true,
|
||||
"reason": "general_docs_found",
|
||||
"answer_mode": "grounded_summary"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "grounded_summary",
|
||||
"answer_length": 1170
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
# v2-docs-summary-10-telegram-integration
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Как сервис взаимодействует с Telegram в целом?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": null,
|
||||
"rag_count": 8,
|
||||
"llm_answer": "- Purpose: точка входа в техническую документацию сервиса `test_echo_app`.\n- Scope: архитектура, HTTP API control plane, цикл отправки уведомлений, health-модель и каталог ошибок.\n- Canonical structure: `docs/architecture`, `docs/api`, `docs/logic`, `docs/domains`, `docs/errors`.\n- Primary parent doc: [Архитектура Telegram Notify App](./architecture/telegram-notify-app-overview.md).\n- Navigation: документы связаны через `related_docs`, `parent`/`children` и markdown-ссылки без дублирования деталей.\n- Purpose: сервис поднимает HTTP control plane и фоновый worker для отправки уведомлений в Telegram.\n- Entry point: `src/telegram_notify_app/main.py`.\n- Main components: `RuntimeManager`, `TelegramControlChannel`, `TelegramNotifyModule`, `TelegramNotifyWorker`, `TelegramSendService`.\n- Configuration: `config/config.yaml` или путь из `CONFIG_PATH`.\n- Related API: [`/health`](../api/health-endpoint.md), [`/actions/{action}`](../api/control-actions-endpoint.md), [`/send`](../api/send-message-endpoint.md).\n- Related logic: [цикл отправки уведомлений](../logic/telegram-notification-loop.md).\n- Related domain: [runtime health](../domains/runtime-health-entity.md).",
|
||||
"answer_mode": "grounded_summary",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%README.md%",
|
||||
"%overview%"
|
||||
],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "GENERAL",
|
||||
"intent": "GENERAL_QA",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": null,
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": null,
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [],
|
||||
"matched_aliases": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "general_qa_grounded_summary",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/architecture/",
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%README.md%",
|
||||
"%overview%"
|
||||
],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "general_qa_grounded_summary",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/README.md",
|
||||
"document_count": 2
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": true,
|
||||
"reason": "general_docs_found",
|
||||
"answer_mode": "grounded_summary"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "grounded_summary",
|
||||
"answer_length": 1170
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- domain: expected DOCS, got GENERAL
|
||||
- intent: expected DOC_EXPLAIN, got GENERAL_QA
|
||||
- anchor_type: expected API_ENDPOINT, got None
|
||||
- doc_scope should contain 'docs/api/send-message-endpoint.md', got ['architecture.telegram_notify_app', 'docs/architecture/telegram-notify-app-overview.md', 'index.test_echo_app_docs', 'docs/README.md']
|
||||
+137
@@ -0,0 +1,137 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-01-health",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"answer_mode: expected deterministic, got deterministic_file_list",
|
||||
"llm: expected answer to contain 'docs/api/health-endpoint.md'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "docs/architecture/telegram-notify-app-overview.md\ndocs/README.md",
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"RuntimeManager",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%health-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/health-endpoint.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": null,
|
||||
"endpoint_paths": [
|
||||
"/health"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/health-endpoint.md"
|
||||
],
|
||||
"matched_aliases": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%health-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/health-endpoint.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"row_count": 12,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "find_files",
|
||||
"primary_file": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"file_count": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": true,
|
||||
"reason": "primary_file_confident",
|
||||
"answer_mode": "deterministic_file_list"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"answer_length": 64
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+170
@@ -0,0 +1,170 @@
|
||||
# v2-find-files-01-health
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
В каком файле описан `/health`?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "docs/architecture/telegram-notify-app-overview.md\ndocs/README.md",
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"RuntimeManager",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%health-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/health-endpoint.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": null,
|
||||
"endpoint_paths": [
|
||||
"/health"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/health-endpoint.md"
|
||||
],
|
||||
"matched_aliases": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%health-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/health-endpoint.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"row_count": 12,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "find_files",
|
||||
"primary_file": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"file_count": 2
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": true,
|
||||
"reason": "primary_file_confident",
|
||||
"answer_mode": "deterministic_file_list"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"answer_length": 64
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- answer_mode: expected deterministic, got deterministic_file_list
|
||||
- llm: expected answer to contain 'docs/api/health-endpoint.md'
|
||||
+137
@@ -0,0 +1,137 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-02-send",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"answer_mode: expected deterministic, got deterministic_file_list",
|
||||
"llm: expected answer to contain 'docs/api/send-message-endpoint.md'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "docs/architecture/telegram-notify-app-overview.md\ndocs/README.md",
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%send-message-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": null,
|
||||
"endpoint_paths": [
|
||||
"/send"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"matched_aliases": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%send-message-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"row_count": 12,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "find_files",
|
||||
"primary_file": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"file_count": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": true,
|
||||
"reason": "primary_file_confident",
|
||||
"answer_mode": "deterministic_file_list"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"answer_length": 64
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+170
@@ -0,0 +1,170 @@
|
||||
# v2-find-files-02-send
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Покажи файл с описанием endpoint `/send`
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "docs/architecture/telegram-notify-app-overview.md\ndocs/README.md",
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%send-message-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": null,
|
||||
"endpoint_paths": [
|
||||
"/send"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
],
|
||||
"matched_aliases": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%send-message-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/send-message-endpoint.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"row_count": 12,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "find_files",
|
||||
"primary_file": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"file_count": 2
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": true,
|
||||
"reason": "primary_file_confident",
|
||||
"answer_mode": "deterministic_file_list"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"answer_length": 64
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- answer_mode: expected deterministic, got deterministic_file_list
|
||||
- llm: expected answer to contain 'docs/api/send-message-endpoint.md'
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-03-actions",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"answer_mode: expected deterministic, got deterministic_file_list",
|
||||
"llm: expected answer to contain 'docs/api/control-actions-endpoint.md'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "docs/architecture/telegram-notify-app-overview.md\ndocs/README.md",
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%control-actions-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/control-actions-endpoint.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": null,
|
||||
"endpoint_paths": [
|
||||
"/actions/{action}"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/control-actions-endpoint.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"/actions/{action}"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%control-actions-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/control-actions-endpoint.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"row_count": 12,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "find_files",
|
||||
"primary_file": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"file_count": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": true,
|
||||
"reason": "primary_file_confident",
|
||||
"answer_mode": "deterministic_file_list"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"answer_length": 64
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
# v2-find-files-03-actions
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Где находится документация по `/actions/{action}`?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "docs/architecture/telegram-notify-app-overview.md\ndocs/README.md",
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%control-actions-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/control-actions-endpoint.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": null,
|
||||
"endpoint_paths": [
|
||||
"/actions/{action}"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/control-actions-endpoint.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"/actions/{action}"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/api/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%control-actions-endpoint.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/api/control-actions-endpoint.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"row_count": 12,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "find_files",
|
||||
"primary_file": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"file_count": 2
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": true,
|
||||
"reason": "primary_file_confident",
|
||||
"answer_mode": "deterministic_file_list"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"answer_length": 64
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- answer_mode: expected deterministic, got deterministic_file_list
|
||||
- llm: expected answer to contain 'docs/api/control-actions-endpoint.md'
|
||||
+141
@@ -0,0 +1,141 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-04-runtime-health",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"sub_intent: expected FIND_FILES, got SUMMARY",
|
||||
"anchor_type: expected FIND_FILES, got DOMAIN_ENTITY",
|
||||
"answer_mode: expected deterministic, got insufficient_evidence",
|
||||
"llm: expected answer to contain 'docs/domains/runtime-health-entity.md'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/domains/",
|
||||
"docs/api/",
|
||||
"docs/architecture/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%runtime-health-entity.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"runtime-health-entity"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_summary_domain_entity",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/domains/",
|
||||
"docs/api/",
|
||||
"docs/architecture/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%runtime-health-entity.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_summary_domain_entity",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+174
@@ -0,0 +1,174 @@
|
||||
# v2-find-files-04-runtime-health
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
В каком документе описан runtime health?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "SUMMARY",
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"rag_count": 8,
|
||||
"llm_answer": "В поднятом контексте не найден целевой документ по запросу. Ближайшие документы: docs/architecture/telegram-notify-app-overview.md.",
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/domains/",
|
||||
"docs/api/",
|
||||
"docs/architecture/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%runtime-health-entity.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "SUMMARY",
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": "DOMAIN_ENTITY",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"runtime-health-entity"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_domain_entity",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/domains/",
|
||||
"docs/api/",
|
||||
"docs/architecture/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%runtime-health-entity.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/domains/runtime-health-entity.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_summary_domain_entity",
|
||||
"row_count": 8,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "summary",
|
||||
"primary_doc": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"document_count": 1
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "target_doc_missing",
|
||||
"answer_mode": "insufficient_evidence"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "insufficient_evidence",
|
||||
"answer_length": 131
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- sub_intent: expected FIND_FILES, got SUMMARY
|
||||
- anchor_type: expected FIND_FILES, got DOMAIN_ENTITY
|
||||
- answer_mode: expected deterministic, got insufficient_evidence
|
||||
- llm: expected answer to contain 'docs/domains/runtime-health-entity.md'
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-05-loop",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"answer_mode: expected deterministic, got deterministic_file_list",
|
||||
"llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "docs/README.md\ndocs/architecture/telegram-notify-app-overview.md",
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService",
|
||||
"RuntimeManager",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": null,
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [],
|
||||
"matched_aliases": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"row_count": 12,
|
||||
"top_files": [
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "find_files",
|
||||
"primary_file": "docs/README.md",
|
||||
"file_count": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "low_confidence_shortlist",
|
||||
"answer_mode": "deterministic_file_list"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"answer_length": 64
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
# v2-find-files-05-loop
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Найди файл с логикой цикла отправки уведомлений
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "docs/README.md\ndocs/architecture/telegram-notify-app-overview.md",
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramSendService",
|
||||
"RuntimeManager",
|
||||
"TelegramControlChannel"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": null,
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [],
|
||||
"matched_aliases": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"row_count": 12,
|
||||
"top_files": [
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "find_files",
|
||||
"primary_file": "docs/README.md",
|
||||
"file_count": 2
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "low_confidence_shortlist",
|
||||
"answer_mode": "deterministic_file_list"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"answer_length": 64
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- answer_mode: expected deterministic, got deterministic_file_list
|
||||
- llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-06-architecture",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"answer_mode: expected deterministic, got deterministic_file_list"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/architecture/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%telegram-notify-app-overview.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": null,
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"telegram-notify-app-overview"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/architecture/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%telegram-notify-app-overview.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"row_count": 12,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "find_files",
|
||||
"primary_file": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"file_count": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": true,
|
||||
"reason": "primary_file_confident",
|
||||
"answer_mode": "deterministic_file_list"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"answer_length": 49
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+167
@@ -0,0 +1,167 @@
|
||||
# v2-find-files-06-architecture
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Где описана архитектура приложения?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/architecture/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%telegram-notify-app-overview.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": null,
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"matched_aliases": [
|
||||
"telegram-notify-app-overview"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/architecture/"
|
||||
],
|
||||
"prefer_like_patterns": [
|
||||
"%telegram-notify-app-overview.md%"
|
||||
],
|
||||
"target_doc_hints": [
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"row_count": 12,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "find_files",
|
||||
"primary_file": "docs/architecture/telegram-notify-app-overview.md",
|
||||
"file_count": 1
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": true,
|
||||
"reason": "primary_file_confident",
|
||||
"answer_mode": "deterministic_file_list"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"answer_length": 49
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- answer_mode: expected deterministic, got deterministic_file_list
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-07-errors-catalog",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"answer_mode: expected deterministic, got deterministic_file_list",
|
||||
"llm: expected answer to contain 'docs/errors/catalog.yaml'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "docs/README.md\ndocs/architecture/telegram-notify-app-overview.md",
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyWorker"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": "FIND_FILES",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [],
|
||||
"matched_aliases": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"row_count": 12,
|
||||
"top_files": [
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "find_files",
|
||||
"primary_file": "docs/README.md",
|
||||
"file_count": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "low_confidence_shortlist",
|
||||
"answer_mode": "deterministic_file_list"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"answer_length": 64
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
# v2-find-files-07-errors-catalog
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
В каком файле лежит каталог ошибок?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "docs/README.md\ndocs/architecture/telegram-notify-app-overview.md",
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"RuntimeManager",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramNotifyWorker"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": "FIND_FILES",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [],
|
||||
"matched_aliases": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"row_count": 12,
|
||||
"top_files": [
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "find_files",
|
||||
"primary_file": "docs/README.md",
|
||||
"file_count": 2
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "low_confidence_shortlist",
|
||||
"answer_mode": "deterministic_file_list"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"answer_length": 64
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- answer_mode: expected deterministic, got deterministic_file_list
|
||||
- llm: expected answer to contain 'docs/errors/catalog.yaml'
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-08-docs-index",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"answer_mode: expected deterministic, got deterministic_file_list"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "docs/README.md\ndocs/architecture/telegram-notify-app-overview.md",
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": "FIND_FILES",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [],
|
||||
"matched_aliases": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"row_count": 12,
|
||||
"top_files": [
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "find_files",
|
||||
"primary_file": "docs/README.md",
|
||||
"file_count": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "low_confidence_shortlist",
|
||||
"answer_mode": "deterministic_file_list"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"answer_length": 64
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+157
@@ -0,0 +1,157 @@
|
||||
# v2-find-files-08-docs-index
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Покажи файл-индекс документации проекта
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "docs/README.md\ndocs/architecture/telegram-notify-app-overview.md",
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager",
|
||||
"TelegramSendService"
|
||||
],
|
||||
"doc_scope": [
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md",
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": "FIND_FILES",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [],
|
||||
"matched_aliases": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"row_count": 12,
|
||||
"top_files": [
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "find_files",
|
||||
"primary_file": "docs/README.md",
|
||||
"file_count": 2
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "low_confidence_shortlist",
|
||||
"answer_mode": "deterministic_file_list"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"answer_length": 64
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- answer_mode: expected deterministic, got deterministic_file_list
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-09-telegram-bot-api",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"answer_mode: expected deterministic, got deterministic_file_list",
|
||||
"llm: expected answer to contain 'docs/api/send-message-endpoint.md'",
|
||||
"llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "docs/README.md\ndocs/architecture/telegram-notify-app-overview.md",
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": "FIND_FILES",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [],
|
||||
"matched_aliases": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"row_count": 12,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "find_files",
|
||||
"primary_file": "docs/README.md",
|
||||
"file_count": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "low_confidence_shortlist",
|
||||
"answer_mode": "deterministic_file_list"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"answer_length": 64
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
+159
@@ -0,0 +1,159 @@
|
||||
# v2-find-files-09-telegram-bot-api
|
||||
|
||||
- source_file: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml
|
||||
- runner: process_v2
|
||||
- mode: full_chain
|
||||
- passed: False
|
||||
|
||||
## Query
|
||||
Какие файлы относятся к Telegram Bot API интеграции?
|
||||
|
||||
## Actual
|
||||
{
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "docs/README.md\ndocs/architecture/telegram-notify-app-overview.md",
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel",
|
||||
"TelegramNotifyWorker",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
|
||||
## Pipeline Steps
|
||||
### router_resolved
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"confidence": 1.0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### anchors_extracted
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"anchor_type": "FIND_FILES",
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [],
|
||||
"matched_aliases": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_profile_selected
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### retrieval_executed
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"row_count": 12,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_assembled
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"mode": "find_files",
|
||||
"primary_file": "docs/README.md",
|
||||
"file_count": 2
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### evidence_gate_checked
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "low_confidence_shortlist",
|
||||
"answer_mode": "deterministic_file_list"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### answer_generated
|
||||
```json
|
||||
{
|
||||
"input": {},
|
||||
"output": {
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"answer_length": 64
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Diagnostics
|
||||
{}
|
||||
|
||||
|
||||
## Mismatches
|
||||
- answer_mode: expected deterministic, got deterministic_file_list
|
||||
- llm: expected answer to contain 'docs/api/send-message-endpoint.md'
|
||||
- llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
{
|
||||
"meta": {
|
||||
"case_id": "v2-find-files-10-telegram-notify-worker",
|
||||
"source_file": "/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup_v3/cases/suite_04_process_v2/full_chain_docs_v2_matrix.yaml",
|
||||
"runner": "process_v2",
|
||||
"mode": "full_chain",
|
||||
"passed": false,
|
||||
"mismatches": [
|
||||
"answer_mode: expected deterministic, got deterministic_file_list",
|
||||
"llm: expected answer to contain 'docs/logic/telegram-notification-loop.md'",
|
||||
"llm: expected answer to contain 'docs/domains/runtime-health-entity.md'"
|
||||
],
|
||||
"actual": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"sub_intent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"rag_count": 12,
|
||||
"llm_answer": "docs/README.md\ndocs/architecture/telegram-notify-app-overview.md",
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"path_scope": [],
|
||||
"symbol_candidates": [],
|
||||
"entity_candidates": [
|
||||
"TelegramNotifyWorker",
|
||||
"TelegramNotifyModule",
|
||||
"TelegramSendService",
|
||||
"TelegramControlChannel",
|
||||
"RuntimeManager"
|
||||
],
|
||||
"doc_scope": [
|
||||
"architecture.telegram_notify_app",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"index.test_echo_app_docs",
|
||||
"docs/README.md"
|
||||
],
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"pipeline_steps": [
|
||||
{
|
||||
"step": "router_resolved",
|
||||
"output": {
|
||||
"domain": "DOCS",
|
||||
"intent": "DOC_EXPLAIN",
|
||||
"subintent": "FIND_FILES",
|
||||
"anchor_type": "FIND_FILES",
|
||||
"confidence": 1.0
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "anchors_extracted",
|
||||
"output": {
|
||||
"anchor_type": null,
|
||||
"endpoint_paths": [],
|
||||
"target_doc_hints": [],
|
||||
"matched_aliases": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_profile_selected",
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"layers": [
|
||||
"D1_DOCUMENT_CATALOG",
|
||||
"D3_ENTITY_CATALOG",
|
||||
"D0_DOC_CHUNKS"
|
||||
],
|
||||
"filters": {
|
||||
"prefer_path_prefixes": [
|
||||
"docs/"
|
||||
],
|
||||
"prefer_like_patterns": [],
|
||||
"target_doc_hints": []
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "retrieval_executed",
|
||||
"output": {
|
||||
"profile": "docs_find_files_catalog",
|
||||
"row_count": 12,
|
||||
"top_files": [
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/README.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md",
|
||||
"docs/architecture/telegram-notify-app-overview.md"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_assembled",
|
||||
"output": {
|
||||
"mode": "find_files",
|
||||
"primary_file": "docs/README.md",
|
||||
"file_count": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "evidence_gate_checked",
|
||||
"output": {
|
||||
"passed": false,
|
||||
"reason": "low_confidence_shortlist",
|
||||
"answer_mode": "deterministic_file_list"
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "answer_generated",
|
||||
"output": {
|
||||
"answer_mode": "deterministic_file_list",
|
||||
"answer_length": 64
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user