Фикс состояния

2026-03-12 16:55:23 +03:00
parent 417b8b6f72
commit 6ba0a18ac9
1445 changed files with 620025 additions and 885 deletions
@@ -0,0 +1,13 @@
+# Golden cases for CODE_QA pipeline calibration
+
+Each case defines:
+- `id`: unique case id
+- `query`: user query text
+- `expected_intent`: CODE_QA (or DOCS_QA for docs-only; this set is code-only)
+- `expected_sub_intent`: OPEN_FILE | EXPLAIN | FIND_TESTS | FIND_ENTRYPOINTS | GENERAL_QA
+- `expected_answer_mode`: normal | degraded | insufficient
+- `expected_target_hint`: optional — path (for OPEN_FILE), symbol (for EXPLAIN), or test-like
+- `expected_layers`: optional — list of layer ids we expect in the retrieval plan
+- `notes`: optional — borderline, negative, or calibration hint
+
+We assert routing, retrieval alignment, evidence sufficiency, and answer mode — not exact LLM wording.
@@ -0,0 +1,142 @@
+# Golden cases for CODE_QA pipeline (fixture repo: code_qa_repo)
+# Scenarios: OPEN_FILE, EXPLAIN, FIND_TESTS, FIND_ENTRYPOINTS, GENERAL_QA
+
+cases:
+  # --- OPEN_FILE ---
+  - id: open_file_main_positive
+    query: "Открой файл main.py"
+    expected_intent: CODE_QA
+    expected_sub_intent: OPEN_FILE
+    expected_answer_mode: normal
+    expected_target_hint: path
+    expected_path_scope_contains: ["main.py"]
+    notes: "Clear path; fixture has main.py (src layout)"
+
+  - id: open_file_api_positive
+    query: "Покажи src/order_app/api/orders.py"
+    expected_intent: CODE_QA
+    expected_sub_intent: OPEN_FILE
+    expected_answer_mode: normal
+    expected_path_scope_contains: ["src/order_app/api/orders.py"]
+    notes: "Handler module (src layout)"
+
+  - id: open_file_borderline
+    query: "Open main"
+    expected_intent: CODE_QA
+    expected_sub_intent: OPEN_FILE
+    expected_answer_mode: normal
+    notes: "Short path hint; may resolve to main.py"
+
+  - id: open_file_negative
+    query: "Открой файл nonexistent/foo.py"
+    expected_intent: CODE_QA
+    expected_sub_intent: OPEN_FILE
+    expected_answer_mode: degraded
+    notes: "Path not in repo; evidence gate should fail or degrade"
+
+  # --- EXPLAIN ---
+  - id: explain_order_positive
+    query: "Объясни класс Order"
+    expected_intent: CODE_QA
+    expected_sub_intent: EXPLAIN
+    expected_answer_mode: normal
+    expected_target_hint: symbol
+    expected_symbol_candidates_contain: ["Order"]
+    notes: "Domain class in fixture"
+
+  - id: explain_order_service_positive
+    query: "Как работает OrderService?"
+    expected_intent: CODE_QA
+    expected_sub_intent: EXPLAIN
+    expected_answer_mode: normal
+    expected_symbol_candidates_contain: ["OrderService"]
+    notes: "Service layer"
+
+  - id: explain_borderline
+    query: "Что делает create_order?"
+    expected_intent: CODE_QA
+    expected_sub_intent: EXPLAIN
+    expected_answer_mode: normal
+    notes: "Function name; may resolve"
+
+  - id: explain_negative
+    query: "Объясни класс NonExistentClass"
+    expected_intent: CODE_QA
+    expected_sub_intent: EXPLAIN
+    expected_answer_mode: degraded
+    notes: "Symbol not in repo"
+
+  # --- FIND_TESTS ---
+  - id: find_tests_positive
+    query: "Где тесты для OrderService?"
+    expected_intent: CODE_QA
+    expected_sub_intent: FIND_TESTS
+    expected_answer_mode: normal
+    expected_target_hint: test-like
+    expected_symbol_candidates_contain: ["OrderService"]
+    notes: "Fixture has tests/test_order_service.py"
+
+  - id: find_tests_order_positive
+    query: "Найди тесты для Order"
+    expected_intent: CODE_QA
+    expected_sub_intent: FIND_TESTS
+    expected_answer_mode: normal
+    notes: "Tests reference Order"
+
+  - id: find_tests_borderline
+    query: "Есть ли тесты на репозиторий?"
+    expected_intent: CODE_QA
+    expected_sub_intent: FIND_TESTS
+    expected_answer_mode: normal
+    notes: "Vague target"
+
+  - id: find_tests_negative
+    query: "Где тесты для NonExistent?"
+    expected_intent: CODE_QA
+    expected_sub_intent: FIND_TESTS
+    expected_answer_mode: degraded
+    notes: "Target not in repo"
+
+  # --- FIND_ENTRYPOINTS ---
+  - id: find_entrypoints_positive
+    query: "Какие точки входа в приложение?"
+    expected_intent: CODE_QA
+    expected_sub_intent: FIND_ENTRYPOINTS
+    expected_answer_mode: normal
+    notes: "Fixture has main.py entrypoint (src layout)"
+
+  - id: find_entrypoints_english
+    query: "Find application entrypoints"
+    expected_intent: CODE_QA
+    expected_sub_intent: FIND_ENTRYPOINTS
+    expected_answer_mode: normal
+    notes: "English variant"
+
+  - id: find_entrypoints_borderline
+    query: "Где main?"
+    expected_intent: CODE_QA
+    expected_sub_intent: FIND_ENTRYPOINTS
+    expected_answer_mode: normal
+    notes: "Short; may route to entrypoints or OPEN_FILE"
+
+  # --- GENERAL_QA ---
+  - id: general_qa_positive
+    query: "Что делает этот проект?"
+    expected_intent: CODE_QA
+    expected_sub_intent: GENERAL_QA
+    expected_answer_mode: normal
+    notes: "Broad question; bounded context"
+
+  - id: general_qa_how
+    query: "How does order creation work?"
+    expected_intent: CODE_QA
+    expected_sub_intent: GENERAL_QA
+    expected_answer_mode: normal
+    notes: "General flow question"
+
+  - id: general_qa_borderline
+    query: "Расскажи про код"
+    expected_intent: CODE_QA
+    expected_sub_intent: GENERAL_QA
+    expected_answer_mode: normal
+    notes: "Very vague; fallback to GENERAL_QA"