Console log: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup/suite_01_synthetic/test_results/code_qa_eval/20260311_233942/console_output.txt
Running evaluation: repo=/Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup/suite_01_synthetic/fixtures/code_qa_repo, fixture=True
Results: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup/suite_01_synthetic/test_results/code_qa_eval/20260311_233942
[1/13] main.py
[2/13] src/order_app/__init__.py
[3/13] src/order_app/api/__init__.py
[4/13] src/order_app/api/orders.py
[5/13] src/order_app/domain/__init__.py
[6/13] src/order_app/domain/order.py
[7/13] src/order_app/repositories/__init__.py
[8/13] src/order_app/repositories/order_repository.py
[9/13] src/order_app/services/__init__.py
[10/13] src/order_app/services/order_service.py
[11/13] src/order_app/utils/__init__.py
[12/13] src/order_app/utils/helpers.py
[13/13] tests/test_order_service.py
rag_session_id=d494838f-a9df-4e86-a11a-a2cf8225dbe5
intent router v2 request: turn=1 active_intent=None user_query=Открой файл main.py languages=['python'] domains=['CODE', 'DOCS']
intent router v2 result: {"schema_version": "1.1", "intent": "CODE_QA", "retrieval_profile": "code", "graph_id": "CodeQAGraph", "conversation_mode": "START", "query_plan": {"raw": "Открой файл main.py", "normalized": "Открой файл main.py", "sub_intent": "OPEN_FILE", "negations": [], "expansions": ["file"], "keyword_hints": [], "path_hints": ["main.py"], "doc_scope_hints": [], "symbol_candidates": [], "symbol_kind_hint": "unknown", "anchors": [{"type": "FILE_PATH", "value": "main.py", "source": "user_text", "subtype": null, "span": {"start": 12, "end": 19}, "confidence": 0.95}, {"type": "KEY_TERM", "value": "файл", "source": "user_text", "subtype": null, "span": {"start": 7, "end": 11}, "confidence": 0.9}]}, "retrieval_spec": {"domains": ["CODE"], "layer_queries": [{"layer_id": "C0_SOURCE_CHUNKS", "top_k": 12}], "filters": {"test_policy": "EXCLUDE", "path_scope": ["main.py"], "language": ["python"]}, "rerank_profile": "code"}, "retrieval_constraints": {"include_globs": ["main.py"], "exclude_globs": ["tests/**", "**/test_*.py", "**/*_test.py"], "prefer_globs": [], "test_file_globs": [], "test_symbol_patterns": [], "max_candidates": 20, "fuzzy_symbol_search": {"enabled": false, "max_distance": 2, "top_k": 5}}, "symbol_resolution": {"status": "not_requested", "resolved_symbol": null, "alternatives": [], "confidence": 0.0}, "evidence_policy": {"require_def": false, "require_flow": false, "require_spec": false, "allow_answer_without_evidence": false}}
intent router v2 request: turn=1 active_intent=None user_query=Покажи src/order_app/api/orders.py languages=['python'] domains=['CODE', 'DOCS']
intent router v2 result: {"schema_version": "1.1", "intent": "CODE_QA", "retrieval_profile": "code", "graph_id": "CodeQAGraph", "conversation_mode": "START", "query_plan": {"raw": "Покажи src/order_app/api/orders.py", "normalized": "Покажи src/order_app/api/orders.py", "sub_intent": "OPEN_FILE", "negations": [], "expansions": [], "keyword_hints": [], "path_hints": ["src/order_app/api/orders.py"], "doc_scope_hints": [], "symbol_candidates": [], "symbol_kind_hint": "unknown", "anchors": [{"type": "FILE_PATH", "value": "src/order_app/api/orders.py", "source": "user_text", "subtype": null, "span": {"start": 7, "end": 34}, "confidence": 0.95}]}, "retrieval_spec": {"domains": ["CODE"], "layer_queries": [{"layer_id": "C0_SOURCE_CHUNKS", "top_k": 12}], "filters": {"test_policy": "EXCLUDE", "path_scope": ["src/order_app/api/orders.py"], "language": ["python"]}, "rerank_profile": "code"}, "retrieval_constraints": {"include_globs": ["src/order_app/api/orders.py"], "exclude_globs": ["tests/**", "**/test_*.py", "**/*_test.py"], "prefer_globs": [], "test_file_globs": [], "test_symbol_patterns": [], "max_candidates": 20, "fuzzy_symbol_search": {"enabled": false, "max_distance": 2, "top_k": 5}}, "symbol_resolution": {"status": "not_requested", "resolved_symbol": null, "alternatives": [], "confidence": 0.0}, "evidence_policy": {"require_def": false, "require_flow": false, "require_spec": false, "allow_answer_without_evidence": false}}
intent router v2 request: turn=1 active_intent=None user_query=Open main languages=['python'] domains=['CODE', 'DOCS']
intent router v2 result: {"schema_version": "1.1", "intent": "CODE_QA", "retrieval_profile": "code", "graph_id": "CodeQAGraph", "conversation_mode": "START", "query_plan": {"raw": "Open main", "normalized": "Open main", "sub_intent": "FIND_ENTRYPOINTS", "negations": [], "expansions": ["Open", "main"], "keyword_hints": ["Open", "main"], "path_hints": [], "doc_scope_hints": [], "symbol_candidates": ["Open", "main"], "symbol_kind_hint": "class", "anchors": [{"type": "SYMBOL", "value": "Open", "source": "user_text", "subtype": null, "span": {"start": 0, "end": 4}, "confidence": 0.88}, {"type": "SYMBOL", "value": "main", "source": "user_text", "subtype": null, "span": {"start": 5, "end": 9}, "confidence": 0.88}]}, "retrieval_spec": {"domains": ["CODE"], "layer_queries": [{"layer_id": "C3_ENTRYPOINTS", "top_k": 12}, {"layer_id": "C0_SOURCE_CHUNKS", "top_k": 6}], "filters": {"test_policy": "EXCLUDE", "path_scope": [], "language": ["python"]}, "rerank_profile": "code"}, "retrieval_constraints": {"include_globs": ["src/**"], "exclude_globs": ["tests/**", "**/test_*.py", "**/*_test.py"], "prefer_globs": [], "test_file_globs": [], "test_symbol_patterns": [], "max_candidates": 20, "fuzzy_symbol_search": {"enabled": true, "max_distance": 2, "top_k": 5}}, "symbol_resolution": {"status": "pending", "resolved_symbol": null, "alternatives": ["Open", "main"], "confidence": 0.0}, "evidence_policy": {"require_def": true, "require_flow": true, "require_spec": false, "allow_answer_without_evidence": false}}
intent router v2 request: turn=1 active_intent=None user_query=Открой файл nonexistent/foo.py languages=['python'] domains=['CODE', 'DOCS']
intent router v2 result: {"schema_version": "1.1", "intent": "CODE_QA", "retrieval_profile": "code", "graph_id": "CodeQAGraph", "conversation_mode": "START", "query_plan": {"raw": "Открой файл nonexistent/foo.py", "normalized": "Открой файл nonexistent/foo.py", "sub_intent": "OPEN_FILE", "negations": [], "expansions": ["file"], "keyword_hints": [], "path_hints": ["nonexistent/foo.py"], "doc_scope_hints": [], "symbol_candidates": [], "symbol_kind_hint": "unknown", "anchors": [{"type": "FILE_PATH", "value": "nonexistent/foo.py", "source": "user_text", "subtype": null, "span": {"start": 12, "end": 30}, "confidence": 0.95}, {"type": "KEY_TERM", "value": "файл", "source": "user_text", "subtype": null, "span": {"start": 7, "end": 11}, "confidence": 0.9}]}, "retrieval_spec": {"domains": ["CODE"], "layer_queries": [{"layer_id": "C0_SOURCE_CHUNKS", "top_k": 12}], "filters": {"test_policy": "EXCLUDE", "path_scope": ["nonexistent/foo.py"], "language": ["python"]}, "rerank_profile": "code"}, "retrieval_constraints": {"include_globs": ["nonexistent/foo.py"], "exclude_globs": ["tests/**", "**/test_*.py", "**/*_test.py"], "prefer_globs": [], "test_file_globs": [], "test_symbol_patterns": [], "max_candidates": 20, "fuzzy_symbol_search": {"enabled": false, "max_distance": 2, "top_k": 5}}, "symbol_resolution": {"status": "not_requested", "resolved_symbol": null, "alternatives": [], "confidence": 0.0}, "evidence_policy": {"require_def": false, "require_flow": false, "require_spec": false, "allow_answer_without_evidence": false}}
intent router v2 request: turn=1 active_intent=None user_query=Объясни класс Order languages=['python'] domains=['CODE', 'DOCS']
intent router v2 result: {"schema_version": "1.1", "intent": "CODE_QA", "retrieval_profile": "code", "graph_id": "CodeQAGraph", "conversation_mode": "START", "query_plan": {"raw": "Объясни класс Order", "normalized": "Объясни класс Order", "sub_intent": "EXPLAIN", "negations": [], "expansions": ["Order"], "keyword_hints": ["Order"], "path_hints": [], "doc_scope_hints": [], "symbol_candidates": ["Order"], "symbol_kind_hint": "class", "anchors": [{"type": "SYMBOL", "value": "Order", "source": "user_text", "subtype": null, "span": {"start": 14, "end": 19}, "confidence": 0.88}, {"type": "KEY_TERM", "value": "класс", "source": "user_text", "subtype": null, "span": {"start": 8, "end": 13}, "confidence": 0.9}]}, "retrieval_spec": {"domains": ["CODE"], "layer_queries": [{"layer_id": "C1_SYMBOL_CATALOG", "top_k": 8}, {"layer_id": "C0_SOURCE_CHUNKS", "top_k": 8}, {"layer_id": "C4_SEMANTIC_ROLES", "top_k": 8}, {"layer_id": "C2_DEPENDENCY_GRAPH", "top_k": 6}, {"layer_id": "C3_ENTRYPOINTS", "top_k": 6}], "filters": {"test_policy": "EXCLUDE", "path_scope": [], "language": ["python"]}, "rerank_profile": "code"}, "retrieval_constraints": {"include_globs": ["src/**"], "exclude_globs": ["tests/**", "**/test_*.py", "**/*_test.py"], "prefer_globs": [], "test_file_globs": [], "test_symbol_patterns": [], "max_candidates": 20, "fuzzy_symbol_search": {"enabled": true, "max_distance": 2, "top_k": 5}}, "symbol_resolution": {"status": "pending", "resolved_symbol": null, "alternatives": ["Order"], "confidence": 0.0}, "evidence_policy": {"require_def": true, "require_flow": true, "require_spec": false, "allow_answer_without_evidence": false}}
intent router v2 request: turn=1 active_intent=None user_query=Как работает OrderService? languages=['python'] domains=['CODE', 'DOCS']
intent router v2 result: {"schema_version": "1.1", "intent": "CODE_QA", "retrieval_profile": "code", "graph_id": "CodeQAGraph", "conversation_mode": "START", "query_plan": {"raw": "Как работает OrderService?", "normalized": "Как работает OrderService?", "sub_intent": "EXPLAIN", "negations": [], "expansions": ["OrderService"], "keyword_hints": ["OrderService"], "path_hints": [], "doc_scope_hints": [], "symbol_candidates": ["OrderService"], "symbol_kind_hint": "class", "anchors": [{"type": "SYMBOL", "value": "OrderService", "source": "user_text", "subtype": null, "span": {"start": 13, "end": 25}, "confidence": 0.88}]}, "retrieval_spec": {"domains": ["CODE"], "layer_queries": [{"layer_id": "C1_SYMBOL_CATALOG", "top_k": 8}, {"layer_id": "C0_SOURCE_CHUNKS", "top_k": 8}, {"layer_id": "C4_SEMANTIC_ROLES", "top_k": 8}, {"layer_id": "C2_DEPENDENCY_GRAPH", "top_k": 6}, {"layer_id": "C3_ENTRYPOINTS", "top_k": 6}], "filters": {"test_policy": "EXCLUDE", "path_scope": [], "language": ["python"]}, "rerank_profile": "code"}, "retrieval_constraints": {"include_globs": ["src/**"], "exclude_globs": ["tests/**", "**/test_*.py", "**/*_test.py"], "prefer_globs": [], "test_file_globs": [], "test_symbol_patterns": [], "max_candidates": 20, "fuzzy_symbol_search": {"enabled": true, "max_distance": 2, "top_k": 5}}, "symbol_resolution": {"status": "pending", "resolved_symbol": null, "alternatives": ["OrderService"], "confidence": 0.0}, "evidence_policy": {"require_def": true, "require_flow": true, "require_spec": false, "allow_answer_without_evidence": false}}
intent router v2 request: turn=1 active_intent=None user_query=Что делает create_order? languages=['python'] domains=['CODE', 'DOCS']
intent router v2 result: {"schema_version": "1.1", "intent": "PROJECT_MISC", "retrieval_profile": "code", "graph_id": "ProjectMiscGraph", "conversation_mode": "START", "query_plan": {"raw": "Что делает create_order?", "normalized": "Что делает create_order?", "sub_intent": "EXPLAIN", "negations": [], "expansions": ["create_order"], "keyword_hints": ["create_order"], "path_hints": [], "doc_scope_hints": [], "symbol_candidates": ["create_order"], "symbol_kind_hint": "function", "anchors": [{"type": "SYMBOL", "value": "create_order", "source": "user_text", "subtype": null, "span": {"start": 11, "end": 23}, "confidence": 0.88}]}, "retrieval_spec": {"domains": ["CODE", "DOCS"], "layer_queries": [{"layer_id": "D1_MODULE_CATALOG", "top_k": 4}, {"layer_id": "D3_SECTION_INDEX", "top_k": 6}, {"layer_id": "C1_SYMBOL_CATALOG", "top_k": 4}, {"layer_id": "C0_SOURCE_CHUNKS", "top_k": 4}], "filters": {"test_policy": "EXCLUDE", "path_scope": [], "language": ["python"], "doc_kinds": [], "doc_language": []}, "rerank_profile": "project"}, "retrieval_constraints": {"include_globs": ["src/**"], "exclude_globs": ["tests/**", "**/test_*.py", "**/*_test.py"], "prefer_globs": [], "test_file_globs": [], "test_symbol_patterns": [], "max_candidates": 20, "fuzzy_symbol_search": {"enabled": true, "max_distance": 2, "top_k": 5}}, "symbol_resolution": {"status": "pending", "resolved_symbol": null, "alternatives": ["create_order"], "confidence": 0.0}, "evidence_policy": {"require_def": false, "require_flow": false, "require_spec": false, "allow_answer_without_evidence": true}}
intent router v2 request: turn=1 active_intent=None user_query=Объясни класс NonExistentClass languages=['python'] domains=['CODE', 'DOCS']
intent router v2 result: {"schema_version": "1.1", "intent": "CODE_QA", "retrieval_profile": "code", "graph_id": "CodeQAGraph", "conversation_mode": "START", "query_plan": {"raw": "Объясни класс NonExistentClass", "normalized": "Объясни класс NonExistentClass", "sub_intent": "EXPLAIN", "negations": [], "expansions": ["NonExistentClass"], "keyword_hints": ["NonExistentClass"], "path_hints": [], "doc_scope_hints": [], "symbol_candidates": ["NonExistentClass"], "symbol_kind_hint": "class", "anchors": [{"type": "SYMBOL", "value": "NonExistentClass", "source": "user_text", "subtype": null, "span": {"start": 14, "end": 30}, "confidence": 0.88}, {"type": "KEY_TERM", "value": "класс", "source": "user_text", "subtype": null, "span": {"start": 8, "end": 13}, "confidence": 0.9}]}, "retrieval_spec": {"domains": ["CODE"], "layer_queries": [{"layer_id": "C1_SYMBOL_CATALOG", "top_k": 8}, {"layer_id": "C0_SOURCE_CHUNKS", "top_k": 8}, {"layer_id": "C4_SEMANTIC_ROLES", "top_k": 8}, {"layer_id": "C2_DEPENDENCY_GRAPH", "top_k": 6}, {"layer_id": "C3_ENTRYPOINTS", "top_k": 6}], "filters": {"test_policy": "EXCLUDE", "path_scope": [], "language": ["python"]}, "rerank_profile": "code"}, "retrieval_constraints": {"include_globs": ["src/**"], "exclude_globs": ["tests/**", "**/test_*.py", "**/*_test.py"], "prefer_globs": [], "test_file_globs": [], "test_symbol_patterns": [], "max_candidates": 20, "fuzzy_symbol_search": {"enabled": true, "max_distance": 2, "top_k": 5}}, "symbol_resolution": {"status": "pending", "resolved_symbol": null, "alternatives": ["NonExistentClass"], "confidence": 0.0}, "evidence_policy": {"require_def": true, "require_flow": true, "require_spec": false, "allow_answer_without_evidence": false}}
intent router v2 request: turn=1 active_intent=None user_query=Где тесты для OrderService? languages=['python'] domains=['CODE', 'DOCS']
intent router v2 result: {"schema_version": "1.1", "intent": "CODE_QA", "retrieval_profile": "code", "graph_id": "CodeQAGraph", "conversation_mode": "START", "query_plan": {"raw": "Где тесты для OrderService?", "normalized": "Где тесты для OrderService?", "sub_intent": "FIND_TESTS", "negations": [], "expansions": ["test", "unit test", "OrderService"], "keyword_hints": ["OrderService"], "path_hints": [], "doc_scope_hints": [], "symbol_candidates": ["OrderService"], "symbol_kind_hint": "class", "anchors": [{"type": "SYMBOL", "value": "OrderService", "source": "user_text", "subtype": null, "span": {"start": 14, "end": 26}, "confidence": 0.88}, {"type": "KEY_TERM", "value": "тест", "source": "user_text", "subtype": null, "span": {"start": 4, "end": 9}, "confidence": 0.9}]}, "retrieval_spec": {"domains": ["CODE"], "layer_queries": [{"layer_id": "C1_SYMBOL_CATALOG", "top_k": 8}, {"layer_id": "C2_DEPENDENCY_GRAPH", "top_k": 6}, {"layer_id": "C0_SOURCE_CHUNKS", "top_k": 10}], "filters": {"test_policy": "INCLUDE", "path_scope": [], "language": ["python"]}, "rerank_profile": "code"}, "retrieval_constraints": {"include_globs": ["src/**", "tests/**", "**/test_*.py", "**/*_test.py", "**/conftest.py"], "exclude_globs": [], "prefer_globs": ["tests/**", "**/test_*.py", "**/*_test.py"], "test_file_globs": ["tests/**", "**/test_*.py", "**/*_test.py", "**/conftest.py"], "test_symbol_patterns": ["test_order_service", "TestOrderService", "OrderService"], "max_candidates": 20, "fuzzy_symbol_search": {"enabled": true, "max_distance": 2, "top_k": 5}}, "symbol_resolution": {"status": "pending", "resolved_symbol": null, "alternatives": ["OrderService"], "confidence": 0.0}, "evidence_policy": {"require_def": true, "require_flow": true, "require_spec": false, "allow_answer_without_evidence": false}}
intent router v2 request: turn=1 active_intent=None user_query=Найди тесты для Order languages=['python'] domains=['CODE', 'DOCS']
intent router v2 result: {"schema_version": "1.1", "intent": "CODE_QA", "retrieval_profile": "code", "graph_id": "CodeQAGraph", "conversation_mode": "START", "query_plan": {"raw": "Найди тесты для Order", "normalized": "Найди тесты для Order", "sub_intent": "FIND_TESTS", "negations": [], "expansions": ["test", "unit test", "Order"], "keyword_hints": ["Order"], "path_hints": [], "doc_scope_hints": [], "symbol_candidates": ["Order"], "symbol_kind_hint": "class", "anchors": [{"type": "SYMBOL", "value": "Order", "source": "user_text", "subtype": null, "span": {"start": 16, "end": 21}, "confidence": 0.88}, {"type": "KEY_TERM", "value": "тест", "source": "user_text", "subtype": null, "span": {"start": 6, "end": 11}, "confidence": 0.9}]}, "retrieval_spec": {"domains": ["CODE"], "layer_queries": [{"layer_id": "C1_SYMBOL_CATALOG", "top_k": 8}, {"layer_id": "C2_DEPENDENCY_GRAPH", "top_k": 6}, {"layer_id": "C0_SOURCE_CHUNKS", "top_k": 10}], "filters": {"test_policy": "INCLUDE", "path_scope": [], "language": ["python"]}, "rerank_profile": "code"}, "retrieval_constraints": {"include_globs": ["src/**", "tests/**", "**/test_*.py", "**/*_test.py", "**/conftest.py"], "exclude_globs": [], "prefer_globs": ["tests/**", "**/test_*.py", "**/*_test.py"], "test_file_globs": ["tests/**", "**/test_*.py", "**/*_test.py", "**/conftest.py"], "test_symbol_patterns": ["test_order", "TestOrder", "Order"], "max_candidates": 20, "fuzzy_symbol_search": {"enabled": true, "max_distance": 2, "top_k": 5}}, "symbol_resolution": {"status": "pending", "resolved_symbol": null, "alternatives": ["Order"], "confidence": 0.0}, "evidence_policy": {"require_def": true, "require_flow": true, "require_spec": false, "allow_answer_without_evidence": false}}
intent router v2 request: turn=1 active_intent=None user_query=Есть ли тесты на репозиторий? languages=['python'] domains=['CODE', 'DOCS']
intent router v2 result: {"schema_version": "1.1", "intent": "CODE_QA", "retrieval_profile": "code", "graph_id": "CodeQAGraph", "conversation_mode": "START", "query_plan": {"raw": "Есть ли тесты на репозиторий?", "normalized": "Есть ли тесты на репозиторий?", "sub_intent": "FIND_TESTS", "negations": [], "expansions": ["test", "unit test"], "keyword_hints": [], "path_hints": [], "doc_scope_hints": [], "symbol_candidates": [], "symbol_kind_hint": "unknown", "anchors": [{"type": "KEY_TERM", "value": "тест", "source": "user_text", "subtype": null, "span": {"start": 8, "end": 13}, "confidence": 0.9}]}, "retrieval_spec": {"domains": ["CODE"], "layer_queries": [{"layer_id": "C1_SYMBOL_CATALOG", "top_k": 8}, {"layer_id": "C2_DEPENDENCY_GRAPH", "top_k": 6}, {"layer_id": "C0_SOURCE_CHUNKS", "top_k": 10}], "filters": {"test_policy": "INCLUDE", "path_scope": [], "language": ["python"]}, "rerank_profile": "code"}, "retrieval_constraints": {"include_globs": ["src/**", "tests/**", "**/test_*.py", "**/*_test.py", "**/conftest.py"], "exclude_globs": [], "prefer_globs": ["tests/**", "**/test_*.py", "**/*_test.py"], "test_file_globs": ["tests/**", "**/test_*.py", "**/*_test.py", "**/conftest.py"], "test_symbol_patterns": [], "max_candidates": 20, "fuzzy_symbol_search": {"enabled": true, "max_distance": 2, "top_k": 5}}, "symbol_resolution": {"status": "not_requested", "resolved_symbol": null, "alternatives": [], "confidence": 0.0}, "evidence_policy": {"require_def": true, "require_flow": true, "require_spec": false, "allow_answer_without_evidence": false}}
intent router v2 request: turn=1 active_intent=None user_query=Где тесты для NonExistent? languages=['python'] domains=['CODE', 'DOCS']
intent router v2 result: {"schema_version": "1.1", "intent": "CODE_QA", "retrieval_profile": "code", "graph_id": "CodeQAGraph", "conversation_mode": "START", "query_plan": {"raw": "Где тесты для NonExistent?", "normalized": "Где тесты для NonExistent?", "sub_intent": "FIND_TESTS", "negations": [], "expansions": ["test", "unit test", "NonExistent"], "keyword_hints": ["NonExistent"], "path_hints": [], "doc_scope_hints": [], "symbol_candidates": ["NonExistent"], "symbol_kind_hint": "class", "anchors": [{"type": "SYMBOL", "value": "NonExistent", "source": "user_text", "subtype": null, "span": {"start": 14, "end": 25}, "confidence": 0.88}, {"type": "KEY_TERM", "value": "тест", "source": "user_text", "subtype": null, "span": {"start": 4, "end": 9}, "confidence": 0.9}]}, "retrieval_spec": {"domains": ["CODE"], "layer_queries": [{"layer_id": "C1_SYMBOL_CATALOG", "top_k": 8}, {"layer_id": "C2_DEPENDENCY_GRAPH", "top_k": 6}, {"layer_id": "C0_SOURCE_CHUNKS", "top_k": 10}], "filters": {"test_policy": "INCLUDE", "path_scope": [], "language": ["python"]}, "rerank_profile": "code"}, "retrieval_constraints": {"include_globs": ["src/**", "tests/**", "**/test_*.py", "**/*_test.py", "**/conftest.py"], "exclude_globs": [], "prefer_globs": ["tests/**", "**/test_*.py", "**/*_test.py"], "test_file_globs": ["tests/**", "**/test_*.py", "**/*_test.py", "**/conftest.py"], "test_symbol_patterns": ["test_non_existent", "TestNonExistent", "NonExistent"], "max_candidates": 20, "fuzzy_symbol_search": {"enabled": true, "max_distance": 2, "top_k": 5}}, "symbol_resolution": {"status": "pending", "resolved_symbol": null, "alternatives": ["NonExistent"], "confidence": 0.0}, "evidence_policy": {"require_def": true, "require_flow": true, "require_spec": false, "allow_answer_without_evidence": false}}
intent router v2 request: turn=1 active_intent=None user_query=Какие точки входа в приложение? languages=['python'] domains=['CODE', 'DOCS']
intent router v2 result: {"schema_version": "1.1", "intent": "PROJECT_MISC", "retrieval_profile": "code", "graph_id": "ProjectMiscGraph", "conversation_mode": "START", "query_plan": {"raw": "Какие точки входа в приложение?", "normalized": "Какие точки входа в приложение?", "sub_intent": "FIND_ENTRYPOINTS", "negations": [], "expansions": [], "keyword_hints": [], "path_hints": [], "doc_scope_hints": [], "symbol_candidates": [], "symbol_kind_hint": "unknown", "anchors": []}, "retrieval_spec": {"domains": ["CODE", "DOCS"], "layer_queries": [{"layer_id": "D1_MODULE_CATALOG", "top_k": 4}, {"layer_id": "D3_SECTION_INDEX", "top_k": 6}, {"layer_id": "C1_SYMBOL_CATALOG", "top_k": 4}, {"layer_id": "C0_SOURCE_CHUNKS", "top_k": 4}], "filters": {"test_policy": "EXCLUDE", "path_scope": [], "language": ["python"], "doc_kinds": [], "doc_language": []}, "rerank_profile": "project"}, "retrieval_constraints": {"include_globs": ["src/**"], "exclude_globs": ["tests/**", "**/test_*.py", "**/*_test.py"], "prefer_globs": [], "test_file_globs": [], "test_symbol_patterns": [], "max_candidates": 20, "fuzzy_symbol_search": {"enabled": true, "max_distance": 2, "top_k": 5}}, "symbol_resolution": {"status": "not_requested", "resolved_symbol": null, "alternatives": [], "confidence": 0.0}, "evidence_policy": {"require_def": false, "require_flow": false, "require_spec": false, "allow_answer_without_evidence": true}}
intent router v2 request: turn=1 active_intent=None user_query=Find application entrypoints languages=['python'] domains=['CODE', 'DOCS']
intent router v2 result: {"schema_version": "1.1", "intent": "CODE_QA", "retrieval_profile": "code", "graph_id": "CodeQAGraph", "conversation_mode": "START", "query_plan": {"raw": "Find application entrypoints", "normalized": "Find application entrypoints", "sub_intent": "FIND_ENTRYPOINTS", "negations": [], "expansions": ["Find", "application", "entrypoints"], "keyword_hints": ["Find", "application", "entrypoints"], "path_hints": [], "doc_scope_hints": [], "symbol_candidates": ["Find", "application", "entrypoints"], "symbol_kind_hint": "class", "anchors": [{"type": "SYMBOL", "value": "Find", "source": "user_text", "subtype": null, "span": {"start": 0, "end": 4}, "confidence": 0.88}, {"type": "SYMBOL", "value": "application", "source": "user_text", "subtype": null, "span": {"start": 5, "end": 16}, "confidence": 0.88}, {"type": "SYMBOL", "value": "entrypoints", "source": "user_text", "subtype": null, "span": {"start": 17, "end": 28}, "confidence": 0.88}]}, "retrieval_spec": {"domains": ["CODE"], "layer_queries": [{"layer_id": "C3_ENTRYPOINTS", "top_k": 12}, {"layer_id": "C0_SOURCE_CHUNKS", "top_k": 6}], "filters": {"test_policy": "EXCLUDE", "path_scope": [], "language": ["python"]}, "rerank_profile": "code"}, "retrieval_constraints": {"include_globs": ["src/**"], "exclude_globs": ["tests/**", "**/test_*.py", "**/*_test.py"], "prefer_globs": [], "test_file_globs": [], "test_symbol_patterns": [], "max_candidates": 20, "fuzzy_symbol_search": {"enabled": true, "max_distance": 2, "top_k": 5}}, "symbol_resolution": {"status": "pending", "resolved_symbol": null, "alternatives": ["Find", "application", "entrypoints"], "confidence": 0.0}, "evidence_policy": {"require_def": true, "require_flow": true, "require_spec": false, "allow_answer_without_evidence": false}}
intent router v2 request: turn=1 active_intent=None user_query=Где main? languages=['python'] domains=['CODE', 'DOCS']
intent router v2 result: {"schema_version": "1.1", "intent": "PROJECT_MISC", "retrieval_profile": "code", "graph_id": "ProjectMiscGraph", "conversation_mode": "START", "query_plan": {"raw": "Где main?", "normalized": "Где main?", "sub_intent": "FIND_ENTRYPOINTS", "negations": [], "expansions": ["main"], "keyword_hints": ["main"], "path_hints": [], "doc_scope_hints": [], "symbol_candidates": ["main"], "symbol_kind_hint": "unknown", "anchors": [{"type": "SYMBOL", "value": "main", "source": "user_text", "subtype": null, "span": {"start": 4, "end": 8}, "confidence": 0.88}]}, "retrieval_spec": {"domains": ["CODE", "DOCS"], "layer_queries": [{"layer_id": "D1_MODULE_CATALOG", "top_k": 4}, {"layer_id": "D3_SECTION_INDEX", "top_k": 6}, {"layer_id": "C1_SYMBOL_CATALOG", "top_k": 4}, {"layer_id": "C0_SOURCE_CHUNKS", "top_k": 4}], "filters": {"test_policy": "EXCLUDE", "path_scope": [], "language": ["python"], "doc_kinds": [], "doc_language": []}, "rerank_profile": "project"}, "retrieval_constraints": {"include_globs": ["src/**"], "exclude_globs": ["tests/**", "**/test_*.py", "**/*_test.py"], "prefer_globs": [], "test_file_globs": [], "test_symbol_patterns": [], "max_candidates": 20, "fuzzy_symbol_search": {"enabled": true, "max_distance": 2, "top_k": 5}}, "symbol_resolution": {"status": "pending", "resolved_symbol": null, "alternatives": ["main"], "confidence": 0.0}, "evidence_policy": {"require_def": false, "require_flow": false, "require_spec": false, "allow_answer_without_evidence": true}}
intent router v2 request: turn=1 active_intent=None user_query=Что делает этот проект? languages=['python'] domains=['CODE', 'DOCS']
intent router v2 result: {"schema_version": "1.1", "intent": "PROJECT_MISC", "retrieval_profile": "code", "graph_id": "ProjectMiscGraph", "conversation_mode": "START", "query_plan": {"raw": "Что делает этот проект?", "normalized": "Что делает этот проект?", "sub_intent": "EXPLAIN", "negations": [], "expansions": [], "keyword_hints": [], "path_hints": [], "doc_scope_hints": [], "symbol_candidates": [], "symbol_kind_hint": "unknown", "anchors": []}, "retrieval_spec": {"domains": ["CODE", "DOCS"], "layer_queries": [{"layer_id": "D1_MODULE_CATALOG", "top_k": 4}, {"layer_id": "D3_SECTION_INDEX", "top_k": 6}, {"layer_id": "C1_SYMBOL_CATALOG", "top_k": 4}, {"layer_id": "C0_SOURCE_CHUNKS", "top_k": 4}], "filters": {"test_policy": "EXCLUDE", "path_scope": [], "language": ["python"], "doc_kinds": [], "doc_language": []}, "rerank_profile": "project"}, "retrieval_constraints": {"include_globs": ["src/**"], "exclude_globs": ["tests/**", "**/test_*.py", "**/*_test.py"], "prefer_globs": [], "test_file_globs": [], "test_symbol_patterns": [], "max_candidates": 20, "fuzzy_symbol_search": {"enabled": true, "max_distance": 2, "top_k": 5}}, "symbol_resolution": {"status": "not_requested", "resolved_symbol": null, "alternatives": [], "confidence": 0.0}, "evidence_policy": {"require_def": false, "require_flow": false, "require_spec": false, "allow_answer_without_evidence": true}}
intent router v2 request: turn=1 active_intent=None user_query=How does order creation work? languages=['python'] domains=['CODE', 'DOCS']
intent router v2 result: {"schema_version": "1.1", "intent": "CODE_QA", "retrieval_profile": "code", "graph_id": "CodeQAGraph", "conversation_mode": "START", "query_plan": {"raw": "How does order creation work?", "normalized": "How does order creation work?", "sub_intent": "ARCHITECTURE", "negations": [], "expansions": ["How", "does", "order", "creation", "work"], "keyword_hints": ["How", "does", "order", "creation", "work"], "path_hints": [], "doc_scope_hints": [], "symbol_candidates": ["How", "does", "order", "creation", "work"], "symbol_kind_hint": "class", "anchors": [{"type": "SYMBOL", "value": "How", "source": "user_text", "subtype": null, "span": {"start": 0, "end": 3}, "confidence": 0.88}, {"type": "SYMBOL", "value": "does", "source": "user_text", "subtype": null, "span": {"start": 4, "end": 8}, "confidence": 0.88}, {"type": "SYMBOL", "value": "order", "source": "user_text", "subtype": null, "span": {"start": 9, "end": 14}, "confidence": 0.88}, {"type": "SYMBOL", "value": "creation", "source": "user_text", "subtype": null, "span": {"start": 15, "end": 23}, "confidence": 0.88}, {"type": "SYMBOL", "value": "work", "source": "user_text", "subtype": null, "span": {"start": 24, "end": 28}, "confidence": 0.88}]}, "retrieval_spec": {"domains": ["CODE"], "layer_queries": [{"layer_id": "C4_SEMANTIC_ROLES", "top_k": 12}, {"layer_id": "C3_ENTRYPOINTS", "top_k": 8}, {"layer_id": "C2_DEPENDENCY_GRAPH", "top_k": 8}, {"layer_id": "C1_SYMBOL_CATALOG", "top_k": 6}, {"layer_id": "C0_SOURCE_CHUNKS", "top_k": 4}], "filters": {"test_policy": "EXCLUDE", "path_scope": [], "language": ["python"]}, "rerank_profile": "code"}, "retrieval_constraints": {"include_globs": ["src/**"], "exclude_globs": ["tests/**", "**/test_*.py", "**/*_test.py"], "prefer_globs": [], "test_file_globs": [], "test_symbol_patterns": [], "max_candidates": 20, "fuzzy_symbol_search": {"enabled": true, "max_distance": 2, "top_k": 5}}, "symbol_resolution": {"status": "pending", "resolved_symbol": null, "alternatives": ["How", "does", "order", "creation", "work"], "confidence": 0.0}, "evidence_policy": {"require_def": true, "require_flow": true, "require_spec": false, "allow_answer_without_evidence": false}}
intent router v2 request: turn=1 active_intent=None user_query=Расскажи про код languages=['python'] domains=['CODE', 'DOCS']
intent router v2 result: {"schema_version": "1.1", "intent": "CODE_QA", "retrieval_profile": "code", "graph_id": "CodeQAGraph", "conversation_mode": "START", "query_plan": {"raw": "Расскажи про код", "normalized": "Расскажи про код", "sub_intent": "EXPLAIN", "negations": [], "expansions": [], "keyword_hints": [], "path_hints": [], "doc_scope_hints": [], "symbol_candidates": [], "symbol_kind_hint": "unknown", "anchors": []}, "retrieval_spec": {"domains": ["CODE"], "layer_queries": [{"layer_id": "C1_SYMBOL_CATALOG", "top_k": 8}, {"layer_id": "C0_SOURCE_CHUNKS", "top_k": 8}, {"layer_id": "C4_SEMANTIC_ROLES", "top_k": 8}, {"layer_id": "C2_DEPENDENCY_GRAPH", "top_k": 6}, {"layer_id": "C3_ENTRYPOINTS", "top_k": 6}], "filters": {"test_policy": "EXCLUDE", "path_scope": [], "language": ["python"]}, "rerank_profile": "code"}, "retrieval_constraints": {"include_globs": ["src/**"], "exclude_globs": ["tests/**", "**/test_*.py", "**/*_test.py"], "prefer_globs": [], "test_file_globs": [], "test_symbol_patterns": [], "max_candidates": 20, "fuzzy_symbol_search": {"enabled": true, "max_distance": 2, "top_k": 5}}, "symbol_resolution": {"status": "not_requested", "resolved_symbol": null, "alternatives": [], "confidence": 0.0}, "evidence_policy": {"require_def": true, "require_flow": true, "require_spec": false, "allow_answer_without_evidence": false}}

8/18 cases passed. Summary: /Users/alex/Dev_projects_v2/ai driven app process/v2/agent/tests/pipeline_setup/suite_01_synthetic/test_results/code_qa_eval/summary_20260311_233942.md
  FAIL open_file_borderline: sub_intent: expected OPEN_FILE, got FIND_ENTRYPOINTS
  FAIL open_file_negative: answer_mode: expected degraded, got insufficient
  FAIL explain_borderline: intent: expected CODE_QA, got PROJECT_MISC
  FAIL explain_negative: answer_mode: expected degraded, got normal
  FAIL find_tests_negative: answer_mode: expected degraded, got normal
  FAIL find_entrypoints_positive: intent: expected CODE_QA, got PROJECT_MISC; answer_mode: expected normal, got degraded
  FAIL find_entrypoints_borderline: intent: expected CODE_QA, got PROJECT_MISC; answer_mode: expected normal, got degraded
  FAIL general_qa_positive: intent: expected CODE_QA, got PROJECT_MISC; sub_intent: expected GENERAL_QA, got EXPLAIN
  FAIL general_qa_how: sub_intent: expected GENERAL_QA, got ARCHITECTURE
  FAIL general_qa_borderline: sub_intent: expected GENERAL_QA, got EXPLAIN
