diff --git a/app/__pycache__/main.cpython-312.pyc b/app/__pycache__/main.cpython-312.pyc index 2ec3f68..b399114 100644 Binary files a/app/__pycache__/main.cpython-312.pyc and b/app/__pycache__/main.cpython-312.pyc differ diff --git a/app/core/__pycache__/logging_setup.cpython-312.pyc b/app/core/__pycache__/logging_setup.cpython-312.pyc new file mode 100644 index 0000000..cfd21a2 Binary files /dev/null and b/app/core/__pycache__/logging_setup.cpython-312.pyc differ diff --git a/app/core/logging_setup.py b/app/core/logging_setup.py new file mode 100644 index 0000000..d32267b --- /dev/null +++ b/app/core/logging_setup.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import logging +import re + + +class ScrubbingFormatter(logging.Formatter): + _KEY_VALUE_PATTERNS = ( + re.compile(r"\b([A-Za-z_][A-Za-z0-9_]*id)=([^\s,]+)"), + re.compile(r"\b([A-Za-z_][A-Za-z0-9_]*_key)=([^\s,]+)"), + ) + _TEXT_PATTERNS = ( + re.compile(r"\b(index|task|dialog|rag|session|plan|artifact|evidence|symbol|edge|entry) id\b[:=]\s*([^\s,]+)", re.IGNORECASE), + ) + + def format(self, record: logging.LogRecord) -> str: + rendered = super().format(record) + scrubbed = self._scrub(rendered).rstrip("\n") + return scrubbed + "\n" + + def _scrub(self, message: str) -> str: + output = message + for pattern in self._KEY_VALUE_PATTERNS: + output = pattern.sub(self._replace_key_value, output) + for pattern in self._TEXT_PATTERNS: + output = pattern.sub(self._replace_text, output) + return output + + def _replace_key_value(self, match: re.Match[str]) -> str: + return f"{match.group(1)}=" + + def _replace_text(self, match: re.Match[str]) -> str: + return f"{match.group(1)} id=" + + +def configure_logging() -> None: + logging.basicConfig( + level=logging.WARNING, + force=True, + format="%(levelname)s:%(name)s:%(message)s", + ) + root_logger = logging.getLogger() + root_logger.setLevel(logging.WARNING) + formatter = ScrubbingFormatter("%(levelname)s:%(name)s:%(message)s") + for handler in root_logger.handlers: + handler.setFormatter(formatter) + logging.getLogger("uvicorn").setLevel(logging.WARNING) + logging.getLogger("uvicorn.error").setLevel(logging.WARNING) + logging.getLogger("uvicorn.access").setLevel(logging.WARNING) diff --git a/app/main.py b/app/main.py index e28c308..9d2eed0 100644 --- a/app/main.py +++ b/app/main.py @@ -1,10 +1,20 @@ +import logging + from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware +from app.core.logging_setup import configure_logging from app.core.error_handlers import register_error_handlers from app.modules.application import ModularApplication +def _configure_logging() -> None: + configure_logging() + + +_configure_logging() + + def create_app() -> FastAPI: app = FastAPI(title="Agent Backend MVP", version="0.1.0") modules = ModularApplication() diff --git a/app/modules/__pycache__/application.cpython-312.pyc b/app/modules/__pycache__/application.cpython-312.pyc index 8a36584..cc9982e 100644 Binary files a/app/modules/__pycache__/application.cpython-312.pyc and b/app/modules/__pycache__/application.cpython-312.pyc differ diff --git a/app/modules/agent/README.md b/app/modules/agent/README.md index ad8574c..d582dc5 100644 --- a/app/modules/agent/README.md +++ b/app/modules/agent/README.md @@ -37,6 +37,8 @@ classDiagram Методы: `run` — строит, валидирует и исполняет execution plan. - `TaskSpecBuilder`: формирует спецификацию задачи для оркестратора. Методы: `build` — собирает `TaskSpec` из route, контекстов и ограничений. +- `ProjectQaConversationGraphFactory`, `ProjectQaClassificationGraphFactory`, `ProjectQaRetrievalGraphFactory`, `ProjectQaAnalysisGraphFactory`, `ProjectQaAnswerGraphFactory`: набор маленьких graph-исполнителей для `project/qa`. + Роли: нормализация запроса; классификация project-question; поздний retrieval из `RAG`; анализ code/docs контекста; сборка финального ответа. - `StorySessionRecorder`: пишет session-scoped артефакты для последующего bind к Story. Методы: `record_run` — сохраняет входные источники и выходные артефакты сессии. - `StoryContextRepository`: репозиторий Story-контекста и его связей. @@ -58,3 +60,32 @@ sequenceDiagram Router->>Confluence: fetch_page(url) Confluence-->>Router: page(content_markdown, metadata) ``` + +### `project/qa` reasoning flow +Назначение: оркестратор планирует шаги, а каждый шаг исполняется отдельным graph. Retrieval вызывается поздно, внутри шага `context_retrieval`. +```mermaid +sequenceDiagram + participant Runtime as GraphAgentRuntime + participant Orch as OrchestratorService + participant G1 as conversation_understanding + participant G2 as question_classification + participant G3 as context_retrieval + participant Rag as RagService + participant G4 as context_analysis + participant G5 as answer_composition + + Runtime->>Orch: run(task) + Orch->>G1: execute + G1-->>Orch: resolved_request + Orch->>G2: execute + G2-->>Orch: question_profile + Orch->>G3: execute + G3->>Rag: retrieve(query) + Rag-->>G3: rag_items + G3-->>Orch: source_bundle + Orch->>G4: execute + G4-->>Orch: analysis_brief + Orch->>G5: execute + G5-->>Orch: final_answer + Orch-->>Runtime: final_answer +``` diff --git a/app/modules/agent/__pycache__/module.cpython-312.pyc b/app/modules/agent/__pycache__/module.cpython-312.pyc index 8912070..82e7d21 100644 Binary files a/app/modules/agent/__pycache__/module.cpython-312.pyc and b/app/modules/agent/__pycache__/module.cpython-312.pyc differ diff --git a/app/modules/agent/__pycache__/repository.cpython-312.pyc b/app/modules/agent/__pycache__/repository.cpython-312.pyc index fdbb0a2..2e9c444 100644 Binary files a/app/modules/agent/__pycache__/repository.cpython-312.pyc and b/app/modules/agent/__pycache__/repository.cpython-312.pyc differ diff --git a/app/modules/agent/__pycache__/service.cpython-312.pyc b/app/modules/agent/__pycache__/service.cpython-312.pyc index a030cfb..1ad28af 100644 Binary files a/app/modules/agent/__pycache__/service.cpython-312.pyc and b/app/modules/agent/__pycache__/service.cpython-312.pyc differ diff --git a/app/modules/agent/engine/graphs/__init__.py b/app/modules/agent/engine/graphs/__init__.py index 2821b8a..2564438 100644 --- a/app/modules/agent/engine/graphs/__init__.py +++ b/app/modules/agent/engine/graphs/__init__.py @@ -1,8 +1,13 @@ __all__ = [ "BaseGraphFactory", "DocsGraphFactory", + "ProjectQaAnalysisGraphFactory", + "ProjectQaAnswerGraphFactory", + "ProjectQaClassificationGraphFactory", + "ProjectQaConversationGraphFactory", "ProjectEditsGraphFactory", "ProjectQaGraphFactory", + "ProjectQaRetrievalGraphFactory", ] @@ -15,6 +20,26 @@ def __getattr__(name: str): from app.modules.agent.engine.graphs.docs_graph import DocsGraphFactory return DocsGraphFactory + if name == "ProjectQaConversationGraphFactory": + from app.modules.agent.engine.graphs.project_qa_step_graphs import ProjectQaConversationGraphFactory + + return ProjectQaConversationGraphFactory + if name == "ProjectQaClassificationGraphFactory": + from app.modules.agent.engine.graphs.project_qa_step_graphs import ProjectQaClassificationGraphFactory + + return ProjectQaClassificationGraphFactory + if name == "ProjectQaRetrievalGraphFactory": + from app.modules.agent.engine.graphs.project_qa_step_graphs import ProjectQaRetrievalGraphFactory + + return ProjectQaRetrievalGraphFactory + if name == "ProjectQaAnalysisGraphFactory": + from app.modules.agent.engine.graphs.project_qa_step_graphs import ProjectQaAnalysisGraphFactory + + return ProjectQaAnalysisGraphFactory + if name == "ProjectQaAnswerGraphFactory": + from app.modules.agent.engine.graphs.project_qa_step_graphs import ProjectQaAnswerGraphFactory + + return ProjectQaAnswerGraphFactory if name == "ProjectEditsGraphFactory": from app.modules.agent.engine.graphs.project_edits_graph import ProjectEditsGraphFactory diff --git a/app/modules/agent/engine/graphs/__pycache__/__init__.cpython-312.pyc b/app/modules/agent/engine/graphs/__pycache__/__init__.cpython-312.pyc index 55518be..435ee9f 100644 Binary files a/app/modules/agent/engine/graphs/__pycache__/__init__.cpython-312.pyc and b/app/modules/agent/engine/graphs/__pycache__/__init__.cpython-312.pyc differ diff --git a/app/modules/agent/engine/graphs/__pycache__/base_graph.cpython-312.pyc b/app/modules/agent/engine/graphs/__pycache__/base_graph.cpython-312.pyc index b8ac226..efb8c1f 100644 Binary files a/app/modules/agent/engine/graphs/__pycache__/base_graph.cpython-312.pyc and b/app/modules/agent/engine/graphs/__pycache__/base_graph.cpython-312.pyc differ diff --git a/app/modules/agent/engine/graphs/__pycache__/docs_graph_logic.cpython-312.pyc b/app/modules/agent/engine/graphs/__pycache__/docs_graph_logic.cpython-312.pyc index ce3f785..3cec23f 100644 Binary files a/app/modules/agent/engine/graphs/__pycache__/docs_graph_logic.cpython-312.pyc and b/app/modules/agent/engine/graphs/__pycache__/docs_graph_logic.cpython-312.pyc differ diff --git a/app/modules/agent/engine/graphs/__pycache__/project_edits_logic.cpython-312.pyc b/app/modules/agent/engine/graphs/__pycache__/project_edits_logic.cpython-312.pyc index 13de4c0..3209a87 100644 Binary files a/app/modules/agent/engine/graphs/__pycache__/project_edits_logic.cpython-312.pyc and b/app/modules/agent/engine/graphs/__pycache__/project_edits_logic.cpython-312.pyc differ diff --git a/app/modules/agent/engine/graphs/__pycache__/project_qa_graph.cpython-312.pyc b/app/modules/agent/engine/graphs/__pycache__/project_qa_graph.cpython-312.pyc index 5166f16..8a4c6ec 100644 Binary files a/app/modules/agent/engine/graphs/__pycache__/project_qa_graph.cpython-312.pyc and b/app/modules/agent/engine/graphs/__pycache__/project_qa_graph.cpython-312.pyc differ diff --git a/app/modules/agent/engine/graphs/__pycache__/project_qa_step_graphs.cpython-312.pyc b/app/modules/agent/engine/graphs/__pycache__/project_qa_step_graphs.cpython-312.pyc new file mode 100644 index 0000000..3a5d88e Binary files /dev/null and b/app/modules/agent/engine/graphs/__pycache__/project_qa_step_graphs.cpython-312.pyc differ diff --git a/app/modules/agent/engine/graphs/__pycache__/state.cpython-312.pyc b/app/modules/agent/engine/graphs/__pycache__/state.cpython-312.pyc index df2056d..79a799f 100644 Binary files a/app/modules/agent/engine/graphs/__pycache__/state.cpython-312.pyc and b/app/modules/agent/engine/graphs/__pycache__/state.cpython-312.pyc differ diff --git a/app/modules/agent/engine/graphs/base_graph.py b/app/modules/agent/engine/graphs/base_graph.py index b153ec9..c2021a2 100644 --- a/app/modules/agent/engine/graphs/base_graph.py +++ b/app/modules/agent/engine/graphs/base_graph.py @@ -59,7 +59,7 @@ class BaseGraphFactory: f"Confluence context:\n{conf}", ] ) - answer = self._llm.generate("general_answer", user_input) + answer = self._llm.generate("general_answer", user_input, log_context="graph.default.answer") emit_progress_sync( state, stage="graph.default.answer.done", diff --git a/app/modules/agent/engine/graphs/docs_graph_logic.py b/app/modules/agent/engine/graphs/docs_graph_logic.py index c1d310c..daf7947 100644 --- a/app/modules/agent/engine/graphs/docs_graph_logic.py +++ b/app/modules/agent/engine/graphs/docs_graph_logic.py @@ -52,7 +52,7 @@ class DocsContextAnalyzer: f"Detected documentation candidates:\n{snippets}", ] ) - raw = self._llm.generate("docs_detect", user_input) + raw = self._llm.generate("docs_detect", user_input, log_context="graph.docs.detect_existing_docs") exists = self.parse_bool_marker(raw, "exists", default=True) summary = self.parse_text_marker(raw, "summary", default="Documentation files detected.") return {"existing_docs_detected": exists, "existing_docs_summary": summary} @@ -71,7 +71,7 @@ class DocsContextAnalyzer: f"Existing docs summary:\n{state.get('existing_docs_summary', '')}", ] ) - raw = self._llm.generate("docs_strategy", user_input) + raw = self._llm.generate("docs_strategy", user_input, log_context="graph.docs.decide_strategy") strategy = self.parse_text_marker(raw, "strategy", default="").lower() if strategy not in {"incremental_update", "from_scratch"}: strategy = "incremental_update" if state.get("existing_docs_detected", False) else "from_scratch" @@ -260,7 +260,7 @@ class DocsContentComposer: f"Examples bundle:\n{state.get('rules_bundle', '')}", ] ) - plan = self._llm.generate("docs_plan_sections", user_input) + plan = self._llm.generate("docs_plan_sections", user_input, log_context="graph.docs.plan_incremental_changes") return { "doc_plan": plan, "target_path": target_path, @@ -279,7 +279,7 @@ class DocsContentComposer: f"Examples bundle:\n{state.get('rules_bundle', '')}", ] ) - plan = self._llm.generate("docs_plan_sections", user_input) + plan = self._llm.generate("docs_plan_sections", user_input, log_context="graph.docs.plan_new_document") return {"doc_plan": plan, "target_path": target_path, "target_file_content": "", "target_file_hash": ""} def generate_doc_content(self, state: AgentGraphState) -> dict: @@ -294,7 +294,7 @@ class DocsContentComposer: f"Examples bundle:\n{state.get('rules_bundle', '')}", ] ) - raw = self._llm.generate("docs_generation", user_input) + raw = self._llm.generate("docs_generation", user_input, log_context="graph.docs.generate_doc_content") bundle = self._bundle.parse_docs_bundle(raw) if bundle: first_content = str(bundle[0].get("content", "")).strip() @@ -369,7 +369,7 @@ class DocsContentComposer: f"Generated document:\n{generated}", ] ) - raw = self._llm.generate("docs_self_check", user_input) + raw = self._llm.generate("docs_self_check", user_input, log_context="graph.docs.self_check") passed = DocsContextAnalyzer.parse_bool_marker(raw, "pass", default=False) feedback = DocsContextAnalyzer.parse_text_marker(raw, "feedback", default="No validation feedback provided.") return {"validation_attempts": attempts, "validation_passed": passed, "validation_feedback": feedback} @@ -379,7 +379,7 @@ class DocsContentComposer: bundle = state.get("generated_docs_bundle", []) or [] strategy = state.get("docs_strategy", "from_scratch") if strategy == "from_scratch" and not self._bundle.bundle_has_required_structure(bundle): - LOGGER.warning( + LOGGER.info( "build_changeset fallback bundle used: strategy=%s bundle_items=%s", strategy, len(bundle), @@ -452,7 +452,11 @@ class DocsContentComposer: ] ) try: - summary = self._llm.generate("docs_execution_summary", user_input).strip() + summary = self._llm.generate( + "docs_execution_summary", + user_input, + log_context="graph.docs.summarize_result", + ).strip() except Exception: summary = "" if not summary: diff --git a/app/modules/agent/engine/graphs/project_edits_logic.py b/app/modules/agent/engine/graphs/project_edits_logic.py index b48245f..661e302 100644 --- a/app/modules/agent/engine/graphs/project_edits_logic.py +++ b/app/modules/agent/engine/graphs/project_edits_logic.py @@ -48,7 +48,9 @@ class ProjectEditsLogic: }, ensure_ascii=False, ) - parsed = self._support.parse_json(self._llm.generate("project_edits_plan", user_input)) + parsed = self._support.parse_json( + self._llm.generate("project_edits_plan", user_input, log_context="graph.project_edits.plan_changes") + ) contracts = self._contracts.parse( parsed, request=str(state.get("message", "")), @@ -165,7 +167,13 @@ class ProjectEditsLogic: "changeset": [{"op": x.op.value, "path": x.path, "reason": x.reason} for x in changeset[:20]], "rule": "Changes must stay inside contract blocks and not affect unrelated sections.", } - parsed = self._support.parse_json(self._llm.generate("project_edits_self_check", json.dumps(payload, ensure_ascii=False))) + parsed = self._support.parse_json( + self._llm.generate( + "project_edits_self_check", + json.dumps(payload, ensure_ascii=False), + log_context="graph.project_edits.self_check", + ) + ) passed = bool(parsed.get("pass")) if isinstance(parsed, dict) else False feedback = str(parsed.get("feedback", "")).strip() if isinstance(parsed, dict) else "" return { @@ -192,7 +200,11 @@ class ProjectEditsLogic: "rag_context": self._support.shorten(state.get("rag_context", ""), 5000), "confluence_context": self._support.shorten(state.get("confluence_context", ""), 5000), } - raw = self._llm.generate("project_edits_hunks", json.dumps(prompt_payload, ensure_ascii=False)) + raw = self._llm.generate( + "project_edits_hunks", + json.dumps(prompt_payload, ensure_ascii=False), + log_context="graph.project_edits.generate_changeset", + ) parsed = self._support.parse_json(raw) hunks = parsed.get("hunks", []) if isinstance(parsed, dict) else [] if not isinstance(hunks, list) or not hunks: diff --git a/app/modules/agent/engine/graphs/project_qa_graph.py b/app/modules/agent/engine/graphs/project_qa_graph.py index 6dead1d..1f6f005 100644 --- a/app/modules/agent/engine/graphs/project_qa_graph.py +++ b/app/modules/agent/engine/graphs/project_qa_graph.py @@ -33,7 +33,7 @@ class ProjectQaGraphFactory: f"Confluence context:\n{state.get('confluence_context', '')}", ] ) - answer = self._llm.generate("project_answer", user_input) + answer = self._llm.generate("project_answer", user_input, log_context="graph.project_qa.answer") emit_progress_sync( state, stage="graph.project_qa.answer.done", diff --git a/app/modules/agent/engine/graphs/project_qa_step_graphs.py b/app/modules/agent/engine/graphs/project_qa_step_graphs.py new file mode 100644 index 0000000..f8059bc --- /dev/null +++ b/app/modules/agent/engine/graphs/project_qa_step_graphs.py @@ -0,0 +1,172 @@ +from __future__ import annotations +import logging + +from langgraph.graph import END, START, StateGraph + +from app.modules.agent.engine.graphs.progress import emit_progress_sync +from app.modules.agent.engine.graphs.state import AgentGraphState +from app.modules.agent.engine.orchestrator.actions.project_qa_analyzer import ProjectQaAnalyzer +from app.modules.agent.engine.orchestrator.actions.project_qa_support import ProjectQaSupport +from app.modules.agent.llm import AgentLlmService +from app.modules.contracts import RagRetriever +from app.modules.rag.explain import ExplainPack, PromptBudgeter + +LOGGER = logging.getLogger(__name__) + + +class ProjectQaConversationGraphFactory: + def __init__(self, llm: AgentLlmService | None = None) -> None: + self._support = ProjectQaSupport() + + def build(self, checkpointer=None): + graph = StateGraph(AgentGraphState) + graph.add_node("resolve_request", self._resolve_request) + graph.add_edge(START, "resolve_request") + graph.add_edge("resolve_request", END) + return graph.compile(checkpointer=checkpointer) + + def _resolve_request(self, state: AgentGraphState) -> dict: + emit_progress_sync(state, stage="graph.project_qa.conversation_understanding", message="Нормализую пользовательский запрос.") + resolved = self._support.resolve_request(str(state.get("message", "") or "")) + LOGGER.warning("graph step result: graph=project_qa/conversation_understanding normalized=%s", resolved.get("normalized_message", "")) + return {"resolved_request": resolved} + + +class ProjectQaClassificationGraphFactory: + def __init__(self, llm: AgentLlmService | None = None) -> None: + self._support = ProjectQaSupport() + + def build(self, checkpointer=None): + graph = StateGraph(AgentGraphState) + graph.add_node("classify_question", self._classify_question) + graph.add_edge(START, "classify_question") + graph.add_edge("classify_question", END) + return graph.compile(checkpointer=checkpointer) + + def _classify_question(self, state: AgentGraphState) -> dict: + resolved = state.get("resolved_request", {}) or {} + message = str(resolved.get("normalized_message") or state.get("message", "") or "") + profile = self._support.build_profile(message) + LOGGER.warning("graph step result: graph=project_qa/question_classification domain=%s intent=%s", profile.get("domain"), profile.get("intent")) + return {"question_profile": profile} + + +class ProjectQaRetrievalGraphFactory: + def __init__(self, rag: RagRetriever, llm: AgentLlmService | None = None) -> None: + self._rag = rag + self._support = ProjectQaSupport() + + def build(self, checkpointer=None): + graph = StateGraph(AgentGraphState) + graph.add_node("retrieve_context", self._retrieve_context) + graph.add_edge(START, "retrieve_context") + graph.add_edge("retrieve_context", END) + return graph.compile(checkpointer=checkpointer) + + def _retrieve_context(self, state: AgentGraphState) -> dict: + emit_progress_sync(state, stage="graph.project_qa.context_retrieval", message="Собираю контекст по проекту.") + resolved = state.get("resolved_request", {}) or {} + profile = state.get("question_profile", {}) or {} + files_map = dict(state.get("files_map", {}) or {}) + rag_items: list[dict] = [] + source_bundle = self._support.build_source_bundle(profile, list(rag_items), files_map) + LOGGER.warning( + "graph step result: graph=project_qa/context_retrieval mode=%s rag_items=%s file_candidates=%s legacy_rag=%s", + profile.get("domain"), + len(source_bundle.get("rag_items", []) or []), + len(source_bundle.get("file_candidates", []) or []), + False, + ) + return {"source_bundle": source_bundle} + + +class ProjectQaAnalysisGraphFactory: + def __init__(self, llm: AgentLlmService | None = None) -> None: + self._support = ProjectQaSupport() + self._analyzer = ProjectQaAnalyzer() + + def build(self, checkpointer=None): + graph = StateGraph(AgentGraphState) + graph.add_node("analyze_context", self._analyze_context) + graph.add_edge(START, "analyze_context") + graph.add_edge("analyze_context", END) + return graph.compile(checkpointer=checkpointer) + + def _analyze_context(self, state: AgentGraphState) -> dict: + explain_pack = state.get("explain_pack") + if explain_pack: + analysis = self._analysis_from_pack(explain_pack) + LOGGER.warning( + "graph step result: graph=project_qa/context_analysis findings=%s evidence=%s", + len(analysis.get("findings", []) or []), + len(analysis.get("evidence", []) or []), + ) + return {"analysis_brief": analysis} + bundle = state.get("source_bundle", {}) or {} + profile = bundle.get("profile", {}) or state.get("question_profile", {}) or {} + rag_items = list(bundle.get("rag_items", []) or []) + file_candidates = list(bundle.get("file_candidates", []) or []) + analysis = self._analyzer.analyze_code(profile, rag_items, file_candidates) if str(profile.get("domain")) == "code" else self._analyzer.analyze_docs(profile, rag_items) + LOGGER.warning( + "graph step result: graph=project_qa/context_analysis findings=%s evidence=%s", + len(analysis.get("findings", []) or []), + len(analysis.get("evidence", []) or []), + ) + return {"analysis_brief": analysis} + + def _analysis_from_pack(self, raw_pack) -> dict: + pack = ExplainPack.model_validate(raw_pack) + findings: list[str] = [] + evidence: list[str] = [] + for entrypoint in pack.selected_entrypoints[:3]: + findings.append(f"Entrypoint `{entrypoint.title}` maps to handler `{entrypoint.metadata.get('handler_symbol_id', '')}`.") + if entrypoint.source: + evidence.append(entrypoint.source) + for path in pack.trace_paths[:3]: + if path.symbol_ids: + findings.append(f"Trace path: {' -> '.join(path.symbol_ids)}") + for excerpt in pack.code_excerpts[:4]: + evidence.append(f"{excerpt.path}:{excerpt.start_line}-{excerpt.end_line} [{excerpt.evidence_id}]") + return { + "subject": pack.intent.normalized_query, + "findings": findings or ["No explain trace was built from the available code evidence."], + "evidence": evidence, + "gaps": list(pack.missing), + "answer_mode": "summary", + } + + +class ProjectQaAnswerGraphFactory: + def __init__(self, llm: AgentLlmService | None = None) -> None: + self._support = ProjectQaSupport() + self._llm = llm + self._budgeter = PromptBudgeter() + + def build(self, checkpointer=None): + graph = StateGraph(AgentGraphState) + graph.add_node("compose_answer", self._compose_answer) + graph.add_edge(START, "compose_answer") + graph.add_edge("compose_answer", END) + return graph.compile(checkpointer=checkpointer) + + def _compose_answer(self, state: AgentGraphState) -> dict: + profile = state.get("question_profile", {}) or {} + analysis = state.get("analysis_brief", {}) or {} + brief = self._support.build_answer_brief(profile, analysis) + explain_pack = state.get("explain_pack") + answer = self._compose_explain_answer(state, explain_pack) + if not answer: + answer = self._support.compose_answer(brief) + LOGGER.warning("graph step result: graph=project_qa/answer_composition answer_len=%s", len(answer or "")) + return {"answer_brief": brief, "final_answer": answer} + + def _compose_explain_answer(self, state: AgentGraphState, raw_pack) -> str: + if raw_pack is None or self._llm is None: + return "" + pack = ExplainPack.model_validate(raw_pack) + prompt_input = self._budgeter.build_prompt_input(str(state.get("message", "") or ""), pack) + return self._llm.generate( + "code_explain_answer_v2", + prompt_input, + log_context="graph.project_qa.answer_v2", + ).strip() diff --git a/app/modules/agent/engine/graphs/state.py b/app/modules/agent/engine/graphs/state.py index 8492114..72e297c 100644 --- a/app/modules/agent/engine/graphs/state.py +++ b/app/modules/agent/engine/graphs/state.py @@ -25,6 +25,12 @@ class AgentGraphState(TypedDict, total=False): validation_passed: bool validation_feedback: str validation_attempts: int + resolved_request: dict + question_profile: dict + source_bundle: dict + analysis_brief: dict + answer_brief: dict + final_answer: str answer: str changeset: list[ChangeItem] edits_requested_path: str diff --git a/app/modules/agent/engine/orchestrator/__pycache__/execution_engine.cpython-312.pyc b/app/modules/agent/engine/orchestrator/__pycache__/execution_engine.cpython-312.pyc index 48a46a2..bbbbd32 100644 Binary files a/app/modules/agent/engine/orchestrator/__pycache__/execution_engine.cpython-312.pyc and b/app/modules/agent/engine/orchestrator/__pycache__/execution_engine.cpython-312.pyc differ diff --git a/app/modules/agent/engine/orchestrator/__pycache__/service.cpython-312.pyc b/app/modules/agent/engine/orchestrator/__pycache__/service.cpython-312.pyc index 91afc9e..d2ae4ff 100644 Binary files a/app/modules/agent/engine/orchestrator/__pycache__/service.cpython-312.pyc and b/app/modules/agent/engine/orchestrator/__pycache__/service.cpython-312.pyc differ diff --git a/app/modules/agent/engine/orchestrator/__pycache__/step_registry.cpython-312.pyc b/app/modules/agent/engine/orchestrator/__pycache__/step_registry.cpython-312.pyc index 83b7e29..75d92fb 100644 Binary files a/app/modules/agent/engine/orchestrator/__pycache__/step_registry.cpython-312.pyc and b/app/modules/agent/engine/orchestrator/__pycache__/step_registry.cpython-312.pyc differ diff --git a/app/modules/agent/engine/orchestrator/__pycache__/template_registry.cpython-312.pyc b/app/modules/agent/engine/orchestrator/__pycache__/template_registry.cpython-312.pyc index 0a99665..f8e4f59 100644 Binary files a/app/modules/agent/engine/orchestrator/__pycache__/template_registry.cpython-312.pyc and b/app/modules/agent/engine/orchestrator/__pycache__/template_registry.cpython-312.pyc differ diff --git a/app/modules/agent/engine/orchestrator/actions/__init__.py b/app/modules/agent/engine/orchestrator/actions/__init__.py index e220f9b..ab47e51 100644 --- a/app/modules/agent/engine/orchestrator/actions/__init__.py +++ b/app/modules/agent/engine/orchestrator/actions/__init__.py @@ -1,13 +1,17 @@ +from app.modules.agent.engine.orchestrator.actions.code_explain_actions import CodeExplainActions from app.modules.agent.engine.orchestrator.actions.docs_actions import DocsActions from app.modules.agent.engine.orchestrator.actions.edit_actions import EditActions from app.modules.agent.engine.orchestrator.actions.explain_actions import ExplainActions from app.modules.agent.engine.orchestrator.actions.gherkin_actions import GherkinActions +from app.modules.agent.engine.orchestrator.actions.project_qa_actions import ProjectQaActions from app.modules.agent.engine.orchestrator.actions.review_actions import ReviewActions __all__ = [ + "CodeExplainActions", "DocsActions", "EditActions", "ExplainActions", "GherkinActions", + "ProjectQaActions", "ReviewActions", ] diff --git a/app/modules/agent/engine/orchestrator/actions/__pycache__/__init__.cpython-312.pyc b/app/modules/agent/engine/orchestrator/actions/__pycache__/__init__.cpython-312.pyc index 6960565..265f766 100644 Binary files a/app/modules/agent/engine/orchestrator/actions/__pycache__/__init__.cpython-312.pyc and b/app/modules/agent/engine/orchestrator/actions/__pycache__/__init__.cpython-312.pyc differ diff --git a/app/modules/agent/engine/orchestrator/actions/__pycache__/code_explain_actions.cpython-312.pyc b/app/modules/agent/engine/orchestrator/actions/__pycache__/code_explain_actions.cpython-312.pyc new file mode 100644 index 0000000..995dd53 Binary files /dev/null and b/app/modules/agent/engine/orchestrator/actions/__pycache__/code_explain_actions.cpython-312.pyc differ diff --git a/app/modules/agent/engine/orchestrator/actions/__pycache__/project_qa_actions.cpython-312.pyc b/app/modules/agent/engine/orchestrator/actions/__pycache__/project_qa_actions.cpython-312.pyc new file mode 100644 index 0000000..0a35d84 Binary files /dev/null and b/app/modules/agent/engine/orchestrator/actions/__pycache__/project_qa_actions.cpython-312.pyc differ diff --git a/app/modules/agent/engine/orchestrator/actions/__pycache__/project_qa_analyzer.cpython-312.pyc b/app/modules/agent/engine/orchestrator/actions/__pycache__/project_qa_analyzer.cpython-312.pyc new file mode 100644 index 0000000..551195d Binary files /dev/null and b/app/modules/agent/engine/orchestrator/actions/__pycache__/project_qa_analyzer.cpython-312.pyc differ diff --git a/app/modules/agent/engine/orchestrator/actions/__pycache__/project_qa_support.cpython-312.pyc b/app/modules/agent/engine/orchestrator/actions/__pycache__/project_qa_support.cpython-312.pyc new file mode 100644 index 0000000..c6671b3 Binary files /dev/null and b/app/modules/agent/engine/orchestrator/actions/__pycache__/project_qa_support.cpython-312.pyc differ diff --git a/app/modules/agent/engine/orchestrator/actions/code_explain_actions.py b/app/modules/agent/engine/orchestrator/actions/code_explain_actions.py new file mode 100644 index 0000000..e94535a --- /dev/null +++ b/app/modules/agent/engine/orchestrator/actions/code_explain_actions.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import logging + +from typing import TYPE_CHECKING + +from app.modules.agent.engine.orchestrator.actions.common import ActionSupport +from app.modules.agent.engine.orchestrator.execution_context import ExecutionContext +from app.modules.agent.engine.orchestrator.models import ArtifactType +from app.modules.rag.explain.intent_builder import ExplainIntentBuilder +from app.modules.rag.explain.models import ExplainPack + +if TYPE_CHECKING: + from app.modules.rag.explain.retriever_v2 import CodeExplainRetrieverV2 + +LOGGER = logging.getLogger(__name__) + + +class CodeExplainActions(ActionSupport): + def __init__(self, retriever: CodeExplainRetrieverV2 | None = None) -> None: + self._retriever = retriever + self._intent_builder = ExplainIntentBuilder() + + def build_code_explain_pack(self, ctx: ExecutionContext) -> list[str]: + file_candidates = list((self.get(ctx, "source_bundle", {}) or {}).get("file_candidates", []) or []) + if self._retriever is None: + pack = ExplainPack( + intent=self._intent_builder.build(ctx.task.user_message), + missing=["code_explain_retriever_unavailable"], + ) + else: + pack = self._retriever.build_pack( + ctx.task.rag_session_id, + ctx.task.user_message, + file_candidates=file_candidates, + ) + LOGGER.warning( + "code explain action: task_id=%s entrypoints=%s seeds=%s paths=%s excerpts=%s missing=%s", + ctx.task.task_id, + len(pack.selected_entrypoints), + len(pack.seed_symbols), + len(pack.trace_paths), + len(pack.code_excerpts), + pack.missing, + ) + return [self.put(ctx, "explain_pack", ArtifactType.STRUCTURED_JSON, pack.model_dump(mode="json"))] diff --git a/app/modules/agent/engine/orchestrator/actions/project_qa_actions.py b/app/modules/agent/engine/orchestrator/actions/project_qa_actions.py new file mode 100644 index 0000000..570680d --- /dev/null +++ b/app/modules/agent/engine/orchestrator/actions/project_qa_actions.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +from app.modules.agent.engine.orchestrator.actions.project_qa_analyzer import ProjectQaAnalyzer +from app.modules.agent.engine.orchestrator.actions.common import ActionSupport +from app.modules.agent.engine.orchestrator.actions.project_qa_support import ProjectQaSupport +from app.modules.agent.engine.orchestrator.execution_context import ExecutionContext +from app.modules.agent.engine.orchestrator.models import ArtifactType + + +class ProjectQaActions(ActionSupport): + def __init__(self) -> None: + self._support = ProjectQaSupport() + self._analyzer = ProjectQaAnalyzer() + + def classify_project_question(self, ctx: ExecutionContext) -> list[str]: + message = str(ctx.task.user_message or "") + profile = self._support.build_profile(message) + return [self.put(ctx, "question_profile", ArtifactType.STRUCTURED_JSON, profile)] + + def collect_project_sources(self, ctx: ExecutionContext) -> list[str]: + profile = self.get(ctx, "question_profile", {}) or {} + terms = list(profile.get("terms", []) or []) + entities = list(profile.get("entities", []) or []) + rag_items = list(ctx.task.metadata.get("rag_items", []) or []) + files_map = dict(ctx.task.metadata.get("files_map", {}) or {}) + explicit_test = any(term in {"test", "tests", "тест", "тесты"} for term in terms) + + ranked_rag = [] + for item in rag_items: + score = self._support.rag_score(item, terms, entities) + source = str(item.get("source", "") or "") + if not explicit_test and self._support.is_test_path(source): + score -= 3 + if score > 0: + ranked_rag.append((score, item)) + ranked_rag.sort(key=lambda pair: pair[0], reverse=True) + + ranked_files = [] + for path, payload in files_map.items(): + score = self._support.file_score(path, payload, terms, entities) + if not explicit_test and self._support.is_test_path(path): + score -= 3 + if score > 0: + ranked_files.append( + ( + score, + { + "path": path, + "content": str(payload.get("content", "")), + "content_hash": str(payload.get("content_hash", "")), + }, + ) + ) + ranked_files.sort(key=lambda pair: pair[0], reverse=True) + + bundle = { + "profile": profile, + "rag_items": [item for _, item in ranked_rag[:12]], + "file_candidates": [item for _, item in ranked_files[:10]], + "rag_total": len(ranked_rag), + "files_total": len(ranked_files), + } + return [self.put(ctx, "source_bundle", ArtifactType.STRUCTURED_JSON, bundle)] + + def analyze_project_sources(self, ctx: ExecutionContext) -> list[str]: + bundle = self.get(ctx, "source_bundle", {}) or {} + profile = bundle.get("profile", {}) or {} + rag_items = list(bundle.get("rag_items", []) or []) + file_candidates = list(bundle.get("file_candidates", []) or []) + + if str(profile.get("domain")) == "code": + analysis = self._analyzer.analyze_code(profile, rag_items, file_candidates) + else: + analysis = self._analyzer.analyze_docs(profile, rag_items) + return [self.put(ctx, "analysis_brief", ArtifactType.STRUCTURED_JSON, analysis)] + + def build_project_answer_brief(self, ctx: ExecutionContext) -> list[str]: + profile = self.get(ctx, "question_profile", {}) or {} + analysis = self.get(ctx, "analysis_brief", {}) or {} + brief = { + "question_profile": profile, + "resolved_subject": analysis.get("subject"), + "key_findings": analysis.get("findings", []), + "supporting_evidence": analysis.get("evidence", []), + "missing_evidence": analysis.get("gaps", []), + "answer_mode": analysis.get("answer_mode", "summary"), + } + return [self.put(ctx, "answer_brief", ArtifactType.STRUCTURED_JSON, brief)] + + def compose_project_answer(self, ctx: ExecutionContext) -> list[str]: + brief = self.get(ctx, "answer_brief", {}) or {} + profile = brief.get("question_profile", {}) or {} + russian = bool(profile.get("russian")) + answer_mode = str(brief.get("answer_mode") or "summary") + findings = list(brief.get("key_findings", []) or []) + evidence = list(brief.get("supporting_evidence", []) or []) + gaps = list(brief.get("missing_evidence", []) or []) + + title = "## Кратко" if russian else "## Summary" + lines = [title] + if answer_mode == "inventory": + lines.append("### Что реализовано" if russian else "### Implemented items") + else: + lines.append("### Что видно по проекту" if russian else "### What the project shows") + if findings: + lines.extend(f"- {item}" for item in findings) + else: + lines.append("Не удалось собрать подтвержденные выводы по доступным данным." if russian else "No supported findings could be assembled from the available data.") + if evidence: + lines.append("") + lines.append("### Где смотреть в проекте" if russian else "### Where to look in the project") + lines.extend(f"- `{item}`" for item in evidence[:5]) + if gaps: + lines.append("") + lines.append("### Что пока не подтверждено кодом" if russian else "### What is not yet confirmed in code") + lines.extend(f"- {item}" for item in gaps[:3]) + return [self.put(ctx, "final_answer", ArtifactType.TEXT, "\n".join(lines))] diff --git a/app/modules/agent/engine/orchestrator/actions/project_qa_analyzer.py b/app/modules/agent/engine/orchestrator/actions/project_qa_analyzer.py new file mode 100644 index 0000000..0374f52 --- /dev/null +++ b/app/modules/agent/engine/orchestrator/actions/project_qa_analyzer.py @@ -0,0 +1,154 @@ +from __future__ import annotations + + +class ProjectQaAnalyzer: + def analyze_code(self, profile: dict, rag_items: list[dict], file_candidates: list[dict]) -> dict: + terms = list(profile.get("terms", []) or []) + intent = str(profile.get("intent") or "lookup") + russian = bool(profile.get("russian")) + findings: list[str] = [] + evidence: list[str] = [] + gaps: list[str] = [] + + symbol_titles = [str(item.get("title", "") or "") for item in rag_items if str(item.get("layer", "")).startswith("C1")] + symbol_set = set(symbol_titles) + file_paths = [str(item.get("path", "") or item.get("source", "") or "") for item in rag_items] + file_paths.extend(str(item.get("path", "") or "") for item in file_candidates) + + if "ConfigManager" in profile.get("entities", []) or "configmanager" in terms or "config_manager" in terms: + alias_file = self.find_path(file_paths, "src/config_manager/__init__.py") + if alias_file: + findings.append( + "Публичный `ConfigManager` экспортируется из `src/config_manager/__init__.py` как alias на `ConfigManagerV2`." + if russian + else "Public `ConfigManager` is exported from `src/config_manager/__init__.py` as an alias to `ConfigManagerV2`." + ) + evidence.append("src/config_manager/__init__.py") + + if "controlchannel" in {name.lower() for name in symbol_set}: + findings.append( + "Базовый контракт управления задает `ControlChannel`: он определяет команды `start` и `stop` для внешнего канала управления." + if russian + else "`ControlChannel` defines the base management contract with `start` and `stop` commands." + ) + evidence.append("src/config_manager/v2/control/base.py") + + if "ControlChannelBridge" in symbol_set: + findings.append( + "`ControlChannelBridge` связывает внешний канал управления с lifecycle-методами менеджера: `on_start`, `on_stop`, `on_status`." + if russian + else "`ControlChannelBridge` connects the external control channel to manager lifecycle methods: `on_start`, `on_stop`, `on_status`." + ) + evidence.append("src/config_manager/v2/core/control_bridge.py") + + implementation_files = self.find_management_implementations(file_candidates) + if implementation_files: + labels = ", ".join(f"`{path}`" for path in implementation_files) + channel_names = self.implementation_names(implementation_files) + findings.append( + f"В коде найдены конкретные реализации каналов управления: {', '.join(channel_names)} ({labels})." + if russian + else f"Concrete management channel implementations were found in code: {', '.join(channel_names)} ({labels})." + ) + evidence.extend(implementation_files) + elif intent == "inventory": + gaps.append( + "В текущем контексте не удалось уверенно подтвердить конкретные файлы-реализации каналов, кроме базового контракта и bridge-слоя." + if russian + else "The current context does not yet confirm concrete channel implementation files beyond the base contract and bridge layer." + ) + + package_doc = self.find_management_doc(file_candidates) + if package_doc: + findings.append( + f"Пакет управления прямо описывает внешние каналы через `{package_doc}`." + if russian + else f"The control package directly describes external channels in `{package_doc}`." + ) + evidence.append(package_doc) + + subject = "management channels" + if profile.get("entities"): + subject = ", ".join(profile["entities"]) + return { + "subject": subject, + "findings": self.dedupe(findings), + "evidence": self.dedupe(evidence), + "gaps": gaps, + "answer_mode": "inventory" if intent == "inventory" else "summary", + } + + def analyze_docs(self, profile: dict, rag_items: list[dict]) -> dict: + findings: list[str] = [] + evidence: list[str] = [] + for item in rag_items[:5]: + title = str(item.get("title", "") or "") + source = str(item.get("source", "") or "") + content = str(item.get("content", "") or "").strip() + if content: + findings.append(content.splitlines()[0][:220]) + if source: + evidence.append(source) + elif title: + evidence.append(title) + return { + "subject": "docs", + "findings": self.dedupe(findings), + "evidence": self.dedupe(evidence), + "gaps": [] if findings else ["Недостаточно данных в документации." if profile.get("russian") else "Not enough data in documentation."], + "answer_mode": "summary", + } + + def find_management_implementations(self, file_candidates: list[dict]) -> list[str]: + found: list[str] = [] + for item in file_candidates: + path = str(item.get("path", "") or "") + lowered = path.lower() + if self.is_test_path(path): + continue + if any(token in lowered for token in ("http_channel.py", "telegram.py", "telegram_channel.py", "http.py")): + found.append(path) + continue + content = str(item.get("content", "") or "").lower() + if "controlchannel" in content and "class " in content: + found.append(path) + continue + if ("channel" in lowered or "control" in lowered) and any(token in content for token in ("http", "telegram", "bot")): + found.append(path) + return self.dedupe(found)[:4] + + def implementation_names(self, paths: list[str]) -> list[str]: + names: list[str] = [] + for path in paths: + stem = path.rsplit("/", 1)[-1].rsplit(".", 1)[0] + label = stem.replace("_", " ").strip() + if label and label not in names: + names.append(label) + return names + + def find_management_doc(self, file_candidates: list[dict]) -> str | None: + for item in file_candidates: + path = str(item.get("path", "") or "") + if self.is_test_path(path): + continue + content = str(item.get("content", "") or "").lower() + if any(token in content for token in ("каналы внешнего управления", "external control channels", "http api", "telegram")): + return path + return None + + def find_path(self, paths: list[str], target: str) -> str | None: + for path in paths: + if path == target: + return path + return None + + def dedupe(self, items: list[str]) -> list[str]: + seen: list[str] = [] + for item in items: + if item and item not in seen: + seen.append(item) + return seen + + def is_test_path(self, path: str) -> bool: + lowered = path.lower() + return lowered.startswith("tests/") or "/tests/" in lowered or lowered.startswith("test_") or "/test_" in lowered diff --git a/app/modules/agent/engine/orchestrator/actions/project_qa_support.py b/app/modules/agent/engine/orchestrator/actions/project_qa_support.py new file mode 100644 index 0000000..d449430 --- /dev/null +++ b/app/modules/agent/engine/orchestrator/actions/project_qa_support.py @@ -0,0 +1,166 @@ +from __future__ import annotations + +import re + +from app.modules.rag.retrieval.query_terms import extract_query_terms + + +class ProjectQaSupport: + def resolve_request(self, message: str) -> dict: + profile = self.build_profile(message) + subject = profile["entities"][0] if profile.get("entities") else "" + return { + "original_message": message, + "normalized_message": " ".join((message or "").split()), + "subject_hint": subject, + "source_hint": profile["domain"], + "russian": profile["russian"], + } + + def build_profile(self, message: str) -> dict: + lowered = message.lower() + return { + "domain": "code" if self.looks_like_code_question(lowered) else "docs", + "intent": self.detect_intent(lowered), + "terms": extract_query_terms(message), + "entities": self.extract_entities(message), + "russian": self.is_russian(message), + } + + def build_retrieval_query(self, resolved_request: dict, profile: dict) -> str: + normalized = str(resolved_request.get("normalized_message") or resolved_request.get("original_message") or "").strip() + if profile.get("domain") == "code" and "по коду" not in normalized.lower(): + return f"по коду {normalized}".strip() + return normalized + + def build_source_bundle(self, profile: dict, rag_items: list[dict], files_map: dict[str, dict]) -> dict: + terms = list(profile.get("terms", []) or []) + entities = list(profile.get("entities", []) or []) + explicit_test = any(term in {"test", "tests", "тест", "тесты"} for term in terms) + + ranked_rag: list[tuple[int, dict]] = [] + for item in rag_items: + score = self.rag_score(item, terms, entities) + source = str(item.get("source", "") or "") + if not explicit_test and self.is_test_path(source): + score -= 3 + if score > 0: + ranked_rag.append((score, item)) + ranked_rag.sort(key=lambda pair: pair[0], reverse=True) + + ranked_files: list[tuple[int, dict]] = [] + for path, payload in files_map.items(): + score = self.file_score(path, payload, terms, entities) + if not explicit_test and self.is_test_path(path): + score -= 3 + if score > 0: + ranked_files.append( + ( + score, + { + "path": path, + "content": str(payload.get("content", "")), + "content_hash": str(payload.get("content_hash", "")), + }, + ) + ) + ranked_files.sort(key=lambda pair: pair[0], reverse=True) + + return { + "profile": profile, + "rag_items": [item for _, item in ranked_rag[:12]], + "file_candidates": [item for _, item in ranked_files[:10]], + "rag_total": len(ranked_rag), + "files_total": len(ranked_files), + } + + def build_answer_brief(self, profile: dict, analysis: dict) -> dict: + return { + "question_profile": profile, + "resolved_subject": analysis.get("subject"), + "key_findings": analysis.get("findings", []), + "supporting_evidence": analysis.get("evidence", []), + "missing_evidence": analysis.get("gaps", []), + "answer_mode": analysis.get("answer_mode", "summary"), + } + + def compose_answer(self, brief: dict) -> str: + profile = brief.get("question_profile", {}) or {} + russian = bool(profile.get("russian")) + answer_mode = str(brief.get("answer_mode") or "summary") + findings = list(brief.get("key_findings", []) or []) + evidence = list(brief.get("supporting_evidence", []) or []) + gaps = list(brief.get("missing_evidence", []) or []) + + title = "## Кратко" if russian else "## Summary" + lines = [title] + lines.append("### Что реализовано" if answer_mode == "inventory" and russian else "### Implemented items" if answer_mode == "inventory" else "### Что видно по проекту" if russian else "### What the project shows") + if findings: + lines.extend(f"- {item}" for item in findings) + else: + lines.append("Не удалось собрать подтвержденные выводы по доступным данным." if russian else "No supported findings could be assembled from the available data.") + if evidence: + lines.append("") + lines.append("### Где смотреть в проекте" if russian else "### Where to look in the project") + lines.extend(f"- `{item}`" for item in evidence[:5]) + if gaps: + lines.append("") + lines.append("### Что пока не подтверждено кодом" if russian else "### What is not yet confirmed in code") + lines.extend(f"- {item}" for item in gaps[:3]) + return "\n".join(lines) + + def detect_intent(self, lowered: str) -> str: + if any(token in lowered for token in ("какие", "что уже реализ", "список", "перечень", "какие есть")): + return "inventory" + if any(token in lowered for token in ("где", "find", "where")): + return "lookup" + if any(token in lowered for token in ("сравни", "compare")): + return "compare" + return "explain" + + def looks_like_code_question(self, lowered: str) -> bool: + code_markers = ("по коду", "код", "реализ", "имплементац", "класс", "метод", "модул", "файл", "канал", "handler", "endpoint") + return any(marker in lowered for marker in code_markers) or bool(re.search(r"\b[A-Z][A-Za-z0-9_]{2,}\b", lowered)) + + def extract_entities(self, message: str) -> list[str]: + return re.findall(r"\b[A-Z][A-Za-z0-9_]{2,}\b", message)[:5] + + def rag_score(self, item: dict, terms: list[str], entities: list[str]) -> int: + haystacks = [ + str(item.get("source", "") or "").lower(), + str(item.get("title", "") or "").lower(), + str(item.get("content", "") or "").lower(), + str((item.get("metadata", {}) or {}).get("qname", "") or "").lower(), + ] + score = 0 + for term in terms: + if any(term in hay for hay in haystacks): + score += 3 + for entity in entities: + if any(entity.lower() in hay for hay in haystacks): + score += 5 + return score + + def file_score(self, path: str, payload: dict, terms: list[str], entities: list[str]) -> int: + content = str(payload.get("content", "") or "").lower() + path_lower = path.lower() + score = 0 + for term in terms: + if term in path_lower: + score += 4 + elif term in content: + score += 2 + for entity in entities: + entity_lower = entity.lower() + if entity_lower in path_lower: + score += 5 + elif entity_lower in content: + score += 3 + return score + + def is_test_path(self, path: str) -> bool: + lowered = path.lower() + return lowered.startswith("tests/") or "/tests/" in lowered or lowered.startswith("test_") or "/test_" in lowered + + def is_russian(self, text: str) -> bool: + return any("а" <= ch.lower() <= "я" or ch.lower() == "ё" for ch in text) diff --git a/app/modules/agent/engine/orchestrator/execution_engine.py b/app/modules/agent/engine/orchestrator/execution_engine.py index 5d87aab..e747fea 100644 --- a/app/modules/agent/engine/orchestrator/execution_engine.py +++ b/app/modules/agent/engine/orchestrator/execution_engine.py @@ -2,6 +2,7 @@ from __future__ import annotations import asyncio import inspect +import logging import time from app.modules.agent.engine.orchestrator.execution_context import ExecutionContext @@ -9,6 +10,8 @@ from app.modules.agent.engine.orchestrator.models import PlanStatus, PlanStep, S from app.modules.agent.engine.orchestrator.quality_gates import QualityGateRunner from app.modules.agent.engine.orchestrator.step_registry import StepRegistry +LOGGER = logging.getLogger(__name__) + class ExecutionEngine: def __init__(self, step_registry: StepRegistry, gates: QualityGateRunner) -> None: @@ -22,17 +25,18 @@ class ExecutionEngine: for step in ctx.plan.steps: dep_issue = self._dependency_issue(step, step_results) if dep_issue: - step_results.append( - StepResult( + result = StepResult( step_id=step.step_id, status=StepStatus.SKIPPED, warnings=[dep_issue], ) - ) + step_results.append(result) + self._log_step_result(ctx, step, result) continue result = await self._run_with_retry(step, ctx) step_results.append(result) + self._log_step_result(ctx, step, result) if result.status in {StepStatus.FAILED, StepStatus.RETRY_EXHAUSTED} and step.on_failure == "fail": ctx.plan.status = PlanStatus.FAILED return step_results @@ -65,6 +69,15 @@ class ExecutionEngine: while attempt < max_attempts: attempt += 1 started_at = time.monotonic() + LOGGER.warning( + "orchestrator step start: task_id=%s step_id=%s action_id=%s executor=%s attempt=%s graph_id=%s", + ctx.task.task_id, + step.step_id, + step.action_id, + step.executor, + attempt, + step.graph_id or "", + ) await self._emit_progress(ctx, f"orchestrator.step.{step.step_id}", step.title) try: @@ -113,3 +126,21 @@ class ExecutionEngine: result = ctx.progress_cb(stage, message, "task_progress", {"layer": "orchestrator"}) if inspect.isawaitable(result): await result + + def _log_step_result(self, ctx: ExecutionContext, step: PlanStep, result: StepResult) -> None: + artifact_keys = [] + for artifact_id in result.produced_artifact_ids: + item = next((artifact for artifact in ctx.artifacts.all_items() if artifact.artifact_id == artifact_id), None) + if item is not None: + artifact_keys.append(item.key) + LOGGER.warning( + "orchestrator step result: task_id=%s step_id=%s action_id=%s status=%s duration_ms=%s artifact_keys=%s warnings=%s error=%s", + ctx.task.task_id, + step.step_id, + step.action_id, + result.status.value, + result.duration_ms, + artifact_keys, + result.warnings, + result.error_message or "", + ) diff --git a/app/modules/agent/engine/orchestrator/service.py b/app/modules/agent/engine/orchestrator/service.py index 06227d0..7b7ace2 100644 --- a/app/modules/agent/engine/orchestrator/service.py +++ b/app/modules/agent/engine/orchestrator/service.py @@ -1,6 +1,7 @@ from __future__ import annotations import inspect +import logging from app.core.exceptions import AppError from app.modules.agent.engine.orchestrator.execution_context import ExecutionContext, GraphInvoker, GraphResolver, ProgressCallback @@ -14,6 +15,8 @@ from app.modules.agent.engine.orchestrator.step_registry import StepRegistry from app.modules.agent.engine.orchestrator.template_registry import ScenarioTemplateRegistry from app.schemas.common import ModuleName +LOGGER = logging.getLogger(__name__) + class OrchestratorService: def __init__( @@ -74,6 +77,21 @@ class OrchestratorService: ) result = self._assembler.assemble(ctx, step_results) await self._emit_progress(progress_cb, "orchestrator.done", "Execution plan completed.") + LOGGER.warning( + "orchestrator decision: task_id=%s scenario=%s plan_status=%s steps=%s changeset_items=%s answer_len=%s", + task.task_id, + task.scenario.value, + result.meta.get("plan", {}).get("status", ""), + [ + { + "step_id": step.step_id, + "status": step.status.value, + } + for step in result.steps + ], + len(result.changeset), + len(result.answer or ""), + ) return result async def _emit_progress(self, progress_cb: ProgressCallback | None, stage: str, message: str) -> None: diff --git a/app/modules/agent/engine/orchestrator/step_registry.py b/app/modules/agent/engine/orchestrator/step_registry.py index 736c473..01bbf71 100644 --- a/app/modules/agent/engine/orchestrator/step_registry.py +++ b/app/modules/agent/engine/orchestrator/step_registry.py @@ -2,29 +2,50 @@ from __future__ import annotations import asyncio from collections.abc import Callable +from typing import TYPE_CHECKING from app.modules.agent.engine.graphs.progress_registry import progress_registry -from app.modules.agent.engine.orchestrator.actions import DocsActions, EditActions, ExplainActions, GherkinActions, ReviewActions +from app.modules.agent.engine.orchestrator.actions import ( + CodeExplainActions, + DocsActions, + EditActions, + ExplainActions, + GherkinActions, + ProjectQaActions, + ReviewActions, +) from app.modules.agent.engine.orchestrator.execution_context import ExecutionContext from app.modules.agent.engine.orchestrator.models import ArtifactType, PlanStep +if TYPE_CHECKING: + from app.modules.rag.explain.retriever_v2 import CodeExplainRetrieverV2 + StepFn = Callable[[ExecutionContext], list[str]] class StepRegistry: - def __init__(self) -> None: + def __init__(self, code_explain_retriever: CodeExplainRetrieverV2 | None = None) -> None: + code_explain = CodeExplainActions(code_explain_retriever) explain = ExplainActions() review = ReviewActions() docs = DocsActions() edits = EditActions() gherkin = GherkinActions() + project_qa = ProjectQaActions() self._functions: dict[str, StepFn] = { "collect_state": self._collect_state, "finalize_graph_output": self._finalize_graph_output, + "execute_project_qa_graph": self._collect_state, + "build_code_explain_pack": code_explain.build_code_explain_pack, "collect_sources": explain.collect_sources, "extract_logic": explain.extract_logic, "summarize": explain.summarize, + "classify_project_question": project_qa.classify_project_question, + "collect_project_sources": project_qa.collect_project_sources, + "analyze_project_sources": project_qa.analyze_project_sources, + "build_project_answer_brief": project_qa.build_project_answer_brief, + "compose_project_answer": project_qa.compose_project_answer, "fetch_source_doc": review.fetch_source_doc, "normalize_document": review.normalize_document, "structural_check": review.structural_check, @@ -66,6 +87,7 @@ class StepRegistry: state = { "task_id": ctx.task.task_id, "project_id": ctx.task.rag_session_id, + "scenario": ctx.task.scenario.value, "message": ctx.task.user_message, "progress_key": ctx.task.task_id, "rag_context": str(ctx.task.metadata.get("rag_context", "")), @@ -86,7 +108,7 @@ class StepRegistry: raise RuntimeError(f"Unsupported graph_id: {graph_key}") graph = ctx.graph_resolver(domain_id, process_id) - state = ctx.artifacts.get_content("agent_state", {}) or {} + state = self._build_graph_state(ctx) if ctx.progress_cb is not None: progress_registry.register(ctx.task.task_id, ctx.progress_cb) @@ -96,8 +118,29 @@ class StepRegistry: if ctx.progress_cb is not None: progress_registry.unregister(ctx.task.task_id) - item = ctx.artifacts.put(key="graph_result", artifact_type=ArtifactType.STRUCTURED_JSON, content=result) - return [item.artifact_id] + return self._store_graph_outputs(step, ctx, result) + + def _build_graph_state(self, ctx: ExecutionContext) -> dict: + state = dict(ctx.artifacts.get_content("agent_state", {}) or {}) + for item in ctx.artifacts.all_items(): + state[item.key] = ctx.artifacts.get_content(item.key) + return state + + def _store_graph_outputs(self, step: PlanStep, ctx: ExecutionContext, result: dict) -> list[str]: + if not isinstance(result, dict): + raise RuntimeError("graph_result must be an object") + if len(step.outputs) == 1 and step.outputs[0].key == "graph_result": + item = ctx.artifacts.put(key="graph_result", artifact_type=ArtifactType.STRUCTURED_JSON, content=result) + return [item.artifact_id] + + artifact_ids: list[str] = [] + for output in step.outputs: + value = result.get(output.key) + if value is None and output.required: + raise RuntimeError(f"graph_output_missing:{step.step_id}:{output.key}") + item = ctx.artifacts.put(key=output.key, artifact_type=output.type, content=value) + artifact_ids.append(item.artifact_id) + return artifact_ids def _finalize_graph_output(self, ctx: ExecutionContext) -> list[str]: raw = ctx.artifacts.get_content("graph_result", {}) or {} diff --git a/app/modules/agent/engine/orchestrator/template_registry.py b/app/modules/agent/engine/orchestrator/template_registry.py index b6554a1..3dd90d6 100644 --- a/app/modules/agent/engine/orchestrator/template_registry.py +++ b/app/modules/agent/engine/orchestrator/template_registry.py @@ -16,6 +16,8 @@ class ScenarioTemplateRegistry: return builders.get(task.scenario, self._general)(task) def _general(self, task: TaskSpec) -> ExecutionPlan: + if task.routing.domain_id == "project" and task.routing.process_id == "qa": + return self._project_qa(task) steps = [ self._step("collect_state", "Collect state", "collect_state", outputs=[self._out("agent_state", ArtifactType.STRUCTURED_JSON)]), self._step( @@ -39,7 +41,77 @@ class ScenarioTemplateRegistry: ] return self._plan(task, "general_qa_v1", steps, [self._gate("non_empty_answer_or_changeset")]) + def _project_qa(self, task: TaskSpec) -> ExecutionPlan: + steps = [ + self._step("collect_state", "Collect state", "collect_state", outputs=[self._out("agent_state", ArtifactType.STRUCTURED_JSON)]), + self._step( + "conversation_understanding", + "Conversation understanding", + "execute_project_qa_graph", + executor="graph", + graph_id="project_qa/conversation_understanding", + depends_on=["collect_state"], + outputs=[self._out("resolved_request", ArtifactType.STRUCTURED_JSON)], + ), + self._step( + "question_classification", + "Question classification", + "execute_project_qa_graph", + executor="graph", + graph_id="project_qa/question_classification", + depends_on=["conversation_understanding"], + outputs=[self._out("question_profile", ArtifactType.STRUCTURED_JSON)], + ), + self._step( + "context_retrieval", + "Context retrieval", + "execute_project_qa_graph", + executor="graph", + graph_id="project_qa/context_retrieval", + depends_on=["question_classification"], + outputs=[self._out("source_bundle", ArtifactType.STRUCTURED_JSON)], + ), + ] + analysis_depends_on = ["context_retrieval"] + if task.scenario == Scenario.EXPLAIN_PART: + steps.append( + self._step( + "code_explain_pack_step", + "Build code explain pack", + "build_code_explain_pack", + depends_on=["context_retrieval"], + outputs=[self._out("explain_pack", ArtifactType.STRUCTURED_JSON)], + ) + ) + analysis_depends_on = ["code_explain_pack_step"] + steps.extend( + [ + self._step( + "context_analysis", + "Context analysis", + "execute_project_qa_graph", + executor="graph", + graph_id="project_qa/context_analysis", + depends_on=analysis_depends_on, + outputs=[self._out("analysis_brief", ArtifactType.STRUCTURED_JSON)], + ), + self._step( + "answer_composition", + "Answer composition", + "execute_project_qa_graph", + executor="graph", + graph_id="project_qa/answer_composition", + depends_on=["context_analysis"], + outputs=[self._out("answer_brief", ArtifactType.STRUCTURED_JSON, required=False), self._out("final_answer", ArtifactType.TEXT)], + gates=[self._gate("non_empty_answer_or_changeset")], + ), + ] + ) + return self._plan(task, "project_qa_reasoning_v1", steps, [self._gate("non_empty_answer_or_changeset")]) + def _explain(self, task: TaskSpec) -> ExecutionPlan: + if task.routing.domain_id == "project" and task.routing.process_id == "qa": + return self._project_qa(task) steps = [ self._step("collect_sources", "Collect sources", "collect_sources", outputs=[self._out("sources", ArtifactType.STRUCTURED_JSON)]), self._step("extract_logic", "Extract logic", "extract_logic", depends_on=["collect_sources"], outputs=[self._out("logic_model", ArtifactType.STRUCTURED_JSON)]), diff --git a/app/modules/agent/engine/router/__init__.py b/app/modules/agent/engine/router/__init__.py index 50da4fa..ee2de63 100644 --- a/app/modules/agent/engine/router/__init__.py +++ b/app/modules/agent/engine/router/__init__.py @@ -2,21 +2,28 @@ from pathlib import Path from typing import TYPE_CHECKING from app.modules.agent.llm import AgentLlmService +from app.modules.contracts import RagRetriever if TYPE_CHECKING: from app.modules.agent.repository import AgentRepository from app.modules.agent.engine.router.router_service import RouterService -def build_router_service(llm: AgentLlmService, agent_repository: "AgentRepository") -> "RouterService": +def build_router_service(llm: AgentLlmService, agent_repository: "AgentRepository", rag: RagRetriever) -> "RouterService": from app.modules.agent.engine.graphs import ( BaseGraphFactory, DocsGraphFactory, ProjectEditsGraphFactory, + ProjectQaAnalysisGraphFactory, + ProjectQaAnswerGraphFactory, + ProjectQaClassificationGraphFactory, + ProjectQaConversationGraphFactory, ProjectQaGraphFactory, + ProjectQaRetrievalGraphFactory, ) from app.modules.agent.engine.router.context_store import RouterContextStore from app.modules.agent.engine.router.intent_classifier import IntentClassifier + from app.modules.agent.engine.router.intent_switch_detector import IntentSwitchDetector from app.modules.agent.engine.router.registry import IntentRegistry from app.modules.agent.engine.router.router_service import RouterService @@ -26,13 +33,20 @@ def build_router_service(llm: AgentLlmService, agent_repository: "AgentRepositor registry.register("project", "qa", ProjectQaGraphFactory(llm).build) registry.register("project", "edits", ProjectEditsGraphFactory(llm).build) registry.register("docs", "generation", DocsGraphFactory(llm).build) + registry.register("project_qa", "conversation_understanding", ProjectQaConversationGraphFactory(llm).build) + registry.register("project_qa", "question_classification", ProjectQaClassificationGraphFactory(llm).build) + registry.register("project_qa", "context_retrieval", ProjectQaRetrievalGraphFactory(rag, llm).build) + registry.register("project_qa", "context_analysis", ProjectQaAnalysisGraphFactory(llm).build) + registry.register("project_qa", "answer_composition", ProjectQaAnswerGraphFactory(llm).build) classifier = IntentClassifier(llm) + switch_detector = IntentSwitchDetector() context_store = RouterContextStore(agent_repository) return RouterService( registry=registry, classifier=classifier, context_store=context_store, + switch_detector=switch_detector, ) diff --git a/app/modules/agent/engine/router/__pycache__/__init__.cpython-312.pyc b/app/modules/agent/engine/router/__pycache__/__init__.cpython-312.pyc index 12f70d9..60dc2e9 100644 Binary files a/app/modules/agent/engine/router/__pycache__/__init__.cpython-312.pyc and b/app/modules/agent/engine/router/__pycache__/__init__.cpython-312.pyc differ diff --git a/app/modules/agent/engine/router/__pycache__/context_store.cpython-312.pyc b/app/modules/agent/engine/router/__pycache__/context_store.cpython-312.pyc index 6d415f4..46bfa72 100644 Binary files a/app/modules/agent/engine/router/__pycache__/context_store.cpython-312.pyc and b/app/modules/agent/engine/router/__pycache__/context_store.cpython-312.pyc differ diff --git a/app/modules/agent/engine/router/__pycache__/intent_classifier.cpython-312.pyc b/app/modules/agent/engine/router/__pycache__/intent_classifier.cpython-312.pyc index 915280e..7411ec2 100644 Binary files a/app/modules/agent/engine/router/__pycache__/intent_classifier.cpython-312.pyc and b/app/modules/agent/engine/router/__pycache__/intent_classifier.cpython-312.pyc differ diff --git a/app/modules/agent/engine/router/__pycache__/intent_switch_detector.cpython-312.pyc b/app/modules/agent/engine/router/__pycache__/intent_switch_detector.cpython-312.pyc new file mode 100644 index 0000000..0509b3d Binary files /dev/null and b/app/modules/agent/engine/router/__pycache__/intent_switch_detector.cpython-312.pyc differ diff --git a/app/modules/agent/engine/router/__pycache__/router_service.cpython-312.pyc b/app/modules/agent/engine/router/__pycache__/router_service.cpython-312.pyc index c544c9c..3a3a5d2 100644 Binary files a/app/modules/agent/engine/router/__pycache__/router_service.cpython-312.pyc and b/app/modules/agent/engine/router/__pycache__/router_service.cpython-312.pyc differ diff --git a/app/modules/agent/engine/router/__pycache__/schemas.cpython-312.pyc b/app/modules/agent/engine/router/__pycache__/schemas.cpython-312.pyc index 04d5bd4..1d05654 100644 Binary files a/app/modules/agent/engine/router/__pycache__/schemas.cpython-312.pyc and b/app/modules/agent/engine/router/__pycache__/schemas.cpython-312.pyc differ diff --git a/app/modules/agent/engine/router/context_store.py b/app/modules/agent/engine/router/context_store.py index c13c500..a514fc4 100644 --- a/app/modules/agent/engine/router/context_store.py +++ b/app/modules/agent/engine/router/context_store.py @@ -17,6 +17,7 @@ class RouterContextStore: process_id: str, user_message: str, assistant_message: str, + decision_type: str = "start", max_history: int = 10, ) -> None: self._repo.update_router_context( @@ -25,5 +26,6 @@ class RouterContextStore: process_id=process_id, user_message=user_message, assistant_message=assistant_message, + decision_type=decision_type, max_history=max_history, ) diff --git a/app/modules/agent/engine/router/intent_classifier.py b/app/modules/agent/engine/router/intent_classifier.py index e478a8e..4e73c9b 100644 --- a/app/modules/agent/engine/router/intent_classifier.py +++ b/app/modules/agent/engine/router/intent_classifier.py @@ -17,11 +17,7 @@ class IntentClassifier: def __init__(self, llm: AgentLlmService) -> None: self._llm = llm - def classify(self, user_message: str, context: RouterContext, mode: str = "auto") -> RouteDecision: - forced = self._from_mode(mode) - if forced: - return forced - + def classify_new_intent(self, user_message: str, context: RouterContext) -> RouteDecision: text = (user_message or "").strip().lower() if text in self._short_confirmations and context.last_routing: return RouteDecision( @@ -30,6 +26,7 @@ class IntentClassifier: confidence=1.0, reason="short_confirmation", use_previous=True, + decision_type="continue", ) deterministic = self._deterministic_route(text) @@ -45,9 +42,10 @@ class IntentClassifier: process_id="general", confidence=0.8, reason="default", + decision_type="start", ) - def _from_mode(self, mode: str) -> RouteDecision | None: + def from_mode(self, mode: str) -> RouteDecision | None: mapping = { "project_qa": ("project", "qa"), "project_edits": ("project", "edits"), @@ -65,6 +63,8 @@ class IntentClassifier: process_id=route[1], confidence=1.0, reason=f"mode_override:{mode}", + decision_type="switch", + explicit_switch=True, ) def _classify_with_llm(self, user_message: str, context: RouterContext) -> RouteDecision | None: @@ -96,6 +96,7 @@ class IntentClassifier: process_id=route[1], confidence=confidence, reason=f"llm_router:{payload.get('reason', 'ok')}", + decision_type="start", ) def _parse_llm_payload(self, raw: str) -> dict[str, str | float] | None: @@ -139,6 +140,8 @@ class IntentClassifier: process_id="edits", confidence=0.97, reason="deterministic_targeted_file_edit", + decision_type="switch", + explicit_switch=True, ) if self._is_broad_docs_request(text): return RouteDecision( @@ -146,6 +149,8 @@ class IntentClassifier: process_id="generation", confidence=0.95, reason="deterministic_docs_generation", + decision_type="switch", + explicit_switch=True, ) return None diff --git a/app/modules/agent/engine/router/intent_switch_detector.py b/app/modules/agent/engine/router/intent_switch_detector.py new file mode 100644 index 0000000..151b57c --- /dev/null +++ b/app/modules/agent/engine/router/intent_switch_detector.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import re + +from app.modules.agent.engine.router.schemas import RouterContext + + +class IntentSwitchDetector: + _EXPLICIT_SWITCH_MARKERS = ( + "теперь", + "а теперь", + "давай теперь", + "переключись", + "переключаемся", + "сейчас другое", + "новая задача", + "new task", + "switch to", + "now do", + "instead", + ) + _FOLLOW_UP_MARKERS = ( + "а еще", + "а ещё", + "подробнее", + "почему", + "зачем", + "что если", + "и еще", + "и ещё", + "покажи подробнее", + "можешь подробнее", + ) + + def should_switch(self, user_message: str, context: RouterContext) -> bool: + if not context.dialog_started or context.active_intent is None: + return False + text = " ".join((user_message or "").strip().lower().split()) + if not text: + return False + if self._is_follow_up(text): + return False + if any(marker in text for marker in self._EXPLICIT_SWITCH_MARKERS): + return True + return self._is_strong_targeted_edit_request(text) or self._is_strong_docs_request(text) + + def _is_follow_up(self, text: str) -> bool: + return any(marker in text for marker in self._FOLLOW_UP_MARKERS) + + def _is_strong_targeted_edit_request(self, text: str) -> bool: + edit_markers = ( + "добавь", + "добавить", + "измени", + "исправь", + "обнови", + "удали", + "замени", + "append", + "update", + "edit", + "remove", + "replace", + ) + has_edit_marker = any(marker in text for marker in edit_markers) + has_file_marker = ( + "readme" in text + or bool(re.search(r"\b[\w.\-/]+\.(md|txt|rst|yaml|yml|json|toml|ini|cfg|py)\b", text)) + ) + return has_edit_marker and has_file_marker + + def _is_strong_docs_request(self, text: str) -> bool: + docs_markers = ( + "подготовь документац", + "сгенерируй документац", + "создай документац", + "опиши документац", + "generate documentation", + "write documentation", + ) + return any(marker in text for marker in docs_markers) diff --git a/app/modules/agent/engine/router/router_service.py b/app/modules/agent/engine/router/router_service.py index 9ebbb84..2ff7c7e 100644 --- a/app/modules/agent/engine/router/router_service.py +++ b/app/modules/agent/engine/router/router_service.py @@ -1,7 +1,8 @@ from app.modules.agent.engine.router.context_store import RouterContextStore from app.modules.agent.engine.router.intent_classifier import IntentClassifier +from app.modules.agent.engine.router.intent_switch_detector import IntentSwitchDetector from app.modules.agent.engine.router.registry import IntentRegistry -from app.modules.agent.engine.router.schemas import RouteResolution +from app.modules.agent.engine.router.schemas import RouteDecision, RouteResolution class RouterService: @@ -10,27 +11,48 @@ class RouterService: registry: IntentRegistry, classifier: IntentClassifier, context_store: RouterContextStore, + switch_detector: IntentSwitchDetector | None = None, min_confidence: float = 0.7, ) -> None: self._registry = registry self._classifier = classifier self._ctx = context_store + self._switch_detector = switch_detector or IntentSwitchDetector() self._min_confidence = min_confidence def resolve(self, user_message: str, conversation_key: str, mode: str = "auto") -> RouteResolution: context = self._ctx.get(conversation_key) - decision = self._classifier.classify(user_message, context, mode=mode) - if decision.confidence < self._min_confidence: - return self._fallback("low_confidence") - if not self._registry.is_valid(decision.domain_id, decision.process_id): - return self._fallback("invalid_route") - return RouteResolution( - domain_id=decision.domain_id, - process_id=decision.process_id, - confidence=decision.confidence, - reason=decision.reason, - fallback_used=False, - ) + forced = self._classifier.from_mode(mode) + if forced: + return self._resolution(forced) + + if not context.dialog_started or context.active_intent is None: + decision = self._classifier.classify_new_intent(user_message, context) + if not self._is_acceptable(decision): + return self._fallback("low_confidence") + return self._resolution( + decision.model_copy( + update={ + "decision_type": "start", + "explicit_switch": False, + } + ) + ) + + if self._switch_detector.should_switch(user_message, context): + decision = self._classifier.classify_new_intent(user_message, context) + if self._is_acceptable(decision): + return self._resolution( + decision.model_copy( + update={ + "decision_type": "switch", + "explicit_switch": True, + } + ) + ) + return self._continue_current(context, "explicit_switch_unresolved_keep_current") + + return self._continue_current(context, "continue_current_intent") def persist_context( self, @@ -40,6 +62,7 @@ class RouterService: process_id: str, user_message: str, assistant_message: str, + decision_type: str = "start", ) -> None: self._ctx.update( conversation_key, @@ -47,6 +70,7 @@ class RouterService: process_id=process_id, user_message=user_message, assistant_message=assistant_message, + decision_type=decision_type, ) def graph_factory(self, domain_id: str, process_id: str): @@ -59,4 +83,32 @@ class RouterService: confidence=0.0, reason=reason, fallback_used=True, + decision_type="start", + explicit_switch=False, + ) + + def _continue_current(self, context, reason: str) -> RouteResolution: + active = context.active_intent or context.last_routing or {"domain_id": "default", "process_id": "general"} + return RouteResolution( + domain_id=str(active["domain_id"]), + process_id=str(active["process_id"]), + confidence=1.0, + reason=reason, + fallback_used=False, + decision_type="continue", + explicit_switch=False, + ) + + def _is_acceptable(self, decision: RouteDecision) -> bool: + return decision.confidence >= self._min_confidence and self._registry.is_valid(decision.domain_id, decision.process_id) + + def _resolution(self, decision: RouteDecision) -> RouteResolution: + return RouteResolution( + domain_id=decision.domain_id, + process_id=decision.process_id, + confidence=decision.confidence, + reason=decision.reason, + fallback_used=False, + decision_type=decision.decision_type, + explicit_switch=decision.explicit_switch, ) diff --git a/app/modules/agent/engine/router/schemas.py b/app/modules/agent/engine/router/schemas.py index 0d15b1a..233d4fa 100644 --- a/app/modules/agent/engine/router/schemas.py +++ b/app/modules/agent/engine/router/schemas.py @@ -7,6 +7,8 @@ class RouteDecision(BaseModel): confidence: float = 0.0 reason: str = "" use_previous: bool = False + decision_type: str = "start" + explicit_switch: bool = False @field_validator("confidence") @classmethod @@ -20,8 +22,13 @@ class RouteResolution(BaseModel): confidence: float reason: str fallback_used: bool = False + decision_type: str = "start" + explicit_switch: bool = False class RouterContext(BaseModel): last_routing: dict[str, str] | None = None message_history: list[dict[str, str]] = Field(default_factory=list) + active_intent: dict[str, str] | None = None + dialog_started: bool = False + turn_index: int = 0 diff --git a/app/modules/agent/llm/__pycache__/service.cpython-312.pyc b/app/modules/agent/llm/__pycache__/service.cpython-312.pyc index efb82e1..59772d4 100644 Binary files a/app/modules/agent/llm/__pycache__/service.cpython-312.pyc and b/app/modules/agent/llm/__pycache__/service.cpython-312.pyc differ diff --git a/app/modules/agent/llm/service.py b/app/modules/agent/llm/service.py index 47af66d..bb1984a 100644 --- a/app/modules/agent/llm/service.py +++ b/app/modules/agent/llm/service.py @@ -1,14 +1,40 @@ +import logging + from app.modules.agent.prompt_loader import PromptLoader from app.modules.shared.gigachat.client import GigaChatClient +LOGGER = logging.getLogger(__name__) + + +def _truncate_for_log(text: str, max_chars: int = 1500) -> str: + value = (text or "").replace("\n", "\\n").strip() + if len(value) <= max_chars: + return value + return value[:max_chars].rstrip() + "...[truncated]" + class AgentLlmService: def __init__(self, client: GigaChatClient, prompts: PromptLoader) -> None: self._client = client self._prompts = prompts - def generate(self, prompt_name: str, user_input: str) -> str: + def generate(self, prompt_name: str, user_input: str, *, log_context: str | None = None) -> str: system_prompt = self._prompts.load(prompt_name) if not system_prompt: system_prompt = "You are a helpful assistant." - return self._client.complete(system_prompt=system_prompt, user_prompt=user_input) + if log_context: + LOGGER.warning( + "graph llm input: context=%s prompt=%s user_input=%s", + log_context, + prompt_name, + _truncate_for_log(user_input), + ) + output = self._client.complete(system_prompt=system_prompt, user_prompt=user_input) + if log_context: + LOGGER.warning( + "graph llm output: context=%s prompt=%s output=%s", + log_context, + prompt_name, + _truncate_for_log(output), + ) + return output diff --git a/app/modules/agent/module.py b/app/modules/agent/module.py index 78cf353..e0bebe6 100644 --- a/app/modules/agent/module.py +++ b/app/modules/agent/module.py @@ -1,5 +1,8 @@ +from __future__ import annotations + from fastapi import APIRouter from pydantic import BaseModel, HttpUrl +from typing import TYPE_CHECKING from app.modules.agent.changeset_validator import ChangeSetValidator from app.modules.agent.confluence_service import ConfluenceService @@ -19,12 +22,17 @@ class ConfluenceFetchRequest(BaseModel): url: HttpUrl +if TYPE_CHECKING: + from app.modules.rag.explain.retriever_v2 import CodeExplainRetrieverV2 + + class AgentModule: def __init__( self, rag_retriever: RagRetriever, agent_repository: AgentRepository, story_context_repository: StoryContextRepository, + code_explain_retriever: CodeExplainRetrieverV2 | None = None, ) -> None: self.confluence = ConfluenceService() self.changeset_validator = ChangeSetValidator() @@ -34,14 +42,16 @@ class AgentModule: client = GigaChatClient(settings, token_provider) prompt_loader = PromptLoader() llm = AgentLlmService(client=client, prompts=prompt_loader) + self.llm = llm story_recorder = StorySessionRecorder(story_context_repository) self.runtime = GraphAgentRuntime( rag=rag_retriever, confluence=self.confluence, changeset_validator=self.changeset_validator, - llm=llm, + llm=self.llm, agent_repository=agent_repository, story_recorder=story_recorder, + code_explain_retriever=code_explain_retriever, ) def internal_router(self) -> APIRouter: diff --git a/app/modules/agent/prompts/code_explain_answer_v2.txt b/app/modules/agent/prompts/code_explain_answer_v2.txt new file mode 100644 index 0000000..394e2fe --- /dev/null +++ b/app/modules/agent/prompts/code_explain_answer_v2.txt @@ -0,0 +1,17 @@ +Объяснение кода осуществляется только с использованием предоставленного ExplainPack. + +Правила: +- Сначала используйте доказательства. +- Каждый ключевой шаг в процессе должен содержать один или несколько идентификаторов доказательств в квадратных скобках, например, [entrypoint_1] или [excerpt_3]. +- Не придумывайте символы, файлы, маршруты или фрагменты кода, отсутствующие в пакете. +- Если доказательства неполные, укажите это явно. +- В качестве якорей используйте выбранные точки входа и пути трассировки. + +Верните Markdown со следующей структурой: +1. Краткое описание +2. Пошаговый процесс +3. Данные и побочные эффекты +4. Ошибки и граничные случаи +5. Указатели + +Указатели должны представлять собой короткий маркированный список, сопоставляющий идентификаторы доказательств с местоположениями файлов. \ No newline at end of file diff --git a/app/modules/agent/prompts/rag_intent_router_v2.txt b/app/modules/agent/prompts/rag_intent_router_v2.txt new file mode 100644 index 0000000..aee7599 --- /dev/null +++ b/app/modules/agent/prompts/rag_intent_router_v2.txt @@ -0,0 +1,24 @@ +Ты intent-router для layered RAG. +На вход ты получаешь JSON с полями: +- message: текущий запрос пользователя +- active_intent: текущий активный intent диалога или null +- last_query: предыдущий запрос пользователя +- allowed_intents: допустимые intent'ы + +Выбери ровно один intent из allowed_intents. +Верни только JSON без markdown и пояснений. + +Строгий формат ответа: +{"intent":"","confidence":,"reason":""} + +Правила: +- CODE_QA: объяснение по коду, архитектуре, классам, методам, файлам, блокам кода, поведению приложения по реализации. +- DOCS_QA: объяснение по документации, README, markdown, specs, runbooks, разделам документации. +- GENERATE_DOCS_FROM_CODE: просьба сгенерировать, подготовить или обновить документацию по коду. +- PROJECT_MISC: прочие вопросы по проекту, не относящиеся явно к коду или документации. + +Приоритет: +- Если пользователь просит именно подготовить документацию по коду, выбирай GENERATE_DOCS_FROM_CODE. +- Если пользователь спрашивает про конкретный класс, файл, метод или блок кода, выбирай CODE_QA. +- Если пользователь спрашивает про README, docs, markdown или конкретную документацию, выбирай DOCS_QA. +- Если сигнал неочевиден, выбирай PROJECT_MISC и confidence <= 0.6. diff --git a/app/modules/agent/repository.py b/app/modules/agent/repository.py index e9d3d46..2e764e0 100644 --- a/app/modules/agent/repository.py +++ b/app/modules/agent/repository.py @@ -18,6 +18,10 @@ class AgentRepository: conversation_key VARCHAR(64) PRIMARY KEY, last_domain_id VARCHAR(64) NULL, last_process_id VARCHAR(64) NULL, + active_domain_id VARCHAR(64) NULL, + active_process_id VARCHAR(64) NULL, + dialog_started BOOLEAN NOT NULL DEFAULT FALSE, + turn_index INTEGER NOT NULL DEFAULT 0, message_history_json TEXT NOT NULL DEFAULT '[]', updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP ) @@ -64,14 +68,24 @@ class AgentRepository: """ ) ) + self._ensure_router_context_columns(conn) conn.commit() + def _ensure_router_context_columns(self, conn) -> None: + for statement in ( + "ALTER TABLE router_context ADD COLUMN IF NOT EXISTS active_domain_id VARCHAR(64) NULL", + "ALTER TABLE router_context ADD COLUMN IF NOT EXISTS active_process_id VARCHAR(64) NULL", + "ALTER TABLE router_context ADD COLUMN IF NOT EXISTS dialog_started BOOLEAN NOT NULL DEFAULT FALSE", + "ALTER TABLE router_context ADD COLUMN IF NOT EXISTS turn_index INTEGER NOT NULL DEFAULT 0", + ): + conn.execute(text(statement)) + def get_router_context(self, conversation_key: str) -> RouterContext: with get_engine().connect() as conn: row = conn.execute( text( """ - SELECT last_domain_id, last_process_id, message_history_json + SELECT last_domain_id, last_process_id, active_domain_id, active_process_id, dialog_started, turn_index, message_history_json FROM router_context WHERE conversation_key = :key """ @@ -82,7 +96,7 @@ class AgentRepository: if not row: return RouterContext() - history_raw = row[2] or "[]" + history_raw = row[6] or "[]" try: history = json.loads(history_raw) except json.JSONDecodeError: @@ -91,6 +105,9 @@ class AgentRepository: last = None if row[0] and row[1]: last = {"domain_id": str(row[0]), "process_id": str(row[1])} + active = None + if row[2] and row[3]: + active = {"domain_id": str(row[2]), "process_id": str(row[3])} clean_history = [] for item in history if isinstance(history, list) else []: @@ -101,7 +118,13 @@ class AgentRepository: if role in {"user", "assistant"} and content: clean_history.append({"role": role, "content": content}) - return RouterContext(last_routing=last, message_history=clean_history) + return RouterContext( + last_routing=last, + message_history=clean_history, + active_intent=active or last, + dialog_started=bool(row[4]), + turn_index=int(row[5] or 0), + ) def update_router_context( self, @@ -111,6 +134,7 @@ class AgentRepository: process_id: str, user_message: str, assistant_message: str, + decision_type: str, max_history: int, ) -> None: current = self.get_router_context(conversation_key) @@ -121,17 +145,29 @@ class AgentRepository: history.append({"role": "assistant", "content": assistant_message}) if max_history > 0: history = history[-max_history:] + current_active = current.active_intent or current.last_routing or {"domain_id": domain_id, "process_id": process_id} + next_active = ( + {"domain_id": domain_id, "process_id": process_id} + if decision_type in {"start", "switch"} + else current_active + ) + next_turn_index = max(0, int(current.turn_index or 0)) + (1 if user_message else 0) with get_engine().connect() as conn: conn.execute( text( """ INSERT INTO router_context ( - conversation_key, last_domain_id, last_process_id, message_history_json - ) VALUES (:key, :domain, :process, :history) + conversation_key, last_domain_id, last_process_id, active_domain_id, active_process_id, + dialog_started, turn_index, message_history_json + ) VALUES (:key, :domain, :process, :active_domain, :active_process, :dialog_started, :turn_index, :history) ON CONFLICT (conversation_key) DO UPDATE SET last_domain_id = EXCLUDED.last_domain_id, last_process_id = EXCLUDED.last_process_id, + active_domain_id = EXCLUDED.active_domain_id, + active_process_id = EXCLUDED.active_process_id, + dialog_started = EXCLUDED.dialog_started, + turn_index = EXCLUDED.turn_index, message_history_json = EXCLUDED.message_history_json, updated_at = CURRENT_TIMESTAMP """ @@ -140,6 +176,10 @@ class AgentRepository: "key": conversation_key, "domain": domain_id, "process": process_id, + "active_domain": str(next_active["domain_id"]), + "active_process": str(next_active["process_id"]), + "dialog_started": True, + "turn_index": next_turn_index, "history": json.dumps(history, ensure_ascii=False), }, ) diff --git a/app/modules/agent/service.py b/app/modules/agent/service.py index 0b3a114..7885ed4 100644 --- a/app/modules/agent/service.py +++ b/app/modules/agent/service.py @@ -1,12 +1,16 @@ +from __future__ import annotations + from dataclasses import dataclass, field from collections.abc import Awaitable, Callable import inspect import logging import re +from typing import TYPE_CHECKING from app.modules.agent.engine.orchestrator import OrchestratorService, TaskSpecBuilder from app.modules.agent.engine.orchestrator.metrics_persister import MetricsPersister from app.modules.agent.engine.orchestrator.models import RoutingMeta +from app.modules.agent.engine.orchestrator.step_registry import StepRegistry from app.modules.agent.engine.router import build_router_service from app.modules.agent.llm import AgentLlmService from app.modules.agent.story_session_recorder import StorySessionRecorder @@ -22,6 +26,9 @@ from app.schemas.common import ModuleName LOGGER = logging.getLogger(__name__) +if TYPE_CHECKING: + from app.modules.rag.explain.retriever_v2 import CodeExplainRetrieverV2 + def _truncate_for_log(text: str | None, max_chars: int = 1500) -> str: value = (text or "").replace("\n", "\\n").strip() @@ -47,13 +54,14 @@ class GraphAgentRuntime: llm: AgentLlmService, agent_repository: AgentRepository, story_recorder: StorySessionRecorder | None = None, + code_explain_retriever: CodeExplainRetrieverV2 | None = None, ) -> None: self._rag = rag self._confluence = confluence self._changeset_validator = changeset_validator - self._router = build_router_service(llm, agent_repository) + self._router = build_router_service(llm, agent_repository, rag) self._task_spec_builder = TaskSpecBuilder() - self._orchestrator = OrchestratorService() + self._orchestrator = OrchestratorService(step_registry=StepRegistry(code_explain_retriever)) self._metrics_persister = MetricsPersister(agent_repository) self._story_recorder = story_recorder self._checkpointer = None @@ -70,7 +78,7 @@ class GraphAgentRuntime: files: list[dict], progress_cb: Callable[[str, str, str, dict | None], Awaitable[None] | None] | None = None, ) -> AgentResult: - LOGGER.warning( + LOGGER.info( "GraphAgentRuntime.run started: task_id=%s dialog_session_id=%s mode=%s", task_id, dialog_session_id, @@ -96,9 +104,7 @@ class GraphAgentRuntime: meta={"domain_id": route.domain_id, "process_id": route.process_id}, ) files_map = self._build_files_map(files) - - await self._emit_progress(progress_cb, "agent.rag", "Собираю релевантный контекст из RAG.") - rag_ctx = await self._rag.retrieve(rag_session_id, message) + rag_ctx: list[dict] = [] await self._emit_progress(progress_cb, "agent.attachments", "Обрабатываю дополнительные вложения.") conf_pages = await self._fetch_confluence_pages(attachments) route_meta = RoutingMeta( @@ -157,8 +163,9 @@ class GraphAgentRuntime: process_id=route.process_id, user_message=message, assistant_message=final_answer, + decision_type=route.decision_type, ) - LOGGER.warning( + LOGGER.info( "final agent answer: task_id=%s route=%s/%s answer=%s", task_id, route.domain_id, @@ -178,7 +185,7 @@ class GraphAgentRuntime: answer=final_answer, meta={ "route": route.model_dump(), - "used_rag": True, + "used_rag": False, "used_confluence": bool(conf_pages), "changeset_filtered_out": True, "orchestrator": orchestrator_meta, @@ -193,6 +200,7 @@ class GraphAgentRuntime: process_id=route.process_id, user_message=message, assistant_message=final_answer or f"changeset:{len(validated)}", + decision_type=route.decision_type, ) final = AgentResult( result_type=TaskResultType.CHANGESET, @@ -200,7 +208,7 @@ class GraphAgentRuntime: changeset=validated, meta={ "route": route.model_dump(), - "used_rag": True, + "used_rag": False, "used_confluence": bool(conf_pages), "orchestrator": orchestrator_meta, "orchestrator_steps": orchestrator_steps, @@ -214,7 +222,7 @@ class GraphAgentRuntime: scenario=str(orchestrator_meta.get("scenario", task_spec.scenario.value)), quality=quality_meta, ) - LOGGER.warning( + LOGGER.info( "GraphAgentRuntime.run completed: task_id=%s route=%s/%s result_type=%s changeset_items=%s", task_id, route.domain_id, @@ -222,7 +230,7 @@ class GraphAgentRuntime: final.result_type.value, len(final.changeset), ) - LOGGER.warning( + LOGGER.info( "final agent answer: task_id=%s route=%s/%s answer=%s", task_id, route.domain_id, @@ -239,13 +247,14 @@ class GraphAgentRuntime: process_id=route.process_id, user_message=message, assistant_message=final_answer, + decision_type=route.decision_type, ) final = AgentResult( result_type=TaskResultType.ANSWER, answer=final_answer, meta={ "route": route.model_dump(), - "used_rag": True, + "used_rag": False, "used_confluence": bool(conf_pages), "orchestrator": orchestrator_meta, "orchestrator_steps": orchestrator_steps, @@ -259,7 +268,7 @@ class GraphAgentRuntime: scenario=str(orchestrator_meta.get("scenario", task_spec.scenario.value)), quality=quality_meta, ) - LOGGER.warning( + LOGGER.info( "GraphAgentRuntime.run completed: task_id=%s route=%s/%s result_type=%s answer_len=%s", task_id, route.domain_id, @@ -267,7 +276,7 @@ class GraphAgentRuntime: final.result_type.value, len(final.answer or ""), ) - LOGGER.warning( + LOGGER.info( "final agent answer: task_id=%s route=%s/%s answer=%s", task_id, route.domain_id, @@ -351,7 +360,7 @@ class GraphAgentRuntime: factory = self._router.graph_factory("default", "general") if factory is None: raise RuntimeError("No graph factory configured") - LOGGER.warning("_resolve_graph resolved: domain_id=%s process_id=%s", domain_id, process_id) + LOGGER.debug("_resolve_graph resolved: domain_id=%s process_id=%s", domain_id, process_id) return factory(self._checkpointer) def _invoke_graph(self, graph, state: dict, dialog_session_id: str): @@ -365,7 +374,7 @@ class GraphAgentRuntime: for item in attachments: if item.get("type") == "confluence_url": pages.append(await self._confluence.fetch_page(item["url"])) - LOGGER.warning("_fetch_confluence_pages completed: pages=%s", len(pages)) + LOGGER.info("_fetch_confluence_pages completed: pages=%s", len(pages)) return pages def _format_rag(self, items: list[dict]) -> str: @@ -411,7 +420,7 @@ class GraphAgentRuntime: "content": str(item.get("content", "")), "content_hash": str(item.get("content_hash", "")), } - LOGGER.warning("_build_files_map completed: files=%s", len(output)) + LOGGER.debug("_build_files_map completed: files=%s", len(output)) return output def _lookup_file(self, files_map: dict[str, dict], path: str) -> dict | None: @@ -437,7 +446,7 @@ class GraphAgentRuntime: ) item.base_hash = str(source["content_hash"]) enriched.append(item) - LOGGER.warning("_enrich_changeset_hashes completed: items=%s", len(enriched)) + LOGGER.debug("_enrich_changeset_hashes completed: items=%s", len(enriched)) return enriched def _sanitize_changeset(self, items: list[ChangeItem], files_map: dict[str, dict]) -> list[ChangeItem]: @@ -462,7 +471,7 @@ class GraphAgentRuntime: continue sanitized.append(item) if dropped_noop or dropped_ws: - LOGGER.warning( + LOGGER.info( "_sanitize_changeset dropped items: noop=%s whitespace_only=%s kept=%s", dropped_noop, dropped_ws, diff --git a/app/modules/application.py b/app/modules/application.py index caab8f3..fd3819b 100644 --- a/app/modules/application.py +++ b/app/modules/application.py @@ -1,9 +1,14 @@ from app.modules.agent.module import AgentModule from app.modules.agent.repository import AgentRepository from app.modules.agent.story_context_repository import StoryContextRepository, StoryContextSchemaRepository +from app.modules.chat.direct_service import CodeExplainChatService +from app.modules.chat.dialog_store import DialogSessionStore from app.modules.chat.repository import ChatRepository from app.modules.chat.module import ChatModule +from app.modules.chat.session_resolver import ChatSessionResolver +from app.modules.chat.task_store import TaskStore from app.modules.rag.persistence.repository import RagRepository +from app.modules.rag.explain import CodeExplainRetrieverV2, CodeGraphRepository, LayeredRetrievalGateway from app.modules.rag_session.module import RagModule from app.modules.rag_repo.module import RagRepoModule from app.modules.shared.bootstrap import bootstrap_database @@ -20,16 +25,32 @@ class ModularApplication: self.agent_repository = AgentRepository() self.story_context_schema_repository = StoryContextSchemaRepository() self.story_context_repository = StoryContextRepository() + self.chat_tasks = TaskStore() self.rag_session = RagModule(event_bus=self.events, retry=self.retry, repository=self.rag_repository) self.rag_repo = RagRepoModule( story_context_repository=self.story_context_repository, rag_repository=self.rag_repository, ) + self.code_explain_retriever = CodeExplainRetrieverV2( + gateway=LayeredRetrievalGateway(self.rag_repository, self.rag_session.embedder), + graph_repository=CodeGraphRepository(), + ) self.agent = AgentModule( rag_retriever=self.rag_session.rag, agent_repository=self.agent_repository, story_context_repository=self.story_context_repository, + code_explain_retriever=self.code_explain_retriever, + ) + self.direct_chat = CodeExplainChatService( + retriever=self.code_explain_retriever, + llm=self.agent.llm, + session_resolver=ChatSessionResolver( + dialogs=DialogSessionStore(self.chat_repository), + rag_session_exists=lambda rag_session_id: self.rag_session.sessions.get(rag_session_id) is not None, + ), + task_store=self.chat_tasks, + message_sink=self.chat_repository.add_message, ) self.chat = ChatModule( agent_runner=self.agent.runtime, @@ -37,6 +58,8 @@ class ModularApplication: retry=self.retry, rag_sessions=self.rag_session.sessions, repository=self.chat_repository, + direct_chat=self.direct_chat, + task_store=self.chat_tasks, ) def startup(self) -> None: diff --git a/app/modules/chat/__pycache__/dialog_store.cpython-312.pyc b/app/modules/chat/__pycache__/dialog_store.cpython-312.pyc index 8389fb6..a818156 100644 Binary files a/app/modules/chat/__pycache__/dialog_store.cpython-312.pyc and b/app/modules/chat/__pycache__/dialog_store.cpython-312.pyc differ diff --git a/app/modules/chat/__pycache__/direct_service.cpython-312.pyc b/app/modules/chat/__pycache__/direct_service.cpython-312.pyc new file mode 100644 index 0000000..662a496 Binary files /dev/null and b/app/modules/chat/__pycache__/direct_service.cpython-312.pyc differ diff --git a/app/modules/chat/__pycache__/evidence_gate.cpython-312.pyc b/app/modules/chat/__pycache__/evidence_gate.cpython-312.pyc new file mode 100644 index 0000000..f9e75fb Binary files /dev/null and b/app/modules/chat/__pycache__/evidence_gate.cpython-312.pyc differ diff --git a/app/modules/chat/__pycache__/module.cpython-312.pyc b/app/modules/chat/__pycache__/module.cpython-312.pyc index ffc426a..9cb0490 100644 Binary files a/app/modules/chat/__pycache__/module.cpython-312.pyc and b/app/modules/chat/__pycache__/module.cpython-312.pyc differ diff --git a/app/modules/chat/__pycache__/service.cpython-312.pyc b/app/modules/chat/__pycache__/service.cpython-312.pyc index b098011..5ea97e9 100644 Binary files a/app/modules/chat/__pycache__/service.cpython-312.pyc and b/app/modules/chat/__pycache__/service.cpython-312.pyc differ diff --git a/app/modules/chat/__pycache__/session_resolver.cpython-312.pyc b/app/modules/chat/__pycache__/session_resolver.cpython-312.pyc new file mode 100644 index 0000000..013c8fb Binary files /dev/null and b/app/modules/chat/__pycache__/session_resolver.cpython-312.pyc differ diff --git a/app/modules/chat/dialog_store.py b/app/modules/chat/dialog_store.py index 16ad8f6..ea8f932 100644 --- a/app/modules/chat/dialog_store.py +++ b/app/modules/chat/dialog_store.py @@ -1,7 +1,11 @@ +from __future__ import annotations + from dataclasses import dataclass +from typing import TYPE_CHECKING from uuid import uuid4 -from app.modules.chat.repository import ChatRepository +if TYPE_CHECKING: + from app.modules.chat.repository import ChatRepository @dataclass diff --git a/app/modules/chat/direct_service.py b/app/modules/chat/direct_service.py new file mode 100644 index 0000000..fe6e063 --- /dev/null +++ b/app/modules/chat/direct_service.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import logging +from uuid import uuid4 + +from app.modules.agent.llm import AgentLlmService +from app.modules.chat.evidence_gate import CodeExplainEvidenceGate +from app.modules.chat.session_resolver import ChatSessionResolver +from app.modules.chat.task_store import TaskState, TaskStore +from app.modules.rag.explain import CodeExplainRetrieverV2, PromptBudgeter +from app.schemas.chat import ChatMessageRequest, TaskQueuedResponse, TaskResultType, TaskStatus + +LOGGER = logging.getLogger(__name__) + + +class CodeExplainChatService: + def __init__( + self, + retriever: CodeExplainRetrieverV2, + llm: AgentLlmService, + session_resolver: ChatSessionResolver, + task_store: TaskStore, + message_sink, + budgeter: PromptBudgeter | None = None, + evidence_gate: CodeExplainEvidenceGate | None = None, + ) -> None: + self._retriever = retriever + self._llm = llm + self._session_resolver = session_resolver + self._task_store = task_store + self._message_sink = message_sink + self._budgeter = budgeter or PromptBudgeter() + self._evidence_gate = evidence_gate or CodeExplainEvidenceGate() + + async def handle_message(self, request: ChatMessageRequest) -> TaskQueuedResponse: + dialog_session_id, rag_session_id = self._session_resolver.resolve(request) + task_id = str(uuid4()) + task = TaskState(task_id=task_id, status=TaskStatus.RUNNING) + self._task_store.save(task) + self._message_sink(dialog_session_id, "user", request.message, task_id=task_id) + pack = self._retriever.build_pack( + rag_session_id, + request.message, + file_candidates=[item.model_dump(mode="json") for item in request.files], + ) + decision = self._evidence_gate.evaluate(pack) + if decision.passed: + prompt_input = self._budgeter.build_prompt_input(request.message, pack) + answer = self._llm.generate( + "code_explain_answer_v2", + prompt_input, + log_context="chat.code_explain.direct", + ).strip() + else: + answer = decision.answer + self._message_sink(dialog_session_id, "assistant", answer, task_id=task_id) + task.status = TaskStatus.DONE + task.result_type = TaskResultType.ANSWER + task.answer = answer + self._task_store.save(task) + LOGGER.warning( + "direct code explain response: task_id=%s rag_session_id=%s excerpts=%s missing=%s", + task_id, + rag_session_id, + len(pack.code_excerpts), + pack.missing, + ) + return TaskQueuedResponse( + task_id=task_id, + status=TaskStatus.DONE.value, + ) diff --git a/app/modules/chat/evidence_gate.py b/app/modules/chat/evidence_gate.py new file mode 100644 index 0000000..6d12257 --- /dev/null +++ b/app/modules/chat/evidence_gate.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from dataclasses import dataclass, field + +from app.modules.rag.explain.models import ExplainPack + + +@dataclass(slots=True) +class EvidenceGateDecision: + passed: bool + answer: str = "" + diagnostics: dict[str, list[str]] = field(default_factory=dict) + + +class CodeExplainEvidenceGate: + def __init__(self, min_excerpts: int = 2) -> None: + self._min_excerpts = min_excerpts + + def evaluate(self, pack: ExplainPack) -> EvidenceGateDecision: + diagnostics = self._diagnostics(pack) + if len(pack.code_excerpts) >= self._min_excerpts: + return EvidenceGateDecision(passed=True, diagnostics=diagnostics) + return EvidenceGateDecision( + passed=False, + answer=self._build_answer(pack, diagnostics), + diagnostics=diagnostics, + ) + + def _diagnostics(self, pack: ExplainPack) -> dict[str, list[str]]: + return { + "entrypoints": [item.title for item in pack.selected_entrypoints[:3] if item.title], + "symbols": [item.title for item in pack.seed_symbols[:5] if item.title], + "paths": self._paths(pack), + "missing": list(pack.missing), + } + + def _paths(self, pack: ExplainPack) -> list[str]: + values: list[str] = [] + for item in pack.selected_entrypoints + pack.seed_symbols: + path = item.source or (item.location.path if item.location else "") + if path and path not in values: + values.append(path) + for excerpt in pack.code_excerpts: + if excerpt.path and excerpt.path not in values: + values.append(excerpt.path) + return values[:6] + + def _build_answer(self, pack: ExplainPack, diagnostics: dict[str, list[str]]) -> str: + lines = [ + "Недостаточно опоры в коде, чтобы дать объяснение без догадок.", + "", + f"Найдено фрагментов кода: {len(pack.code_excerpts)} из {self._min_excerpts} минимально необходимых.", + ] + if diagnostics["paths"]: + lines.append(f"Пути: {', '.join(diagnostics['paths'])}") + if diagnostics["entrypoints"]: + lines.append(f"Entrypoints: {', '.join(diagnostics['entrypoints'])}") + if diagnostics["symbols"]: + lines.append(f"Символы: {', '.join(diagnostics['symbols'])}") + if diagnostics["missing"]: + lines.append(f"Диагностика: {', '.join(diagnostics['missing'])}") + return "\n".join(lines).strip() diff --git a/app/modules/chat/module.py b/app/modules/chat/module.py index 10c8964..e967bb7 100644 --- a/app/modules/chat/module.py +++ b/app/modules/chat/module.py @@ -1,13 +1,16 @@ +from __future__ import annotations + +import os +from typing import TYPE_CHECKING + from fastapi import APIRouter, Header from fastapi.responses import StreamingResponse from app.core.exceptions import AppError +from app.modules.chat.direct_service import CodeExplainChatService from app.modules.chat.dialog_store import DialogSessionStore -from app.modules.chat.repository import ChatRepository from app.modules.chat.service import ChatOrchestrator from app.modules.chat.task_store import TaskStore -from app.modules.contracts import AgentRunner -from app.modules.rag_session.session_store import RagSessionStore from app.modules.shared.event_bus import EventBus from app.modules.shared.idempotency_store import IdempotencyStore from app.modules.shared.retry_executor import RetryExecutor @@ -20,6 +23,11 @@ from app.schemas.chat import ( ) from app.schemas.common import ModuleName +if TYPE_CHECKING: + from app.modules.chat.repository import ChatRepository + from app.modules.contracts import AgentRunner + from app.modules.rag_session.session_store import RagSessionStore + class ChatModule: def __init__( @@ -29,12 +37,16 @@ class ChatModule: retry: RetryExecutor, rag_sessions: RagSessionStore, repository: ChatRepository, + direct_chat: CodeExplainChatService | None = None, + task_store: TaskStore | None = None, ) -> None: self._rag_sessions = rag_sessions - self.tasks = TaskStore() + self._simple_code_explain_only = os.getenv("SIMPLE_CODE_EXPLAIN_ONLY", "true").lower() in {"1", "true", "yes"} + self.tasks = task_store or TaskStore() self.dialogs = DialogSessionStore(repository) self.idempotency = IdempotencyStore() self.events = event_bus + self.direct_chat = direct_chat self.chat = ChatOrchestrator( task_store=self.tasks, dialogs=self.dialogs, @@ -59,11 +71,13 @@ class ChatModule: rag_session_id=dialog.rag_session_id, ) - @router.post("/api/chat/messages", response_model=TaskQueuedResponse) + @router.post("/api/chat/messages", response_model=TaskQueuedResponse | TaskResultResponse) async def send_message( request: ChatMessageRequest, idempotency_key: str | None = Header(default=None, alias="Idempotency-Key"), - ) -> TaskQueuedResponse: + ) -> TaskQueuedResponse | TaskResultResponse: + if self._simple_code_explain_only and self.direct_chat is not None: + return await self.direct_chat.handle_message(request) task = await self.chat.enqueue_message(request, idempotency_key) return TaskQueuedResponse(task_id=task.task_id, status=task.status.value) diff --git a/app/modules/chat/service.py b/app/modules/chat/service.py index 2ae6277..abaf539 100644 --- a/app/modules/chat/service.py +++ b/app/modules/chat/service.py @@ -6,6 +6,7 @@ from app.modules.contracts import AgentRunner from app.schemas.chat import ChatMessageRequest, TaskResultType, TaskStatus from app.schemas.common import ErrorPayload, ModuleName from app.modules.chat.dialog_store import DialogSessionStore +from app.modules.chat.session_resolver import ChatSessionResolver from app.modules.chat.task_store import TaskState, TaskStore from app.modules.shared.event_bus import EventBus from app.modules.shared.idempotency_store import IdempotencyStore @@ -41,6 +42,7 @@ class ChatOrchestrator: self._retry = retry self._rag_session_exists = rag_session_exists self._message_sink = message_sink + self._session_resolver = ChatSessionResolver(dialogs, rag_session_exists) async def enqueue_message( self, @@ -52,7 +54,7 @@ class ChatOrchestrator: if existing: task = self._task_store.get(existing) if task: - LOGGER.warning( + LOGGER.info( "enqueue_message reused task by idempotency key: task_id=%s mode=%s", task.task_id, request.mode.value, @@ -63,7 +65,7 @@ class ChatOrchestrator: if idempotency_key: self._idempotency.put(idempotency_key, task.task_id) asyncio.create_task(self._process_task(task.task_id, request)) - LOGGER.warning( + LOGGER.info( "enqueue_message created task: task_id=%s mode=%s", task.task_id, request.mode.value, @@ -135,6 +137,13 @@ class ChatOrchestrator: task.changeset = result.changeset if task.result_type == TaskResultType.ANSWER and task.answer: self._message_sink(dialog_session_id, "assistant", task.answer, task_id=task_id) + LOGGER.warning( + "outgoing chat response: task_id=%s dialog_session_id=%s result_type=%s answer=%s", + task_id, + dialog_session_id, + task.result_type.value, + _truncate_for_log(task.answer), + ) elif task.result_type == TaskResultType.CHANGESET: self._message_sink( dialog_session_id, @@ -146,6 +155,14 @@ class ChatOrchestrator: "changeset": [item.model_dump(mode="json") for item in task.changeset], }, ) + LOGGER.warning( + "outgoing chat response: task_id=%s dialog_session_id=%s result_type=%s changeset_items=%s answer=%s", + task_id, + dialog_session_id, + task.result_type.value, + len(task.changeset), + _truncate_for_log(task.answer or ""), + ) self._task_store.save(task) await self._events.publish( task_id, @@ -160,7 +177,7 @@ class ChatOrchestrator: }, ) await self._publish_progress(task_id, "task.done", "Обработка завершена.", progress=100) - LOGGER.warning( + LOGGER.info( "_process_task completed: task_id=%s status=%s result_type=%s changeset_items=%s", task_id, task.status.value, @@ -232,7 +249,7 @@ class ChatOrchestrator: if progress is not None: payload["progress"] = max(0, min(100, int(progress))) await self._events.publish(task_id, kind, payload) - LOGGER.warning( + LOGGER.debug( "_publish_progress emitted: task_id=%s kind=%s stage=%s progress=%s", task_id, kind, @@ -259,35 +276,7 @@ class ChatOrchestrator: meta={"heartbeat": True}, ) index += 1 - LOGGER.warning("_run_heartbeat stopped: task_id=%s ticks=%s", task_id, index) + LOGGER.debug("_run_heartbeat stopped: task_id=%s ticks=%s", task_id, index) def _resolve_sessions(self, request: ChatMessageRequest) -> tuple[str, str]: - # Legacy compatibility: old session_id/project_id flow. - if request.dialog_session_id and request.rag_session_id: - dialog = self._dialogs.get(request.dialog_session_id) - if not dialog: - raise AppError("dialog_not_found", "Dialog session not found", ModuleName.BACKEND) - if dialog.rag_session_id != request.rag_session_id: - raise AppError("dialog_rag_mismatch", "Dialog session does not belong to rag session", ModuleName.BACKEND) - LOGGER.warning( - "_resolve_sessions resolved by dialog_session_id: dialog_session_id=%s rag_session_id=%s", - request.dialog_session_id, - request.rag_session_id, - ) - return request.dialog_session_id, request.rag_session_id - - if request.session_id and request.project_id: - if not self._rag_session_exists(request.project_id): - raise AppError("rag_session_not_found", "RAG session not found", ModuleName.RAG) - LOGGER.warning( - "_resolve_sessions resolved by legacy session/project: session_id=%s project_id=%s", - request.session_id, - request.project_id, - ) - return request.session_id, request.project_id - - raise AppError( - "missing_sessions", - "dialog_session_id and rag_session_id are required", - ModuleName.BACKEND, - ) + return self._session_resolver.resolve(request) diff --git a/app/modules/chat/session_resolver.py b/app/modules/chat/session_resolver.py new file mode 100644 index 0000000..653523b --- /dev/null +++ b/app/modules/chat/session_resolver.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from app.core.exceptions import AppError +from app.schemas.chat import ChatMessageRequest +from app.schemas.common import ModuleName + +if TYPE_CHECKING: + from app.modules.chat.dialog_store import DialogSessionStore + + +class ChatSessionResolver: + def __init__(self, dialogs: DialogSessionStore, rag_session_exists) -> None: + self._dialogs = dialogs + self._rag_session_exists = rag_session_exists + + def resolve(self, request: ChatMessageRequest) -> tuple[str, str]: + if request.dialog_session_id and request.rag_session_id: + dialog = self._dialogs.get(request.dialog_session_id) + if not dialog: + raise AppError("dialog_not_found", "Dialog session not found", ModuleName.BACKEND) + if dialog.rag_session_id != request.rag_session_id: + raise AppError("dialog_rag_mismatch", "Dialog session does not belong to rag session", ModuleName.BACKEND) + return request.dialog_session_id, request.rag_session_id + + if request.session_id and request.project_id: + if not self._rag_session_exists(request.project_id): + raise AppError("rag_session_not_found", "RAG session not found", ModuleName.RAG) + return request.session_id, request.project_id + + raise AppError( + "missing_sessions", + "dialog_session_id and rag_session_id are required", + ModuleName.BACKEND, + ) diff --git a/app/modules/rag/README.md b/app/modules/rag/README.md index a30afbc..e0b9c35 100644 --- a/app/modules/rag/README.md +++ b/app/modules/rag/README.md @@ -90,6 +90,41 @@ sequenceDiagram Rag-->>Agent: items ``` +### Retrieval + project/qa reasoning +Назначение: `RAG` вызывается не в начале runtime, а внутри отдельного graph-шага `context_retrieval` для `project/qa`. +```mermaid +sequenceDiagram + participant Agent as GraphAgentRuntime + participant Orch as OrchestratorService + participant G1 as conversation_understanding + participant G2 as question_classification + participant G3 as context_retrieval + participant Rag as RagService + participant G4 as context_analysis + participant G5 as answer_composition + + Agent->>Orch: run(task) + Orch->>G1: execute + G1-->>Orch: resolved_request + Orch->>G2: execute + G2-->>Orch: question_profile + Orch->>G3: execute + G3->>Rag: retrieve(query) + Rag-->>G3: rag_items + G3-->>Orch: source_bundle + Orch->>G4: execute + G4-->>Orch: analysis_brief + Orch->>G5: execute + G5-->>Orch: final_answer + Orch-->>Agent: final_answer +``` + +Для `project/qa` это означает: +- ранний глобальный retrieval больше не нужен; +- `RAG` возвращает записи только для конкретного шага `context_retrieval`; +- оркестратор управляет цепочкой graph-шагов; +- пользовательский ответ собирается после анализа, а не напрямую из сырого retrieval. + ## 5. Слои, фиксируемые в RAG ### 5.1. Слои DOCS diff --git a/app/modules/rag/explain/__init__.py b/app/modules/rag/explain/__init__.py new file mode 100644 index 0000000..de44c1d --- /dev/null +++ b/app/modules/rag/explain/__init__.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from importlib import import_module + +__all__ = [ + "CodeExcerpt", + "CodeExplainRetrieverV2", + "CodeGraphRepository", + "EvidenceItem", + "ExplainIntent", + "ExplainIntentBuilder", + "ExplainPack", + "LayeredRetrievalGateway", + "PromptBudgeter", + "TracePath", +] + + +def __getattr__(name: str): + module_map = { + "CodeExcerpt": "app.modules.rag.explain.models", + "EvidenceItem": "app.modules.rag.explain.models", + "ExplainIntent": "app.modules.rag.explain.models", + "ExplainPack": "app.modules.rag.explain.models", + "TracePath": "app.modules.rag.explain.models", + "ExplainIntentBuilder": "app.modules.rag.explain.intent_builder", + "PromptBudgeter": "app.modules.rag.explain.budgeter", + "LayeredRetrievalGateway": "app.modules.rag.explain.layered_gateway", + "CodeGraphRepository": "app.modules.rag.explain.graph_repository", + "CodeExplainRetrieverV2": "app.modules.rag.explain.retriever_v2", + } + module_name = module_map.get(name) + if module_name is None: + raise AttributeError(name) + module = import_module(module_name) + return getattr(module, name) diff --git a/app/modules/rag/explain/__pycache__/__init__.cpython-312.pyc b/app/modules/rag/explain/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..5f717da Binary files /dev/null and b/app/modules/rag/explain/__pycache__/__init__.cpython-312.pyc differ diff --git a/app/modules/rag/explain/__pycache__/budgeter.cpython-312.pyc b/app/modules/rag/explain/__pycache__/budgeter.cpython-312.pyc new file mode 100644 index 0000000..3953cd8 Binary files /dev/null and b/app/modules/rag/explain/__pycache__/budgeter.cpython-312.pyc differ diff --git a/app/modules/rag/explain/__pycache__/excerpt_planner.cpython-312.pyc b/app/modules/rag/explain/__pycache__/excerpt_planner.cpython-312.pyc new file mode 100644 index 0000000..3c24cfe Binary files /dev/null and b/app/modules/rag/explain/__pycache__/excerpt_planner.cpython-312.pyc differ diff --git a/app/modules/rag/explain/__pycache__/graph_repository.cpython-312.pyc b/app/modules/rag/explain/__pycache__/graph_repository.cpython-312.pyc new file mode 100644 index 0000000..7fe95d4 Binary files /dev/null and b/app/modules/rag/explain/__pycache__/graph_repository.cpython-312.pyc differ diff --git a/app/modules/rag/explain/__pycache__/intent_builder.cpython-312.pyc b/app/modules/rag/explain/__pycache__/intent_builder.cpython-312.pyc new file mode 100644 index 0000000..2fad4aa Binary files /dev/null and b/app/modules/rag/explain/__pycache__/intent_builder.cpython-312.pyc differ diff --git a/app/modules/rag/explain/__pycache__/layered_gateway.cpython-312.pyc b/app/modules/rag/explain/__pycache__/layered_gateway.cpython-312.pyc new file mode 100644 index 0000000..4ec6b1a Binary files /dev/null and b/app/modules/rag/explain/__pycache__/layered_gateway.cpython-312.pyc differ diff --git a/app/modules/rag/explain/__pycache__/models.cpython-312.pyc b/app/modules/rag/explain/__pycache__/models.cpython-312.pyc new file mode 100644 index 0000000..78c4126 Binary files /dev/null and b/app/modules/rag/explain/__pycache__/models.cpython-312.pyc differ diff --git a/app/modules/rag/explain/__pycache__/retriever_v2.cpython-312.pyc b/app/modules/rag/explain/__pycache__/retriever_v2.cpython-312.pyc new file mode 100644 index 0000000..d59a350 Binary files /dev/null and b/app/modules/rag/explain/__pycache__/retriever_v2.cpython-312.pyc differ diff --git a/app/modules/rag/explain/__pycache__/source_excerpt_fetcher.cpython-312.pyc b/app/modules/rag/explain/__pycache__/source_excerpt_fetcher.cpython-312.pyc new file mode 100644 index 0000000..1a39fb0 Binary files /dev/null and b/app/modules/rag/explain/__pycache__/source_excerpt_fetcher.cpython-312.pyc differ diff --git a/app/modules/rag/explain/__pycache__/trace_builder.cpython-312.pyc b/app/modules/rag/explain/__pycache__/trace_builder.cpython-312.pyc new file mode 100644 index 0000000..28cce2b Binary files /dev/null and b/app/modules/rag/explain/__pycache__/trace_builder.cpython-312.pyc differ diff --git a/app/modules/rag/explain/budgeter.py b/app/modules/rag/explain/budgeter.py new file mode 100644 index 0000000..adcddfd --- /dev/null +++ b/app/modules/rag/explain/budgeter.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import json + +from app.modules.rag.explain.models import ExplainPack + + +class PromptBudgeter: + def __init__( + self, + *, + max_paths: int = 3, + max_symbols: int = 25, + max_excerpts: int = 40, + max_chars: int = 30000, + ) -> None: + self._max_paths = max_paths + self._max_symbols = max_symbols + self._max_excerpts = max_excerpts + self._max_chars = max_chars + + def build_prompt_input(self, question: str, pack: ExplainPack) -> str: + symbol_ids: list[str] = [] + for path in pack.trace_paths[: self._max_paths]: + for symbol_id in path.symbol_ids: + if symbol_id and symbol_id not in symbol_ids and len(symbol_ids) < self._max_symbols: + symbol_ids.append(symbol_id) + excerpts = [] + total_chars = 0 + for excerpt in pack.code_excerpts: + if symbol_ids and excerpt.symbol_id and excerpt.symbol_id not in symbol_ids: + continue + body = excerpt.content.strip() + remaining = self._max_chars - total_chars + if remaining <= 0 or len(excerpts) >= self._max_excerpts: + break + if len(body) > remaining: + body = body[:remaining].rstrip() + "...[truncated]" + excerpts.append( + { + "evidence_id": excerpt.evidence_id, + "title": excerpt.title, + "path": excerpt.path, + "start_line": excerpt.start_line, + "end_line": excerpt.end_line, + "focus": excerpt.focus, + "content": body, + } + ) + total_chars += len(body) + payload = { + "question": question, + "intent": pack.intent.model_dump(mode="json"), + "selected_entrypoints": [item.model_dump(mode="json") for item in pack.selected_entrypoints[:5]], + "seed_symbols": [item.model_dump(mode="json") for item in pack.seed_symbols[: self._max_symbols]], + "trace_paths": [path.model_dump(mode="json") for path in pack.trace_paths[: self._max_paths]], + "evidence_index": {key: value.model_dump(mode="json") for key, value in pack.evidence_index.items()}, + "code_excerpts": excerpts, + "missing": pack.missing, + "conflicts": pack.conflicts, + } + return json.dumps(payload, ensure_ascii=False, indent=2) diff --git a/app/modules/rag/explain/excerpt_planner.py b/app/modules/rag/explain/excerpt_planner.py new file mode 100644 index 0000000..04f98ba --- /dev/null +++ b/app/modules/rag/explain/excerpt_planner.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from app.modules.rag.explain.models import CodeExcerpt, LayeredRetrievalItem + + +class ExcerptPlanner: + _FOCUS_TOKENS = ("raise", "except", "db", "select", "insert", "update", "delete", "http", "publish", "emit") + + def plan(self, chunk: LayeredRetrievalItem, *, evidence_id: str, symbol_id: str | None) -> list[CodeExcerpt]: + location = chunk.location + if location is None: + return [] + excerpts = [ + CodeExcerpt( + evidence_id=evidence_id, + symbol_id=symbol_id, + title=chunk.title, + path=location.path, + start_line=location.start_line, + end_line=location.end_line, + content=chunk.content.strip(), + focus="overview", + ) + ] + focus = self._focus_excerpt(chunk, evidence_id=evidence_id, symbol_id=symbol_id) + if focus is not None: + excerpts.append(focus) + return excerpts + + def _focus_excerpt( + self, + chunk: LayeredRetrievalItem, + *, + evidence_id: str, + symbol_id: str | None, + ) -> CodeExcerpt | None: + location = chunk.location + if location is None: + return None + lines = chunk.content.splitlines() + for index, line in enumerate(lines): + lowered = line.lower() + if not any(token in lowered for token in self._FOCUS_TOKENS): + continue + start = max(0, index - 2) + end = min(len(lines), index + 3) + if end - start >= len(lines): + return None + return CodeExcerpt( + evidence_id=evidence_id, + symbol_id=symbol_id, + title=f"{chunk.title}:focus", + path=location.path, + start_line=(location.start_line or 1) + start, + end_line=(location.start_line or 1) + end - 1, + content="\n".join(lines[start:end]).strip(), + focus="focus", + ) + return None diff --git a/app/modules/rag/explain/graph_repository.py b/app/modules/rag/explain/graph_repository.py new file mode 100644 index 0000000..65d40db --- /dev/null +++ b/app/modules/rag/explain/graph_repository.py @@ -0,0 +1,216 @@ +from __future__ import annotations + +import json + +from sqlalchemy import text + +from app.modules.rag.explain.models import CodeLocation, LayeredRetrievalItem +from app.modules.shared.db import get_engine + + +class CodeGraphRepository: + def get_out_edges( + self, + rag_session_id: str, + src_symbol_ids: list[str], + edge_types: list[str], + limit_per_src: int, + ) -> list[LayeredRetrievalItem]: + if not src_symbol_ids: + return [] + sql = """ + SELECT path, content, layer, title, metadata_json, span_start, span_end + FROM rag_chunks + WHERE rag_session_id = :sid + AND layer = 'C2_DEPENDENCY_GRAPH' + AND CAST(metadata_json AS jsonb)->>'src_symbol_id' = ANY(:src_ids) + AND CAST(metadata_json AS jsonb)->>'edge_type' = ANY(:edge_types) + ORDER BY path, span_start + """ + with get_engine().connect() as conn: + rows = conn.execute( + text(sql), + {"sid": rag_session_id, "src_ids": src_symbol_ids, "edge_types": edge_types}, + ).mappings().fetchall() + grouped: dict[str, int] = {} + items: list[LayeredRetrievalItem] = [] + for row in rows: + metadata = self._loads(row.get("metadata_json")) + src_symbol_id = str(metadata.get("src_symbol_id") or "") + grouped[src_symbol_id] = grouped.get(src_symbol_id, 0) + 1 + if grouped[src_symbol_id] > limit_per_src: + continue + items.append(self._to_item(row, metadata)) + return items + + def get_in_edges( + self, + rag_session_id: str, + dst_symbol_ids: list[str], + edge_types: list[str], + limit_per_dst: int, + ) -> list[LayeredRetrievalItem]: + if not dst_symbol_ids: + return [] + sql = """ + SELECT path, content, layer, title, metadata_json, span_start, span_end + FROM rag_chunks + WHERE rag_session_id = :sid + AND layer = 'C2_DEPENDENCY_GRAPH' + AND CAST(metadata_json AS jsonb)->>'dst_symbol_id' = ANY(:dst_ids) + AND CAST(metadata_json AS jsonb)->>'edge_type' = ANY(:edge_types) + ORDER BY path, span_start + """ + with get_engine().connect() as conn: + rows = conn.execute( + text(sql), + {"sid": rag_session_id, "dst_ids": dst_symbol_ids, "edge_types": edge_types}, + ).mappings().fetchall() + grouped: dict[str, int] = {} + items: list[LayeredRetrievalItem] = [] + for row in rows: + metadata = self._loads(row.get("metadata_json")) + dst_symbol_id = str(metadata.get("dst_symbol_id") or "") + grouped[dst_symbol_id] = grouped.get(dst_symbol_id, 0) + 1 + if grouped[dst_symbol_id] > limit_per_dst: + continue + items.append(self._to_item(row, metadata)) + return items + + def resolve_symbol_by_ref( + self, + rag_session_id: str, + dst_ref: str, + package_hint: str | None = None, + ) -> LayeredRetrievalItem | None: + ref = (dst_ref or "").strip() + if not ref: + return None + with get_engine().connect() as conn: + rows = conn.execute( + text( + """ + SELECT path, content, layer, title, metadata_json, span_start, span_end, qname + FROM rag_chunks + WHERE rag_session_id = :sid + AND layer = 'C1_SYMBOL_CATALOG' + AND (qname = :ref OR title = :ref OR qname LIKE :tail) + ORDER BY path + LIMIT 12 + """ + ), + {"sid": rag_session_id, "ref": ref, "tail": f"%{ref}"}, + ).mappings().fetchall() + best: LayeredRetrievalItem | None = None + best_score = -1 + for row in rows: + metadata = self._loads(row.get("metadata_json")) + package = str(metadata.get("package_or_module") or "") + score = 0 + if str(row.get("qname") or "") == ref: + score += 3 + if str(row.get("title") or "") == ref: + score += 2 + if package_hint and package.startswith(package_hint): + score += 3 + if package_hint and package_hint in str(row.get("path") or ""): + score += 1 + if score > best_score: + best = self._to_item(row, metadata) + best_score = score + return best + + def get_symbols_by_ids(self, rag_session_id: str, symbol_ids: list[str]) -> list[LayeredRetrievalItem]: + if not symbol_ids: + return [] + with get_engine().connect() as conn: + rows = conn.execute( + text( + """ + SELECT path, content, layer, title, metadata_json, span_start, span_end + FROM rag_chunks + WHERE rag_session_id = :sid + AND layer = 'C1_SYMBOL_CATALOG' + AND symbol_id = ANY(:symbol_ids) + ORDER BY path, span_start + """ + ), + {"sid": rag_session_id, "symbol_ids": symbol_ids}, + ).mappings().fetchall() + return [self._to_item(row, self._loads(row.get("metadata_json"))) for row in rows] + + def get_chunks_by_symbol_ids( + self, + rag_session_id: str, + symbol_ids: list[str], + prefer_chunk_type: str = "symbol_block", + ) -> list[LayeredRetrievalItem]: + symbols = self.get_symbols_by_ids(rag_session_id, symbol_ids) + chunks: list[LayeredRetrievalItem] = [] + for symbol in symbols: + location = symbol.location + if location is None: + continue + chunk = self._chunk_for_symbol(rag_session_id, symbol, prefer_chunk_type=prefer_chunk_type) + if chunk is not None: + chunks.append(chunk) + return chunks + + def _chunk_for_symbol( + self, + rag_session_id: str, + symbol: LayeredRetrievalItem, + *, + prefer_chunk_type: str, + ) -> LayeredRetrievalItem | None: + location = symbol.location + if location is None: + return None + with get_engine().connect() as conn: + rows = conn.execute( + text( + """ + SELECT path, content, layer, title, metadata_json, span_start, span_end + FROM rag_chunks + WHERE rag_session_id = :sid + AND layer = 'C0_SOURCE_CHUNKS' + AND path = :path + AND COALESCE(span_start, 0) <= :end_line + AND COALESCE(span_end, 999999) >= :start_line + ORDER BY + CASE WHEN CAST(metadata_json AS jsonb)->>'chunk_type' = :prefer_chunk_type THEN 0 ELSE 1 END, + ABS(COALESCE(span_start, 0) - :start_line) + LIMIT 1 + """ + ), + { + "sid": rag_session_id, + "path": location.path, + "start_line": location.start_line or 0, + "end_line": location.end_line or 999999, + "prefer_chunk_type": prefer_chunk_type, + }, + ).mappings().fetchall() + if not rows: + return None + row = rows[0] + return self._to_item(row, self._loads(row.get("metadata_json"))) + + def _to_item(self, row, metadata: dict) -> LayeredRetrievalItem: + return LayeredRetrievalItem( + source=str(row.get("path") or ""), + content=str(row.get("content") or ""), + layer=str(row.get("layer") or ""), + title=str(row.get("title") or ""), + metadata=metadata, + location=CodeLocation( + path=str(row.get("path") or ""), + start_line=row.get("span_start"), + end_line=row.get("span_end"), + ), + ) + + def _loads(self, value) -> dict: + if not value: + return {} + return json.loads(str(value)) diff --git a/app/modules/rag/explain/intent_builder.py b/app/modules/rag/explain/intent_builder.py new file mode 100644 index 0000000..cd4cc3b --- /dev/null +++ b/app/modules/rag/explain/intent_builder.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import re + +from app.modules.rag.explain.models import ExplainHints, ExplainIntent +from app.modules.rag.retrieval.query_terms import extract_query_terms + + +class ExplainIntentBuilder: + _ROUTE_RE = re.compile(r"(/[A-Za-z0-9_./{}:-]+)") + _FILE_RE = re.compile(r"([A-Za-z0-9_./-]+\.py)") + _SYMBOL_RE = re.compile(r"\b([A-Z][A-Za-z0-9_]*\.[A-Za-z_][A-Za-z0-9_]*|[A-Z][A-Za-z0-9_]{2,}|[a-z_][A-Za-z0-9_]{2,})\b") + _COMMAND_RE = re.compile(r"`([A-Za-z0-9:_-]+)`") + _TEST_KEYWORDS = ( + "тест", + "tests", + "test ", + "unit-test", + "unit test", + "юнит-тест", + "pytest", + "spec", + "как покрыто тестами", + "как проверяется", + "how is it tested", + "how it's tested", + ) + + def build(self, user_query: str) -> ExplainIntent: + normalized = " ".join((user_query or "").split()) + lowered = normalized.lower() + keywords = self._keywords(normalized) + hints = ExplainHints( + paths=self._dedupe(self._FILE_RE.findall(normalized)), + symbols=self._symbols(normalized), + endpoints=self._dedupe(self._ROUTE_RE.findall(normalized)), + commands=self._commands(normalized, lowered), + ) + return ExplainIntent( + raw_query=user_query, + normalized_query=normalized, + keywords=keywords[:12], + hints=hints, + include_tests=self._include_tests(lowered), + expected_entry_types=self._entry_types(lowered, hints), + depth=self._depth(lowered), + ) + + def _keywords(self, text: str) -> list[str]: + keywords = extract_query_terms(text) + for token in self._symbols(text): + if token not in keywords: + keywords.append(token) + for token in self._ROUTE_RE.findall(text): + if token not in keywords: + keywords.append(token) + return self._dedupe(keywords) + + def _symbols(self, text: str) -> list[str]: + values = [] + for raw in self._SYMBOL_RE.findall(text): + token = raw.strip() + if len(token) < 3: + continue + if token.endswith(".py"): + continue + values.append(token) + return self._dedupe(values) + + def _commands(self, text: str, lowered: str) -> list[str]: + values = list(self._COMMAND_RE.findall(text)) + if " command " in f" {lowered} ": + values.extend(re.findall(r"command\s+([A-Za-z0-9:_-]+)", lowered)) + if " cli " in f" {lowered} ": + values.extend(re.findall(r"cli\s+([A-Za-z0-9:_-]+)", lowered)) + return self._dedupe(values) + + def _entry_types(self, lowered: str, hints: ExplainHints) -> list[str]: + if hints.endpoints or any(token in lowered for token in ("endpoint", "route", "handler", "http", "api")): + return ["http"] + if hints.commands or any(token in lowered for token in ("cli", "command", "click", "typer")): + return ["cli"] + return ["http", "cli"] + + def _depth(self, lowered: str) -> str: + if any(token in lowered for token in ("deep", "подроб", "деталь", "full flow", "trace")): + return "deep" + if any(token in lowered for token in ("high level", "overview", "кратко", "summary")): + return "high" + return "medium" + + def _include_tests(self, lowered: str) -> bool: + normalized = f" {lowered} " + return any(token in normalized for token in self._TEST_KEYWORDS) + + def _dedupe(self, values: list[str]) -> list[str]: + result: list[str] = [] + for value in values: + item = value.strip() + if item and item not in result: + result.append(item) + return result diff --git a/app/modules/rag/explain/layered_gateway.py b/app/modules/rag/explain/layered_gateway.py new file mode 100644 index 0000000..fb104eb --- /dev/null +++ b/app/modules/rag/explain/layered_gateway.py @@ -0,0 +1,289 @@ +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Callable + +from app.modules.rag.explain.models import CodeLocation, LayeredRetrievalItem +from app.modules.rag.retrieval.test_filter import build_test_filters, debug_disable_test_filter + +LOGGER = logging.getLogger(__name__) + +if TYPE_CHECKING: + from app.modules.rag.persistence.repository import RagRepository + from app.modules.rag_session.embedding.gigachat_embedder import GigaChatEmbedder + + +@dataclass(slots=True) +class LayerRetrievalResult: + items: list[LayeredRetrievalItem] + missing: list[str] = field(default_factory=list) + + +class LayeredRetrievalGateway: + def __init__(self, repository: RagRepository, embedder: GigaChatEmbedder) -> None: + self._repository = repository + self._embedder = embedder + + def retrieve_layer( + self, + rag_session_id: str, + query: str, + layer: str, + *, + limit: int, + path_prefixes: list[str] | None = None, + exclude_tests: bool = True, + prefer_non_tests: bool = False, + include_spans: bool = False, + ) -> LayerRetrievalResult: + effective_exclude_tests = exclude_tests and not debug_disable_test_filter() + filter_args = self._filter_args(effective_exclude_tests) + query_embedding: list[float] | None = None + try: + query_embedding = self._embedder.embed([query])[0] + rows = self._repository.retrieve( + rag_session_id, + query_embedding, + query_text=query, + limit=limit, + layers=[layer], + path_prefixes=path_prefixes, + exclude_path_prefixes=filter_args["exclude_path_prefixes"], + exclude_like_patterns=filter_args["exclude_like_patterns"], + prefer_non_tests=prefer_non_tests or not effective_exclude_tests, + ) + return self._success_result( + rows, + rag_session_id=rag_session_id, + label="layered retrieval", + include_spans=include_spans, + layer=layer, + exclude_tests=effective_exclude_tests, + path_prefixes=path_prefixes, + ) + except Exception as exc: + if query_embedding is None: + self._log_failure( + label="layered retrieval", + rag_session_id=rag_session_id, + layer=layer, + exclude_tests=effective_exclude_tests, + path_prefixes=path_prefixes, + exc=exc, + ) + return LayerRetrievalResult(items=[], missing=[self._failure_missing(f"layer:{layer} retrieval_failed", exc)]) + retry_result = self._retry_without_test_filter( + operation=lambda: self._repository.retrieve( + rag_session_id, + query_embedding, + query_text=query, + limit=limit, + layers=[layer], + path_prefixes=path_prefixes, + exclude_path_prefixes=None, + exclude_like_patterns=None, + prefer_non_tests=True, + ), + label="layered retrieval", + rag_session_id=rag_session_id, + include_spans=include_spans, + layer=layer, + exclude_tests=effective_exclude_tests, + path_prefixes=path_prefixes, + exc=exc, + missing_prefix=f"layer:{layer} retrieval_failed", + ) + if retry_result is not None: + return retry_result + return LayerRetrievalResult(items=[], missing=[self._failure_missing(f"layer:{layer} retrieval_failed", exc)]) + + def retrieve_lexical_code( + self, + rag_session_id: str, + query: str, + *, + limit: int, + path_prefixes: list[str] | None = None, + exclude_tests: bool = True, + include_spans: bool = False, + ) -> LayerRetrievalResult: + effective_exclude_tests = exclude_tests and not debug_disable_test_filter() + filter_args = self._filter_args(effective_exclude_tests) + try: + rows = self._repository.retrieve_lexical_code( + rag_session_id, + query_text=query, + limit=limit, + path_prefixes=path_prefixes, + exclude_path_prefixes=filter_args["exclude_path_prefixes"], + exclude_like_patterns=filter_args["exclude_like_patterns"], + prefer_non_tests=not effective_exclude_tests, + ) + return self._success_result( + rows, + rag_session_id=rag_session_id, + label="lexical retrieval", + include_spans=include_spans, + exclude_tests=effective_exclude_tests, + path_prefixes=path_prefixes, + ) + except Exception as exc: + retry_result = self._retry_without_test_filter( + operation=lambda: self._repository.retrieve_lexical_code( + rag_session_id, + query_text=query, + limit=limit, + path_prefixes=path_prefixes, + exclude_path_prefixes=None, + exclude_like_patterns=None, + prefer_non_tests=True, + ), + label="lexical retrieval", + rag_session_id=rag_session_id, + include_spans=include_spans, + exclude_tests=effective_exclude_tests, + path_prefixes=path_prefixes, + exc=exc, + missing_prefix="layer:C0 lexical_retrieval_failed", + ) + if retry_result is not None: + return retry_result + return LayerRetrievalResult(items=[], missing=[self._failure_missing("layer:C0 lexical_retrieval_failed", exc)]) + + def _retry_without_test_filter( + self, + *, + operation: Callable[[], list[dict]], + label: str, + rag_session_id: str, + include_spans: bool, + exclude_tests: bool, + path_prefixes: list[str] | None, + exc: Exception, + missing_prefix: str, + layer: str | None = None, + ) -> LayerRetrievalResult | None: + if not exclude_tests: + self._log_failure( + label=label, + rag_session_id=rag_session_id, + layer=layer, + exclude_tests=exclude_tests, + path_prefixes=path_prefixes, + exc=exc, + ) + return None + self._log_failure( + label=label, + rag_session_id=rag_session_id, + layer=layer, + exclude_tests=exclude_tests, + path_prefixes=path_prefixes, + exc=exc, + retried_without_test_filter=True, + ) + try: + rows = operation() + except Exception as retry_exc: + self._log_failure( + label=f"{label} retry", + rag_session_id=rag_session_id, + layer=layer, + exclude_tests=False, + path_prefixes=path_prefixes, + exc=retry_exc, + ) + return None + result = self._success_result( + rows, + rag_session_id=rag_session_id, + label=f"{label} retry", + include_spans=include_spans, + layer=layer, + exclude_tests=False, + path_prefixes=path_prefixes, + ) + result.missing.append(f"{missing_prefix}:retried_without_test_filter") + return result + + def _success_result( + self, + rows: list[dict], + *, + rag_session_id: str, + label: str, + include_spans: bool, + exclude_tests: bool, + path_prefixes: list[str] | None, + layer: str | None = None, + ) -> LayerRetrievalResult: + items = [self._to_item(row, include_spans=include_spans) for row in rows] + LOGGER.warning( + "%s: rag_session_id=%s layer=%s exclude_tests=%s path_prefixes=%s returned_count=%s top_paths=%s", + label, + rag_session_id, + layer, + exclude_tests, + path_prefixes or [], + len(items), + [item.source for item in items[:3]], + ) + return LayerRetrievalResult(items=items) + + def _log_failure( + self, + *, + label: str, + rag_session_id: str, + exclude_tests: bool, + path_prefixes: list[str] | None, + exc: Exception, + layer: str | None = None, + retried_without_test_filter: bool = False, + ) -> None: + LOGGER.warning( + "%s failed: rag_session_id=%s layer=%s exclude_tests=%s path_prefixes=%s retried_without_test_filter=%s error=%s", + label, + rag_session_id, + layer, + exclude_tests, + path_prefixes or [], + retried_without_test_filter, + self._exception_summary(exc), + exc_info=True, + ) + + def _filter_args(self, exclude_tests: bool) -> dict[str, list[str] | None]: + test_filters = build_test_filters() if exclude_tests else None + return { + "exclude_path_prefixes": test_filters.exclude_path_prefixes if test_filters else None, + "exclude_like_patterns": test_filters.exclude_like_patterns if test_filters else None, + } + + def _failure_missing(self, prefix: str, exc: Exception) -> str: + return f"{prefix}:{self._exception_summary(exc)}" + + def _exception_summary(self, exc: Exception) -> str: + message = " ".join(str(exc).split()) + if len(message) > 180: + message = message[:177] + "..." + return f"{type(exc).__name__}:{message or 'no_message'}" + + def _to_item(self, row: dict, *, include_spans: bool) -> LayeredRetrievalItem: + location = None + if include_spans: + location = CodeLocation( + path=str(row.get("path") or ""), + start_line=row.get("span_start"), + end_line=row.get("span_end"), + ) + return LayeredRetrievalItem( + source=str(row.get("path") or ""), + content=str(row.get("content") or ""), + layer=str(row.get("layer") or ""), + title=str(row.get("title") or ""), + metadata=dict(row.get("metadata", {}) or {}), + score=row.get("distance"), + location=location, + ) diff --git a/app/modules/rag/explain/models.py b/app/modules/rag/explain/models.py new file mode 100644 index 0000000..90552cd --- /dev/null +++ b/app/modules/rag/explain/models.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field + + +class ExplainHints(BaseModel): + model_config = ConfigDict(extra="forbid") + + paths: list[str] = Field(default_factory=list) + symbols: list[str] = Field(default_factory=list) + endpoints: list[str] = Field(default_factory=list) + commands: list[str] = Field(default_factory=list) + + +class ExplainIntent(BaseModel): + model_config = ConfigDict(extra="forbid") + + raw_query: str + normalized_query: str + keywords: list[str] = Field(default_factory=list) + hints: ExplainHints = Field(default_factory=ExplainHints) + include_tests: bool = False + expected_entry_types: list[Literal["http", "cli"]] = Field(default_factory=list) + depth: Literal["high", "medium", "deep"] = "medium" + + +class CodeLocation(BaseModel): + model_config = ConfigDict(extra="forbid") + + path: str + start_line: int | None = None + end_line: int | None = None + + +class LayeredRetrievalItem(BaseModel): + model_config = ConfigDict(extra="forbid") + + source: str + content: str + layer: str + title: str + metadata: dict[str, Any] = Field(default_factory=dict) + score: float | None = None + location: CodeLocation | None = None + + +class TracePath(BaseModel): + model_config = ConfigDict(extra="forbid") + + symbol_ids: list[str] = Field(default_factory=list) + score: float = 0.0 + entrypoint_id: str | None = None + notes: list[str] = Field(default_factory=list) + + +class EvidenceItem(BaseModel): + model_config = ConfigDict(extra="forbid") + + evidence_id: str + kind: Literal["entrypoint", "symbol", "edge", "excerpt"] + summary: str + location: CodeLocation | None = None + supports: list[str] = Field(default_factory=list) + + +class CodeExcerpt(BaseModel): + model_config = ConfigDict(extra="forbid") + + evidence_id: str + symbol_id: str | None = None + title: str + path: str + start_line: int | None = None + end_line: int | None = None + content: str + focus: str = "overview" + + +class ExplainPack(BaseModel): + model_config = ConfigDict(extra="forbid") + + intent: ExplainIntent + selected_entrypoints: list[LayeredRetrievalItem] = Field(default_factory=list) + seed_symbols: list[LayeredRetrievalItem] = Field(default_factory=list) + trace_paths: list[TracePath] = Field(default_factory=list) + evidence_index: dict[str, EvidenceItem] = Field(default_factory=dict) + code_excerpts: list[CodeExcerpt] = Field(default_factory=list) + missing: list[str] = Field(default_factory=list) + conflicts: list[str] = Field(default_factory=list) diff --git a/app/modules/rag/explain/retriever_v2.py b/app/modules/rag/explain/retriever_v2.py new file mode 100644 index 0000000..cf31820 --- /dev/null +++ b/app/modules/rag/explain/retriever_v2.py @@ -0,0 +1,328 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +from app.modules.rag.contracts.enums import RagLayer +from app.modules.rag.explain.intent_builder import ExplainIntentBuilder +from app.modules.rag.explain.layered_gateway import LayerRetrievalResult, LayeredRetrievalGateway +from app.modules.rag.explain.models import CodeExcerpt, EvidenceItem, ExplainPack, LayeredRetrievalItem +from app.modules.rag.explain.source_excerpt_fetcher import SourceExcerptFetcher +from app.modules.rag.explain.trace_builder import TraceBuilder +from app.modules.rag.retrieval.test_filter import exclude_tests_default, is_test_path + +LOGGER = logging.getLogger(__name__) +_MIN_EXCERPTS = 2 + +if TYPE_CHECKING: + from app.modules.rag.explain.graph_repository import CodeGraphRepository + from app.modules.rag.explain.models import ExplainIntent + + +class CodeExplainRetrieverV2: + def __init__( + self, + gateway: LayeredRetrievalGateway, + graph_repository: CodeGraphRepository, + intent_builder: ExplainIntentBuilder | None = None, + trace_builder: TraceBuilder | None = None, + excerpt_fetcher: SourceExcerptFetcher | None = None, + ) -> None: + self._gateway = gateway + self._graph = graph_repository + self._intent_builder = intent_builder or ExplainIntentBuilder() + self._trace_builder = trace_builder or TraceBuilder(graph_repository) + self._excerpt_fetcher = excerpt_fetcher or SourceExcerptFetcher(graph_repository) + + def build_pack( + self, + rag_session_id: str, + user_query: str, + *, + file_candidates: list[dict] | None = None, + ) -> ExplainPack: + intent = self._intent_builder.build(user_query) + path_prefixes = _path_prefixes(intent, file_candidates or []) + exclude_tests = exclude_tests_default() and not intent.include_tests + pack = self._run_pass(rag_session_id, intent, path_prefixes, exclude_tests=exclude_tests) + if exclude_tests and len(pack.code_excerpts) < _MIN_EXCERPTS: + self._merge_test_fallback(pack, rag_session_id, intent, path_prefixes) + self._log_pack(rag_session_id, pack) + return pack + + def _run_pass( + self, + rag_session_id: str, + intent: ExplainIntent, + path_prefixes: list[str], + *, + exclude_tests: bool, + ) -> ExplainPack: + missing: list[str] = [] + entrypoints_result = self._entrypoints(rag_session_id, intent, path_prefixes, exclude_tests=exclude_tests) + missing.extend(entrypoints_result.missing) + selected_entrypoints = self._filter_entrypoints(intent, entrypoints_result.items) + if not selected_entrypoints: + missing.append("layer:C3 empty") + seed_result = self._seed_symbols(rag_session_id, intent, path_prefixes, selected_entrypoints, exclude_tests=exclude_tests) + missing.extend(seed_result.missing) + seed_symbols = seed_result.items + if not seed_symbols: + missing.append("layer:C1 empty") + depth = 4 if intent.depth == "deep" else 3 if intent.depth == "medium" else 2 + trace_paths = self._trace_builder.build_paths(rag_session_id, seed_symbols, max_depth=depth) if seed_symbols else [] + excerpts, excerpt_evidence = self._excerpt_fetcher.fetch(rag_session_id, trace_paths) if trace_paths else ([], {}) + if not excerpts: + lexical_result = self._gateway.retrieve_lexical_code( + rag_session_id, + intent.normalized_query, + limit=6, + path_prefixes=path_prefixes or None, + exclude_tests=exclude_tests, + include_spans=True, + ) + missing.extend(lexical_result.missing) + excerpts, excerpt_evidence = _lexical_excerpts(lexical_result.items) + if not excerpts: + missing.append("layer:C0 empty") + evidence_index = _evidence_index(selected_entrypoints, seed_symbols) + evidence_index.update(excerpt_evidence) + missing.extend(_missing(selected_entrypoints, seed_symbols, trace_paths, excerpts)) + return ExplainPack( + intent=intent, + selected_entrypoints=selected_entrypoints, + seed_symbols=seed_symbols, + trace_paths=trace_paths, + evidence_index=evidence_index, + code_excerpts=excerpts, + missing=_cleanup_missing(_dedupe(missing), has_excerpts=bool(excerpts)), + conflicts=[], + ) + + def _merge_test_fallback( + self, + pack: ExplainPack, + rag_session_id: str, + intent: ExplainIntent, + path_prefixes: list[str], + ) -> None: + lexical_result = self._gateway.retrieve_lexical_code( + rag_session_id, + intent.normalized_query, + limit=6, + path_prefixes=path_prefixes or None, + exclude_tests=False, + include_spans=True, + ) + excerpt_offset = len([key for key in pack.evidence_index if key.startswith("excerpt_")]) + excerpts, evidence = _lexical_excerpts( + lexical_result.items, + start_index=excerpt_offset, + is_test_fallback=True, + ) + if not excerpts: + pack.missing = _dedupe(pack.missing + lexical_result.missing) + return + seen = {(item.path, item.start_line, item.end_line, item.content) for item in pack.code_excerpts} + for excerpt in excerpts: + key = (excerpt.path, excerpt.start_line, excerpt.end_line, excerpt.content) + if key in seen: + continue + pack.code_excerpts.append(excerpt) + seen.add(key) + pack.evidence_index.update(evidence) + pack.missing = _cleanup_missing(_dedupe(pack.missing + lexical_result.missing), has_excerpts=bool(pack.code_excerpts)) + + def _entrypoints( + self, + rag_session_id: str, + intent: ExplainIntent, + path_prefixes: list[str], + *, + exclude_tests: bool, + ) -> LayerRetrievalResult: + return self._gateway.retrieve_layer( + rag_session_id, + intent.normalized_query, + RagLayer.CODE_ENTRYPOINTS, + limit=6, + path_prefixes=path_prefixes or None, + exclude_tests=exclude_tests, + prefer_non_tests=True, + include_spans=True, + ) + + def _filter_entrypoints(self, intent: ExplainIntent, items: list[LayeredRetrievalItem]) -> list[LayeredRetrievalItem]: + if not intent.expected_entry_types: + return items[:3] + filtered = [item for item in items if str(item.metadata.get("entry_type") or "") in intent.expected_entry_types] + return filtered[:3] or items[:3] + + def _seed_symbols( + self, + rag_session_id: str, + intent: ExplainIntent, + path_prefixes: list[str], + entrypoints: list[LayeredRetrievalItem], + *, + exclude_tests: bool, + ) -> LayerRetrievalResult: + symbol_result = self._gateway.retrieve_layer( + rag_session_id, + intent.normalized_query, + RagLayer.CODE_SYMBOL_CATALOG, + limit=12, + path_prefixes=path_prefixes or None, + exclude_tests=exclude_tests, + prefer_non_tests=True, + include_spans=True, + ) + handlers: list[LayeredRetrievalItem] = [] + handler_ids = [str(item.metadata.get("handler_symbol_id") or "") for item in entrypoints] + if handler_ids: + handlers = self._graph.get_symbols_by_ids(rag_session_id, [item for item in handler_ids if item]) + seeds: list[LayeredRetrievalItem] = [] + seen: set[str] = set() + for item in handlers + symbol_result.items: + symbol_id = str(item.metadata.get("symbol_id") or "") + if not symbol_id or symbol_id in seen: + continue + seen.add(symbol_id) + seeds.append(item) + if len(seeds) >= 8: + break + return LayerRetrievalResult(items=seeds, missing=list(symbol_result.missing)) + + def _log_pack(self, rag_session_id: str, pack: ExplainPack) -> None: + prod_excerpt_count = len([excerpt for excerpt in pack.code_excerpts if not _is_test_excerpt(excerpt)]) + test_excerpt_count = len(pack.code_excerpts) - prod_excerpt_count + LOGGER.warning( + "code explain pack: rag_session_id=%s entrypoints=%s seeds=%s paths=%s excerpts=%s prod_excerpt_count=%s test_excerpt_count=%s missing=%s", + rag_session_id, + len(pack.selected_entrypoints), + len(pack.seed_symbols), + len(pack.trace_paths), + len(pack.code_excerpts), + prod_excerpt_count, + test_excerpt_count, + pack.missing, + ) + + +def _evidence_index( + entrypoints: list[LayeredRetrievalItem], + seed_symbols: list[LayeredRetrievalItem], +) -> dict[str, EvidenceItem]: + result: dict[str, EvidenceItem] = {} + for index, item in enumerate(entrypoints, start=1): + evidence_id = f"entrypoint_{index}" + result[evidence_id] = EvidenceItem( + evidence_id=evidence_id, + kind="entrypoint", + summary=item.title, + location=item.location, + supports=[str(item.metadata.get("handler_symbol_id") or "")], + ) + for index, item in enumerate(seed_symbols, start=1): + evidence_id = f"symbol_{index}" + result[evidence_id] = EvidenceItem( + evidence_id=evidence_id, + kind="symbol", + summary=item.title, + location=item.location, + supports=[str(item.metadata.get("symbol_id") or "")], + ) + return result + + +def _missing( + entrypoints: list[LayeredRetrievalItem], + seed_symbols: list[LayeredRetrievalItem], + trace_paths, + excerpts, +) -> list[str]: + missing: list[str] = [] + if not entrypoints: + missing.append("entrypoints") + if not seed_symbols: + missing.append("seed_symbols") + if not trace_paths: + missing.append("trace_paths") + if not excerpts: + missing.append("code_excerpts") + return missing + + +def _lexical_excerpts( + items: list[LayeredRetrievalItem], + *, + start_index: int = 0, + is_test_fallback: bool = False, +) -> tuple[list[CodeExcerpt], dict[str, EvidenceItem]]: + excerpts: list[CodeExcerpt] = [] + evidence_index: dict[str, EvidenceItem] = {} + for item in items: + evidence_id = f"excerpt_{start_index + len(evidence_index) + 1}" + location = item.location + evidence_index[evidence_id] = EvidenceItem( + evidence_id=evidence_id, + kind="excerpt", + summary=item.title or item.source, + location=location, + supports=[], + ) + focus = "lexical" + if _item_is_test(item): + focus = "test:lexical" + elif is_test_fallback: + focus = "lexical" + excerpts.append( + CodeExcerpt( + evidence_id=evidence_id, + symbol_id=str(item.metadata.get("symbol_id") or "") or None, + title=item.title or item.source, + path=item.source, + start_line=location.start_line if location else None, + end_line=location.end_line if location else None, + content=item.content, + focus=focus, + ) + ) + return excerpts, evidence_index + + +def _item_is_test(item: LayeredRetrievalItem) -> bool: + return bool(item.metadata.get("is_test")) or is_test_path(item.source) + + +def _is_test_excerpt(excerpt: CodeExcerpt) -> bool: + return excerpt.focus.startswith("test:") or is_test_path(excerpt.path) + + +def _path_prefixes(intent: ExplainIntent, file_candidates: list[dict]) -> list[str]: + values: list[str] = [] + for path in intent.hints.paths: + prefix = path.rsplit("/", 1)[0] if "/" in path else path + if prefix and prefix not in values: + values.append(prefix) + for item in file_candidates[:6]: + path = str(item.get("path") or "") + prefix = path.rsplit("/", 1)[0] if "/" in path else "" + if prefix and prefix not in values: + values.append(prefix) + return values + + +def _cleanup_missing(values: list[str], *, has_excerpts: bool) -> list[str]: + if not has_excerpts: + return values + return [value for value in values if value not in {"code_excerpts", "layer:C0 empty"}] + + +def _dedupe(values: list[str]) -> list[str]: + result: list[str] = [] + for value in values: + item = value.strip() + if item and item not in result: + result.append(item) + return result diff --git a/app/modules/rag/explain/source_excerpt_fetcher.py b/app/modules/rag/explain/source_excerpt_fetcher.py new file mode 100644 index 0000000..b45f6e7 --- /dev/null +++ b/app/modules/rag/explain/source_excerpt_fetcher.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from app.modules.rag.explain.excerpt_planner import ExcerptPlanner +from app.modules.rag.explain.models import CodeExcerpt, EvidenceItem, TracePath +from app.modules.rag.retrieval.test_filter import is_test_path + +if TYPE_CHECKING: + from app.modules.rag.explain.graph_repository import CodeGraphRepository + + +class SourceExcerptFetcher: + def __init__(self, graph_repository: CodeGraphRepository, planner: ExcerptPlanner | None = None) -> None: + self._graph = graph_repository + self._planner = planner or ExcerptPlanner() + + def fetch( + self, + rag_session_id: str, + trace_paths: list[TracePath], + *, + max_excerpts: int = 40, + ) -> tuple[list[CodeExcerpt], dict[str, EvidenceItem]]: + ordered_symbol_ids: list[str] = [] + for path in trace_paths: + for symbol_id in path.symbol_ids: + if symbol_id and symbol_id not in ordered_symbol_ids: + ordered_symbol_ids.append(symbol_id) + chunks = self._graph.get_chunks_by_symbol_ids(rag_session_id, ordered_symbol_ids) + excerpts: list[CodeExcerpt] = [] + evidence_index: dict[str, EvidenceItem] = {} + for chunk in chunks: + symbol_id = str(chunk.metadata.get("symbol_id") or "") + evidence_id = f"excerpt_{len(evidence_index) + 1}" + location = chunk.location + evidence_index[evidence_id] = EvidenceItem( + evidence_id=evidence_id, + kind="excerpt", + summary=chunk.title, + location=location, + supports=[symbol_id] if symbol_id else [], + ) + is_test_chunk = bool(chunk.metadata.get("is_test")) or is_test_path(location.path if location else chunk.source) + for excerpt in self._planner.plan(chunk, evidence_id=evidence_id, symbol_id=symbol_id): + if len(excerpts) >= max_excerpts: + break + if is_test_chunk and not excerpt.focus.startswith("test:"): + excerpt.focus = f"test:{excerpt.focus}" + excerpts.append(excerpt) + if len(excerpts) >= max_excerpts: + break + return excerpts, evidence_index diff --git a/app/modules/rag/explain/trace_builder.py b/app/modules/rag/explain/trace_builder.py new file mode 100644 index 0000000..791c160 --- /dev/null +++ b/app/modules/rag/explain/trace_builder.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from app.modules.rag.explain.models import LayeredRetrievalItem, TracePath + +if TYPE_CHECKING: + from app.modules.rag.explain.graph_repository import CodeGraphRepository + + +class TraceBuilder: + def __init__(self, graph_repository: CodeGraphRepository) -> None: + self._graph = graph_repository + + def build_paths( + self, + rag_session_id: str, + seed_symbols: list[LayeredRetrievalItem], + *, + max_depth: int, + max_paths: int = 3, + edge_types: list[str] | None = None, + ) -> list[TracePath]: + edges_filter = edge_types or ["calls", "imports", "inherits"] + symbol_map = self._symbol_map(seed_symbols) + paths: list[TracePath] = [] + for seed in seed_symbols: + seed_id = str(seed.metadata.get("symbol_id") or "") + if not seed_id: + continue + queue: list[tuple[list[str], float, list[str]]] = [([seed_id], 0.0, [])] + while queue and len(paths) < max_paths * 3: + current_path, score, notes = queue.pop(0) + src_symbol_id = current_path[-1] + out_edges = self._graph.get_out_edges(rag_session_id, [src_symbol_id], edges_filter, limit_per_src=4) + if not out_edges or len(current_path) >= max_depth: + paths.append(TracePath(symbol_ids=current_path, score=score, notes=notes)) + continue + for edge in out_edges: + metadata = edge.metadata + dst_symbol_id = str(metadata.get("dst_symbol_id") or "") + next_notes = list(notes) + next_score = score + self._edge_score(edge, symbol_map.get(src_symbol_id)) + if not dst_symbol_id: + dst_ref = str(metadata.get("dst_ref") or "") + package_hint = self._package_hint(symbol_map.get(src_symbol_id)) + resolved = self._graph.resolve_symbol_by_ref(rag_session_id, dst_ref, package_hint=package_hint) + if resolved is not None: + dst_symbol_id = str(resolved.metadata.get("symbol_id") or "") + symbol_map[dst_symbol_id] = resolved + next_score += 2.0 + next_notes.append(f"resolved:{dst_ref}") + if not dst_symbol_id or dst_symbol_id in current_path: + paths.append(TracePath(symbol_ids=current_path, score=next_score, notes=next_notes)) + continue + if dst_symbol_id not in symbol_map: + symbols = self._graph.get_symbols_by_ids(rag_session_id, [dst_symbol_id]) + if symbols: + symbol_map[dst_symbol_id] = symbols[0] + queue.append((current_path + [dst_symbol_id], next_score, next_notes)) + unique = self._unique_paths(paths) + unique.sort(key=lambda item: item.score, reverse=True) + return unique[:max_paths] or [TracePath(symbol_ids=[seed.metadata.get("symbol_id", "")], score=0.0) for seed in seed_symbols[:1]] + + def _edge_score(self, edge: LayeredRetrievalItem, source_symbol: LayeredRetrievalItem | None) -> float: + metadata = edge.metadata + score = 1.0 + if str(metadata.get("resolution") or "") == "resolved": + score += 2.0 + source_path = source_symbol.source if source_symbol is not None else "" + if source_path and edge.source == source_path: + score += 1.0 + if "tests/" in edge.source or "/tests/" in edge.source: + score -= 3.0 + return score + + def _package_hint(self, symbol: LayeredRetrievalItem | None) -> str | None: + if symbol is None: + return None + package = str(symbol.metadata.get("package_or_module") or "") + if not package: + return None + return ".".join(package.split(".")[:-1]) or package + + def _symbol_map(self, items: list[LayeredRetrievalItem]) -> dict[str, LayeredRetrievalItem]: + result: dict[str, LayeredRetrievalItem] = {} + for item in items: + symbol_id = str(item.metadata.get("symbol_id") or "") + if symbol_id: + result[symbol_id] = item + return result + + def _unique_paths(self, items: list[TracePath]) -> list[TracePath]: + result: list[TracePath] = [] + seen: set[tuple[str, ...]] = set() + for item in items: + key = tuple(symbol_id for symbol_id in item.symbol_ids if symbol_id) + if not key or key in seen: + continue + seen.add(key) + result.append(item) + return result diff --git a/app/modules/rag/indexing/code/code_text/__pycache__/document_builder.cpython-312.pyc b/app/modules/rag/indexing/code/code_text/__pycache__/document_builder.cpython-312.pyc index 7893715..0596ea0 100644 Binary files a/app/modules/rag/indexing/code/code_text/__pycache__/document_builder.cpython-312.pyc and b/app/modules/rag/indexing/code/code_text/__pycache__/document_builder.cpython-312.pyc differ diff --git a/app/modules/rag/indexing/code/code_text/document_builder.py b/app/modules/rag/indexing/code/code_text/document_builder.py index c42f37e..aa489e9 100644 --- a/app/modules/rag/indexing/code/code_text/document_builder.py +++ b/app/modules/rag/indexing/code/code_text/document_builder.py @@ -2,6 +2,7 @@ from __future__ import annotations from app.modules.rag.contracts import RagDocument, RagLayer, RagSource, RagSpan from app.modules.rag.indexing.code.code_text.chunker import CodeChunk +from app.modules.rag.retrieval.test_filter import is_test_path class CodeTextDocumentBuilder: @@ -17,6 +18,7 @@ class CodeTextDocumentBuilder: "chunk_index": chunk_index, "chunk_type": chunk.chunk_type, "module_or_unit": source.path.replace("/", ".").removesuffix(".py"), + "is_test": is_test_path(source.path), "artifact_type": "CODE", }, ) diff --git a/app/modules/rag/indexing/code/edges/__pycache__/document_builder.cpython-312.pyc b/app/modules/rag/indexing/code/edges/__pycache__/document_builder.cpython-312.pyc index 4e8a6b1..5829843 100644 Binary files a/app/modules/rag/indexing/code/edges/__pycache__/document_builder.cpython-312.pyc and b/app/modules/rag/indexing/code/edges/__pycache__/document_builder.cpython-312.pyc differ diff --git a/app/modules/rag/indexing/code/edges/document_builder.py b/app/modules/rag/indexing/code/edges/document_builder.py index cc6f784..8fb2b62 100644 --- a/app/modules/rag/indexing/code/edges/document_builder.py +++ b/app/modules/rag/indexing/code/edges/document_builder.py @@ -2,6 +2,7 @@ from __future__ import annotations from app.modules.rag.contracts import EvidenceLink, EvidenceType, RagDocument, RagLayer, RagSource, RagSpan from app.modules.rag.indexing.code.edges.extractor import PyEdge +from app.modules.rag.retrieval.test_filter import is_test_path class EdgeDocumentBuilder: @@ -22,6 +23,7 @@ class EdgeDocumentBuilder: "dst_symbol_id": edge.dst_symbol_id, "dst_ref": edge.dst_ref, "resolution": edge.resolution, + "is_test": is_test_path(source.path), "lang_payload": edge.metadata, "artifact_type": "CODE", }, diff --git a/app/modules/rag/indexing/code/entrypoints/__pycache__/document_builder.cpython-312.pyc b/app/modules/rag/indexing/code/entrypoints/__pycache__/document_builder.cpython-312.pyc index 4693571..29139cc 100644 Binary files a/app/modules/rag/indexing/code/entrypoints/__pycache__/document_builder.cpython-312.pyc and b/app/modules/rag/indexing/code/entrypoints/__pycache__/document_builder.cpython-312.pyc differ diff --git a/app/modules/rag/indexing/code/entrypoints/document_builder.py b/app/modules/rag/indexing/code/entrypoints/document_builder.py index 0315cfe..9a03147 100644 --- a/app/modules/rag/indexing/code/entrypoints/document_builder.py +++ b/app/modules/rag/indexing/code/entrypoints/document_builder.py @@ -2,6 +2,7 @@ from __future__ import annotations from app.modules.rag.contracts import EvidenceLink, EvidenceType, RagDocument, RagLayer, RagSource, RagSpan from app.modules.rag.indexing.code.entrypoints.registry import Entrypoint +from app.modules.rag.retrieval.test_filter import is_test_path class EntrypointDocumentBuilder: @@ -19,6 +20,7 @@ class EntrypointDocumentBuilder: "framework": entrypoint.framework, "route_or_command": entrypoint.route_or_command, "handler_symbol_id": entrypoint.handler_symbol_id, + "is_test": is_test_path(source.path), "lang_payload": entrypoint.metadata, "artifact_type": "CODE", }, diff --git a/app/modules/rag/indexing/code/symbols/__pycache__/document_builder.cpython-312.pyc b/app/modules/rag/indexing/code/symbols/__pycache__/document_builder.cpython-312.pyc index d32cf32..3acb054 100644 Binary files a/app/modules/rag/indexing/code/symbols/__pycache__/document_builder.cpython-312.pyc and b/app/modules/rag/indexing/code/symbols/__pycache__/document_builder.cpython-312.pyc differ diff --git a/app/modules/rag/indexing/code/symbols/document_builder.py b/app/modules/rag/indexing/code/symbols/document_builder.py index 22085cc..2f81b5f 100644 --- a/app/modules/rag/indexing/code/symbols/document_builder.py +++ b/app/modules/rag/indexing/code/symbols/document_builder.py @@ -2,6 +2,7 @@ from __future__ import annotations from app.modules.rag.contracts import RagDocument, RagLayer, RagSource, RagSpan from app.modules.rag.indexing.code.symbols.extractor import PySymbol +from app.modules.rag.retrieval.test_filter import is_test_path class SymbolDocumentBuilder: @@ -26,6 +27,7 @@ class SymbolDocumentBuilder: "parent_symbol_id": symbol.parent_symbol_id, "package_or_module": source.path.replace("/", ".").removesuffix(".py"), "is_entry_candidate": bool(symbol.decorators), + "is_test": is_test_path(source.path), "lang_payload": symbol.lang_payload, "artifact_type": "CODE", }, diff --git a/app/modules/rag/indexing/common/__pycache__/document_upserter.cpython-312.pyc b/app/modules/rag/indexing/common/__pycache__/document_upserter.cpython-312.pyc new file mode 100644 index 0000000..2a3131f Binary files /dev/null and b/app/modules/rag/indexing/common/__pycache__/document_upserter.cpython-312.pyc differ diff --git a/app/modules/rag/indexing/common/__pycache__/report.cpython-312.pyc b/app/modules/rag/indexing/common/__pycache__/report.cpython-312.pyc new file mode 100644 index 0000000..61db02c Binary files /dev/null and b/app/modules/rag/indexing/common/__pycache__/report.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router.md b/app/modules/rag/intent_router.md new file mode 100644 index 0000000..e4c7cac --- /dev/null +++ b/app/modules/rag/intent_router.md @@ -0,0 +1,201 @@ +# Intent Router Specification (MVP) — v1.1 +Version: 1.1 +Scope: Routing + query normalization + anchor extraction for layered RAG (CODE + DOCS) + +--- + +## 1) Цель + +Intent Router принимает: +- `user_query: string` +- `conversation_state: object` +- `repo_context: object` (язык/структура репо/доступные слои) + +И возвращает: +- `intent` +- `graph_id` +- `conversation_mode` +- `query_plan` (нормализация + якоря) +- `retrieval_spec` (запрос по слоям RAG) +- `evidence_policy` + +Router **не делает** retrieval и **не генерирует** ответ. + +--- + +## 2) MVP интенты (строго 4) + +- `CODE_QA` — объяснение/поиск по коду +- `DOCS_QA` — объяснение/поиск по документации +- `GENERATE_DOCS_FROM_CODE` — генерация документации по коду +- `PROJECT_MISC` — прочие вопросы по проекту + +--- + +## 3) Диалоговый режим (контекст темы) + +### 3.1 Политика +Router обязан сохранять intent в рамках темы. + +- Если `conversation_state.active_intent` задан +- и нет явного сигнала смены темы +- то `intent = conversation_state.active_intent` и `conversation_mode = CONTINUE` + +Смена intent допускается только если: +- есть явный сигнал смены домена/задачи, или +- новый запрос явно не соответствует текущему intent (жёсткое несоответствие) + +--- + +## 4) Обязательная нормализация запроса и извлечение якорей + +Router обязан выполнять: + +### 4.1 Query normalization +Выход должен содержать: +- `raw` — исходный запрос +- `normalized` — каноническая, детерминированная и meaning-preserving форма `raw` +- `expansions[]` — добавочные токены для retrieval/rerank +- `keyword_hints[]` — компактные ключевые токены (символы/пути/доменные термины) + +Требования: +- `raw` хранит исходную строку пользователя без изменений +- `normalized` строится **только** из `raw` и безопасных правил форматирования +- `normalized` не должен включать appended expansions, синонимы и догаданные keywords +- все enrichment должны жить только в `expansions[]`, `keyword_hints[]`, `anchors[]` + +### 4.2 RU→EN mapping (минимальный словарь) +Router обязан поддерживать RU→EN mapping терминов только как `expansions`: + +- `класс` → `class` +- `метод` → `method` +- `функция` → `function`, `def` +- `модуль` → `module` +- `пакет` → `package` +- `файл` → `file` +- `тест`, `юнит-тест` → `test`, `unit test` + +Словарь должен быть расширяемым, но эти ключи обязательны. + +### 4.3 Anchor extraction (якоря) +Router обязан извлекать **явные якоря** из user_query и conversation_state: + +Типы якорей: +- `FILE_PATH` — путь/часть пути (`src/...`, `package/module.py`, `README.md`) +- `SYMBOL` — идентификатор (CamelCase, snake_case, dotted path) +- `DOC_REF` — ссылка на doc file/section (если есть явные маркеры) +- `KEY_TERM` — важные термины, влияющие на retrieval (класс/метод/функция и т.п.) + +Каждый якорь должен возвращаться структурировано. + +--- + +## 5) Контракт выхода Router + +Top-level: + +```json +{ + "schema_version": "1.1", + "intent": "CODE_QA", + "graph_id": "CodeQAGraph", + "conversation_mode": "CONTINUE", + "query_plan": { + "raw": "", + "normalized": "", + "expansions": [], + "keyword_hints": [], + "anchors": [] + }, + "retrieval_spec": { + "domains": [], + "layer_queries": [], + "filters": {}, + "rerank_profile": "" + }, + "evidence_policy": { + "require_def": false, + "require_flow": false, + "require_spec": false, + "allow_answer_without_evidence": false + } +} + + +## 6) query_plan.anchors контракт + +{ + "type": "FILE_PATH | SYMBOL | DOC_REF | KEY_TERM", + "value": "string", + "subtype": "optional string", + "span": { "start": 0, "end": 0 }, + "confidence": 0.0 +} + +Требования: +- FILE_PATH.value хранит путь как в запросе (без попытки “исправить”) +- SYMBOL.value хранит символ как в запросе (с сохранением регистра) +- KEY_TERM используется для выставления expected evidence и выбора слоёв +- anchors может быть пустым, но router должен пытаться извлечь их всегда + + +## 7) retrieval_spec контракт (слои + фильтры) + +### 7.1 Структура + +{ + "domains": ["CODE", "DOCS"], + "layer_queries": [ + { "layer_id": "C1", "top_k": 30 }, + { "layer_id": "C3", "top_k": 15 } + ], + "filters": { + "test_policy": "EXCLUDE", + "path_scope": [], + "language": [] + }, + "rerank_profile": "code" +} + +## 7.2 Требования по intent + +- CODE_QA → domains = ["CODE"], rerank_profile="code" +- DOCS_QA → domains = ["DOCS"], rerank_profile="docs" +- GENERATE_DOCS_FROM_CODE → domains = ["CODE"], rerank_profile="generate" +- PROJECT_MISC → domains = ["CODE","DOCS"], rerank_profile="project" + +## 7.3 Требования по якорям + +- Если найден FILE_PATH → router обязан добавить filters.path_scope (минимум: этот путь/директория) +- Если найден SYMBOL → router обязан добавить SYMBOL в query_plan.keyword_hints и query_plan.expansions (при необходимости) +- Если найден KEY_TERM (например "класс") → router обязан добавить RU→EN expansions + +## 8) evidence_policy (минимальные требования) +{ + "require_def": true, + "require_flow": true, + "require_spec": false, + "allow_answer_without_evidence": false +} + +Требования: +- CODE_QA: require_def=true; require_flow=true +- DOCS_QA: require_spec=true +- GENERATE_DOCS_FROM_CODE: require_def=true +- PROJECT_MISC: allow_answer_without_evidence=true + +## 9) Минимально обязательные поля (строго) + +Router обязан всегда возвращать: +- intent +- graph_id +- conversation_mode +- query_plan.raw +- query_plan.normalized +- query_plan.expansions +- query_plan.anchors +- retrieval_spec.domains +- retrieval_spec.layer_queries +- retrieval_spec.filters.test_policy +- retrieval_spec.rerank_profile +- evidence_policy.* diff --git a/app/modules/rag/intent_router_v2/__init__.py b/app/modules/rag/intent_router_v2/__init__.py new file mode 100644 index 0000000..5933990 --- /dev/null +++ b/app/modules/rag/intent_router_v2/__init__.py @@ -0,0 +1,23 @@ +from app.modules.rag.intent_router_v2.factory import GigaChatIntentRouterFactory +from app.modules.rag.intent_router_v2.local_runner import IntentRouterScenarioRunner +from app.modules.rag.intent_router_v2.models import ( + ConversationState, + IntentDecision, + IntentRouterResult, + QueryAnchor, + QueryPlan, + RepoContext, +) +from app.modules.rag.intent_router_v2.router import IntentRouterV2 + +__all__ = [ + "ConversationState", + "GigaChatIntentRouterFactory", + "IntentDecision", + "IntentRouterResult", + "IntentRouterScenarioRunner", + "IntentRouterV2", + "QueryAnchor", + "QueryPlan", + "RepoContext", +] diff --git a/app/modules/rag/intent_router_v2/__pycache__/__init__.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..b19f91f Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/__init__.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/anchor_extractor.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/anchor_extractor.cpython-312.pyc new file mode 100644 index 0000000..cb56e07 Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/anchor_extractor.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/anchor_span_validator.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/anchor_span_validator.cpython-312.pyc new file mode 100644 index 0000000..70eee15 Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/anchor_span_validator.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/classifier.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/classifier.cpython-312.pyc new file mode 100644 index 0000000..c092c08 Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/classifier.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/conversation_anchor_builder.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/conversation_anchor_builder.cpython-312.pyc new file mode 100644 index 0000000..f4ab1a8 Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/conversation_anchor_builder.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/conversation_policy.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/conversation_policy.cpython-312.pyc new file mode 100644 index 0000000..0f966b5 Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/conversation_policy.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/evidence_policy_factory.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/evidence_policy_factory.cpython-312.pyc new file mode 100644 index 0000000..e2544ab Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/evidence_policy_factory.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/factory.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/factory.cpython-312.pyc new file mode 100644 index 0000000..23aa0d8 Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/factory.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/followup_detector.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/followup_detector.cpython-312.pyc new file mode 100644 index 0000000..e4e07a8 Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/followup_detector.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/graph_id_resolver.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/graph_id_resolver.cpython-312.pyc new file mode 100644 index 0000000..45bfbd5 Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/graph_id_resolver.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/keyword_hint_builder.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/keyword_hint_builder.cpython-312.pyc new file mode 100644 index 0000000..7de9ed8 Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/keyword_hint_builder.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/keyword_hint_sanitizer.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/keyword_hint_sanitizer.cpython-312.pyc new file mode 100644 index 0000000..89d05ea Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/keyword_hint_sanitizer.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/layer_query_builder.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/layer_query_builder.cpython-312.pyc new file mode 100644 index 0000000..b0807d6 Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/layer_query_builder.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/local_runner.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/local_runner.cpython-312.pyc new file mode 100644 index 0000000..bea058f Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/local_runner.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/logger.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/logger.cpython-312.pyc new file mode 100644 index 0000000..d40a1d5 Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/logger.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/models.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/models.cpython-312.pyc new file mode 100644 index 0000000..b2d2810 Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/models.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/negation_detector.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/negation_detector.cpython-312.pyc new file mode 100644 index 0000000..8e48d46 Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/negation_detector.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/normalization.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/normalization.cpython-312.pyc new file mode 100644 index 0000000..55bd0d4 Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/normalization.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/normalization_terms.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/normalization_terms.cpython-312.pyc new file mode 100644 index 0000000..8ca67ec Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/normalization_terms.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/protocols.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/protocols.cpython-312.pyc new file mode 100644 index 0000000..27aae31 Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/protocols.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/query_normalizer.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/query_normalizer.cpython-312.pyc new file mode 100644 index 0000000..69b024f Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/query_normalizer.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/query_plan_builder.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/query_plan_builder.cpython-312.pyc new file mode 100644 index 0000000..038341e Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/query_plan_builder.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/retrieval_filter_builder.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/retrieval_filter_builder.cpython-312.pyc new file mode 100644 index 0000000..8d931a6 Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/retrieval_filter_builder.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/retrieval_spec_factory.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/retrieval_spec_factory.cpython-312.pyc new file mode 100644 index 0000000..d1fb8b8 Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/retrieval_spec_factory.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/router.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/router.cpython-312.pyc new file mode 100644 index 0000000..d1c763f Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/router.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/sub_intent_detector.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/sub_intent_detector.cpython-312.pyc new file mode 100644 index 0000000..c1dd93d Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/sub_intent_detector.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/symbol_rules.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/symbol_rules.cpython-312.pyc new file mode 100644 index 0000000..d3f4c14 Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/symbol_rules.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/term_mapping.cpython-312.pyc b/app/modules/rag/intent_router_v2/__pycache__/term_mapping.cpython-312.pyc new file mode 100644 index 0000000..926800f Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/term_mapping.cpython-312.pyc differ diff --git a/app/modules/rag/intent_router_v2/__pycache__/test_signals.cpython-312-pytest-9.0.2.pyc b/app/modules/rag/intent_router_v2/__pycache__/test_signals.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000..a9bc3dd Binary files /dev/null and b/app/modules/rag/intent_router_v2/__pycache__/test_signals.cpython-312-pytest-9.0.2.pyc differ diff --git a/app/modules/rag/intent_router_v2/anchor_extractor.py b/app/modules/rag/intent_router_v2/anchor_extractor.py new file mode 100644 index 0000000..f01e08c --- /dev/null +++ b/app/modules/rag/intent_router_v2/anchor_extractor.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import re + +from app.modules.rag.intent_router_v2.models import AnchorSpan, QueryAnchor +from app.modules.rag.intent_router_v2.normalization_terms import KeyTermCanonicalizer +from app.modules.rag.intent_router_v2.symbol_rules import COMMON_PATH_SEGMENTS, PY_KEYWORDS +from app.modules.rag.intent_router_v2.term_mapping import RuEnTermMapper + +_FILE_PATTERN = re.compile(r"(?P\b(?:[\w.-]+/)*[\w.-]+\.(?:py|md|rst|txt|yaml|yml|json|toml|ini|cfg)\b)") +_PATH_HINT_PATTERN = re.compile(r"(?P\b(?:src|app|docs|tests)/[\w./-]*[\w-]\b)") +_SYMBOL_PATTERN = re.compile( + r"\b(?P[A-Z][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)+|[A-Z][A-Za-z0-9_]{2,}|[a-z_][A-Za-z0-9_]{2,})\b" +) +_DOC_SECTION_PATTERN = re.compile(r"(?:section|раздел)\s+[\"'`#]?(?P[A-Za-zА-Яа-я0-9_ ./:-]{2,})", re.IGNORECASE) +_WORD_RE = re.compile(r"[A-Za-zА-Яа-яЁё-]+") + + +class AnchorExtractor: + def __init__( + self, + mapper: RuEnTermMapper | None = None, + canonicalizer: KeyTermCanonicalizer | None = None, + ) -> None: + self._mapper = mapper or RuEnTermMapper() + self._canonicalizer = canonicalizer or KeyTermCanonicalizer() + + def extract(self, text: str) -> list[QueryAnchor]: + anchors = self._file_anchors(text) + anchors.extend(self._symbol_anchors(text, file_anchors=anchors)) + anchors.extend(self._doc_ref_anchors(text)) + anchors.extend(self._key_term_anchors(text)) + return self._dedupe(anchors) + + def _file_anchors(self, text: str) -> list[QueryAnchor]: + anchors = self._anchors_from_matches(_FILE_PATTERN.finditer(text), anchor_type="FILE_PATH", confidence=0.95) + anchors.extend(self._anchors_from_matches(_PATH_HINT_PATTERN.finditer(text), anchor_type="FILE_PATH", confidence=0.8)) + return anchors + + def _symbol_anchors(self, text: str, *, file_anchors: list[QueryAnchor]) -> list[QueryAnchor]: + anchors: list[QueryAnchor] = [] + path_ranges = [(anchor.span.start, anchor.span.end) for anchor in file_anchors if anchor.span is not None] + path_segments = self._path_segments(file_anchors) + for match in _SYMBOL_PATTERN.finditer(text): + value = match.group("value") + if value.endswith((".py", ".md")) or "/" in value: + continue + if self._is_inside_path(match.start("value"), match.end("value"), path_ranges): + continue + if self._is_keyword(value): + continue + if file_anchors and value.lower() in path_segments: + continue + anchors.append(self._anchor("SYMBOL", value, match.start("value"), match.end("value"), 0.88, source="user_text")) + return anchors + + def _doc_ref_anchors(self, text: str) -> list[QueryAnchor]: + anchors = self._anchors_from_matches(_DOC_SECTION_PATTERN.finditer(text), anchor_type="DOC_REF", confidence=0.75, subtype="section") + for match in _FILE_PATTERN.finditer(text): + value = match.group("value") + if not value.lower().endswith((".md", ".rst", ".txt")): + continue + anchors.append(self._anchor("DOC_REF", value, match.start("value"), match.end("value"), 0.92, subtype="file", source="user_text")) + return anchors + + def _key_term_anchors(self, text: str) -> list[QueryAnchor]: + literals = set(self._mapper.all_literal_terms()) + anchors: list[QueryAnchor] = [] + for token in _WORD_RE.finditer(text): + value = token.group(0) + normalized = value.lower() + canonical = self._canonicalizer.canonicalize(value) + if canonical is None and normalized not in literals: + continue + anchors.append( + self._anchor( + "KEY_TERM", + canonical or value, + token.start(), + token.end(), + 0.9, + source="user_text", + ) + ) + return anchors + + def _anchors_from_matches( + self, + matches, + *, + anchor_type: str, + confidence: float, + subtype: str | None = None, + ) -> list[QueryAnchor]: + return [ + self._anchor(anchor_type, match.group("value"), match.start("value"), match.end("value"), confidence, subtype=subtype) + for match in matches + ] + + def _anchor( + self, + anchor_type: str, + value: str, + start: int, + end: int, + confidence: float, + subtype: str | None = None, + source: str = "user_text", + ) -> QueryAnchor: + return QueryAnchor( + type=anchor_type, + value=value, + subtype=subtype, + source=source, + span=AnchorSpan(start=start, end=end), + confidence=confidence, + ) + + def _dedupe(self, anchors: list[QueryAnchor]) -> list[QueryAnchor]: + result: list[QueryAnchor] = [] + seen: set[tuple[str, str, str | None, str]] = set() + for anchor in anchors: + key = (anchor.type, anchor.value, anchor.subtype, anchor.source) + if key in seen: + continue + seen.add(key) + result.append(anchor) + return result + + def _is_inside_path(self, start: int, end: int, ranges: list[tuple[int, int]]) -> bool: + return any(start >= left and end <= right for left, right in ranges) + + def _is_keyword(self, token: str) -> bool: + return token.lower() in PY_KEYWORDS + + def _path_segments(self, anchors: list[QueryAnchor]) -> set[str]: + values: set[str] = set() + for anchor in anchors: + parts = re.split(r"[/.]+", anchor.value.lower()) + for part in parts: + if not part: + continue + values.add(part) + return values | COMMON_PATH_SEGMENTS diff --git a/app/modules/rag/intent_router_v2/anchor_span_validator.py b/app/modules/rag/intent_router_v2/anchor_span_validator.py new file mode 100644 index 0000000..3065972 --- /dev/null +++ b/app/modules/rag/intent_router_v2/anchor_span_validator.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from app.modules.rag.intent_router_v2.models import QueryAnchor + + +class AnchorSpanValidator: + def sanitize(self, anchors: list[QueryAnchor], raw_len: int) -> list[QueryAnchor]: + result: list[QueryAnchor] = [] + for anchor in anchors: + if anchor.source != "user_text": + result.append(anchor.model_copy(update={"span": None})) + continue + if anchor.span is None: + result.append(anchor) + continue + start = int(anchor.span.start) + end = int(anchor.span.end) + if 0 <= start < end <= raw_len: + result.append(anchor) + continue + result.append(anchor.model_copy(update={"span": None, "confidence": max(anchor.confidence * 0.5, 0.0)})) + return result diff --git a/app/modules/rag/intent_router_v2/classifier.py b/app/modules/rag/intent_router_v2/classifier.py new file mode 100644 index 0000000..0ba2c3a --- /dev/null +++ b/app/modules/rag/intent_router_v2/classifier.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import json +import re + +from app.modules.rag.intent_router_v2.models import ConversationState, IntentDecision +from app.modules.rag.intent_router_v2.protocols import TextGenerator +from app.modules.rag.intent_router_v2.test_signals import has_test_focus + +_CODE_FILE_PATH_RE = re.compile( + r"\b(?:[\w.-]+/)*[\w.-]+\.(?:py|js|jsx|ts|tsx|java|kt|go|rb|php|c|cc|cpp|h|hpp|cs|swift|rs)(?!\w)\b", + re.IGNORECASE, +) + + +class IntentClassifierV2: + _GENERATE_DOCS_MARKERS = ( + "сгенерируй документац", + "подготовь документац", + "создай документац", + "генерац", + "generate documentation", + "write documentation", + ) + _DOCS_MARKERS = ("документац", "readme", "docs/", ".md", "spec", "runbook", "markdown") + _CODE_MARKERS = ("по коду", "код", "класс", "метод", "функц", "модул", "пакет", "файл", "block", "блок", "handler", "endpoint") + + def __init__(self, llm: TextGenerator | None = None) -> None: + self._llm = llm + + def classify(self, user_query: str, conversation_state: ConversationState) -> IntentDecision: + deterministic = self._deterministic(user_query) + if deterministic: + return deterministic + llm_decision = self._classify_with_llm(user_query, conversation_state) + if llm_decision: + return llm_decision + return IntentDecision(intent="PROJECT_MISC", confidence=0.55, reason="fallback_project_misc") + + def _deterministic(self, user_query: str) -> IntentDecision | None: + text = " ".join((user_query or "").lower().split()) + if any(marker in text for marker in self._GENERATE_DOCS_MARKERS): + return IntentDecision(intent="GENERATE_DOCS_FROM_CODE", confidence=0.97, reason="deterministic_generate_docs") + if self._looks_like_docs_question(text): + return IntentDecision(intent="DOCS_QA", confidence=0.9, reason="deterministic_docs") + if self._looks_like_code_question(user_query, text): + return IntentDecision(intent="CODE_QA", confidence=0.9, reason="deterministic_code") + return None + + def _classify_with_llm(self, user_query: str, conversation_state: ConversationState) -> IntentDecision | None: + if self._llm is None: + return None + payload = json.dumps( + { + "message": user_query, + "active_intent": conversation_state.active_intent, + "last_query": conversation_state.last_query, + "allowed_intents": ["CODE_QA", "DOCS_QA", "GENERATE_DOCS_FROM_CODE", "PROJECT_MISC"], + }, + ensure_ascii=False, + ) + try: + raw = self._llm.generate("rag_intent_router_v2", payload, log_context="rag.intent_router_v2.classify").strip() + except Exception: + return None + parsed = self._parse(raw) + if parsed is None: + return None + return parsed + + def _parse(self, raw: str) -> IntentDecision | None: + candidate = self._strip_code_fence(raw) + try: + payload = json.loads(candidate) + except json.JSONDecodeError: + return None + intent = str(payload.get("intent") or "").strip().upper() + if intent not in {"CODE_QA", "DOCS_QA", "GENERATE_DOCS_FROM_CODE", "PROJECT_MISC"}: + return None + return IntentDecision( + intent=intent, + confidence=float(payload.get("confidence") or 0.7), + reason=str(payload.get("reason") or "llm").strip() or "llm", + ) + + def _strip_code_fence(self, text: str) -> str: + if not text.startswith("```"): + return text + lines = text.splitlines() + if len(lines) < 3 or lines[-1].strip() != "```": + return text + return "\n".join(lines[1:-1]).strip() + + def _looks_like_docs_question(self, text: str) -> bool: + if self._has_code_file_path(text): + return False + return any(marker in text for marker in self._DOCS_MARKERS) + + def _looks_like_code_question(self, raw_text: str, lowered: str) -> bool: + if self._has_code_file_path(raw_text): + return True + if has_test_focus(lowered): + return True + if any(marker in lowered for marker in self._DOCS_MARKERS) and not any(marker in lowered for marker in self._CODE_MARKERS): + return False + if any(marker in lowered for marker in self._CODE_MARKERS): + return True + if re.search(r"\b[A-Z][A-Za-z0-9_]{2,}(?:\.[A-Za-z_][A-Za-z0-9_]*)*\b", raw_text or ""): + return True + return bool(re.search(r"\b[a-z_][A-Za-z0-9_]{2,}\(", raw_text or "")) + + def _has_code_file_path(self, text: str) -> bool: + return bool(_CODE_FILE_PATH_RE.search(text or "")) diff --git a/app/modules/rag/intent_router_v2/conversation_anchor_builder.py b/app/modules/rag/intent_router_v2/conversation_anchor_builder.py new file mode 100644 index 0000000..2ebaa07 --- /dev/null +++ b/app/modules/rag/intent_router_v2/conversation_anchor_builder.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from app.modules.rag.intent_router_v2.followup_detector import FollowUpDetector +from app.modules.rag.intent_router_v2.models import ConversationState, QueryAnchor + + +class ConversationAnchorBuilder: + def __init__(self, followup: FollowUpDetector | None = None) -> None: + self._followup = followup or FollowUpDetector() + + def build( + self, + raw: str, + state: ConversationState, + *, + continue_mode: bool, + has_user_symbol: bool, + has_user_file_path: bool, + ) -> list[QueryAnchor]: + if not continue_mode: + return [] + anchors: list[QueryAnchor] = [] + if has_user_file_path: + return anchors + for path in self._paths_for_carryover(state.active_path_scope): + anchors.append( + QueryAnchor( + type="FILE_PATH", + value=path, + source="conversation_state", + span=None, + confidence=0.6, + ) + ) + if has_user_symbol: + return anchors + if not self._followup.is_follow_up(raw): + return anchors + symbol = state.active_symbol or (state.active_code_span_symbols[0] if state.active_code_span_symbols else None) + if symbol: + anchors.append( + QueryAnchor( + type="SYMBOL", + value=symbol, + source="conversation_state", + span=None, + confidence=0.64, + ) + ) + return anchors + + def _paths_for_carryover(self, active_path_scope: list[str]) -> list[str]: + paths = list(active_path_scope or []) + file_paths = [path for path in paths if self._looks_like_file(path)] + if file_paths: + return file_paths[:1] + return paths[:1] + + def _looks_like_file(self, value: str) -> bool: + tail = (value or "").rsplit("/", 1)[-1] + return "." in tail diff --git a/app/modules/rag/intent_router_v2/conversation_policy.py b/app/modules/rag/intent_router_v2/conversation_policy.py new file mode 100644 index 0000000..3777f03 --- /dev/null +++ b/app/modules/rag/intent_router_v2/conversation_policy.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from app.modules.rag.intent_router_v2.models import ConversationState, IntentDecision + + +class ConversationPolicy: + _SWITCH_MARKERS = ( + "теперь", + "а теперь", + "давай теперь", + "переключ", + "new task", + "switch to", + "instead", + ) + _DOCS_SIGNALS = ("документац", "readme", "docs/", ".md") + _CODE_SIGNALS = ("по коду", "класс", "метод", "файл", "блок кода", "function", "class") + + def resolve(self, decision: IntentDecision, user_query: str, conversation_state: ConversationState) -> tuple[str, str]: + active_intent = conversation_state.active_intent + if active_intent is None: + return decision.intent, "START" + if active_intent == decision.intent: + return active_intent, "CONTINUE" + if self._has_explicit_switch(user_query): + return decision.intent, "SWITCH" + if self._is_hard_mismatch(active_intent, decision.intent, user_query): + return decision.intent, "SWITCH" + return active_intent, "CONTINUE" + + def _has_explicit_switch(self, user_query: str) -> bool: + text = " ".join((user_query or "").lower().split()) + return any(marker in text for marker in self._SWITCH_MARKERS) + + def _is_hard_mismatch(self, active_intent: str, candidate_intent: str, user_query: str) -> bool: + if active_intent == candidate_intent: + return False + text = " ".join((user_query or "").lower().split()) + if candidate_intent == "GENERATE_DOCS_FROM_CODE": + return True + if candidate_intent == "DOCS_QA": + return any(signal in text for signal in self._DOCS_SIGNALS) + if candidate_intent == "CODE_QA" and active_intent == "DOCS_QA": + return any(signal in text for signal in self._CODE_SIGNALS) + return False diff --git a/app/modules/rag/intent_router_v2/evidence_policy_factory.py b/app/modules/rag/intent_router_v2/evidence_policy_factory.py new file mode 100644 index 0000000..c090bd6 --- /dev/null +++ b/app/modules/rag/intent_router_v2/evidence_policy_factory.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from app.modules.rag.intent_router_v2.models import EvidencePolicy + + +class EvidencePolicyFactory: + def build( + self, + intent: str, + *, + sub_intent: str = "EXPLAIN", + negations: list[str] | None = None, + has_user_anchor: bool = True, + ) -> EvidencePolicy: + negations_set = set(negations or []) + if intent == "CODE_QA": + if sub_intent == "OPEN_FILE": + return EvidencePolicy(require_def=False, require_flow=False, require_spec=False, allow_answer_without_evidence=False) + if sub_intent == "EXPLAIN_LOCAL": + return EvidencePolicy(require_def=True, require_flow=False, require_spec=False, allow_answer_without_evidence=False) + if "tests" in negations_set and not has_user_anchor: + return EvidencePolicy(require_def=True, require_flow=False, require_spec=False, allow_answer_without_evidence=False) + return EvidencePolicy(require_def=True, require_flow=True, require_spec=False, allow_answer_without_evidence=False) + if intent == "DOCS_QA": + return EvidencePolicy(require_def=False, require_flow=False, require_spec=True, allow_answer_without_evidence=False) + if intent == "GENERATE_DOCS_FROM_CODE": + return EvidencePolicy(require_def=True, require_flow=False, require_spec=False, allow_answer_without_evidence=False) + return EvidencePolicy(require_def=False, require_flow=False, require_spec=False, allow_answer_without_evidence=True) diff --git a/app/modules/rag/intent_router_v2/factory.py b/app/modules/rag/intent_router_v2/factory.py new file mode 100644 index 0000000..e16bc49 --- /dev/null +++ b/app/modules/rag/intent_router_v2/factory.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from app.modules.agent.llm import AgentLlmService +from app.modules.agent.prompt_loader import PromptLoader +from app.modules.rag.intent_router_v2.classifier import IntentClassifierV2 +from app.modules.rag.intent_router_v2.router import IntentRouterV2 +from app.modules.shared.env_loader import load_workspace_env +from app.modules.shared.gigachat.client import GigaChatClient +from app.modules.shared.gigachat.settings import GigaChatSettings +from app.modules.shared.gigachat.token_provider import GigaChatTokenProvider + + +class GigaChatIntentRouterFactory: + def build(self) -> IntentRouterV2: + load_workspace_env() + settings = GigaChatSettings.from_env() + token_provider = GigaChatTokenProvider(settings) + client = GigaChatClient(settings, token_provider) + prompt_loader = PromptLoader() + llm = AgentLlmService(client=client, prompts=prompt_loader) + classifier = IntentClassifierV2(llm=llm) + return IntentRouterV2(classifier=classifier) diff --git a/app/modules/rag/intent_router_v2/followup_detector.py b/app/modules/rag/intent_router_v2/followup_detector.py new file mode 100644 index 0000000..d91d662 --- /dev/null +++ b/app/modules/rag/intent_router_v2/followup_detector.py @@ -0,0 +1,22 @@ +from __future__ import annotations + + +class FollowUpDetector: + _MARKERS = ( + "что дальше", + "почему", + "зачем", + "а что", + "уточни", + "подробнее", + "как именно", + "покажи подробнее", + ) + + def is_follow_up(self, raw: str) -> bool: + text = " ".join((raw or "").lower().split()) + if not text: + return False + if len(text.split()) <= 4: + return True + return any(marker in text for marker in self._MARKERS) diff --git a/app/modules/rag/intent_router_v2/graph_id_resolver.py b/app/modules/rag/intent_router_v2/graph_id_resolver.py new file mode 100644 index 0000000..a6cbf2e --- /dev/null +++ b/app/modules/rag/intent_router_v2/graph_id_resolver.py @@ -0,0 +1,13 @@ +from __future__ import annotations + + +class GraphIdResolver: + _GRAPH_MAP = { + "CODE_QA": "CodeQAGraph", + "DOCS_QA": "DocsQAGraph", + "GENERATE_DOCS_FROM_CODE": "GenerateDocsFromCodeGraph", + "PROJECT_MISC": "ProjectMiscGraph", + } + + def resolve(self, intent: str) -> str: + return self._GRAPH_MAP[intent] diff --git a/app/modules/rag/intent_router_v2/keyword_hint_builder.py b/app/modules/rag/intent_router_v2/keyword_hint_builder.py new file mode 100644 index 0000000..ed10577 --- /dev/null +++ b/app/modules/rag/intent_router_v2/keyword_hint_builder.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import re + +from app.modules.rag.intent_router_v2.normalization import FILE_PATH_RE +from app.modules.rag.intent_router_v2.symbol_rules import COMMON_PATH_SEGMENTS, PY_KEYWORDS + +_IDENTIFIER_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]{2,}") + + +class KeywordHintBuilder: + def build(self, text: str) -> list[str]: + hints: list[str] = [] + path_segments = self._path_segments(text) + for token in _IDENTIFIER_RE.findall(text or ""): + if token.lower() in PY_KEYWORDS: + continue + if token.lower() in path_segments: + continue + if token not in hints: + hints.append(token) + for match in FILE_PATH_RE.finditer(text or ""): + candidate = match.group(0).lower() + if candidate not in hints: + hints.append(candidate) + return hints[:12] + + def _path_segments(self, text: str) -> set[str]: + values: set[str] = set(COMMON_PATH_SEGMENTS) + for match in FILE_PATH_RE.finditer(text or ""): + for part in re.split(r"[/.]+", match.group(0).lower()): + if part: + values.add(part) + return values diff --git a/app/modules/rag/intent_router_v2/keyword_hint_sanitizer.py b/app/modules/rag/intent_router_v2/keyword_hint_sanitizer.py new file mode 100644 index 0000000..7d44f26 --- /dev/null +++ b/app/modules/rag/intent_router_v2/keyword_hint_sanitizer.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from app.modules.rag.intent_router_v2.models import QueryAnchor + + +class KeywordHintSanitizer: + _GENERIC_KEY_TERMS = {"файл", "класс", "метод", "функция", "документация"} + _DOMAIN_ALLOWLIST = {"RAG", "API", "HTTP", "SQL"} + _DIR_SCOPE_MARKERS = ("в папке", "в директории", "в каталоге") + + def sanitize(self, raw: str, anchors: list[QueryAnchor], base_hints: list[str]) -> list[str]: + text = (raw or "").lower() + allow_dirs = any(marker in text for marker in self._DIR_SCOPE_MARKERS) + file_paths = [anchor.value for anchor in anchors if anchor.type == "FILE_PATH" and self._looks_like_file(anchor.value)] + known_dirs = {path.rsplit("/", 1)[0] for path in file_paths if "/" in path} + result: list[str] = [] + + for anchor in anchors: + if anchor.type == "FILE_PATH": + if self._looks_like_directory(anchor.value): + if not allow_dirs and (known_dirs or file_paths): + continue + self._append(result, anchor.value) + if anchor.type == "SYMBOL": + self._append(result, anchor.value) + + for token in base_hints: + if token in self._DOMAIN_ALLOWLIST: + self._append(result, token) + continue + lowered = token.lower() + if lowered in self._GENERIC_KEY_TERMS: + continue + if token in known_dirs and not allow_dirs: + continue + if "/" in token and "." not in token and not allow_dirs and file_paths: + continue + self._append(result, token) + return result[:8] + + def _append(self, values: list[str], candidate: str) -> None: + if candidate and candidate not in values: + values.append(candidate) + + def _looks_like_file(self, value: str) -> bool: + tail = (value or "").rsplit("/", 1)[-1] + return "." in tail + + def _looks_like_directory(self, value: str) -> bool: + return "/" in (value or "") and not self._looks_like_file(value) diff --git a/app/modules/rag/intent_router_v2/layer_query_builder.py b/app/modules/rag/intent_router_v2/layer_query_builder.py new file mode 100644 index 0000000..b1f1e77 --- /dev/null +++ b/app/modules/rag/intent_router_v2/layer_query_builder.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from app.modules.rag.intent_router_v2.models import LayerQuery, RepoContext + + +class LayerQueryBuilder: + def build(self, intent: str, repo_context: RepoContext, *, domains: list[str], layers_map: dict[str, list[tuple[str, int]]]) -> list[LayerQuery]: + available = set(repo_context.available_layers or []) + result: list[LayerQuery] = [] + for layer_id, top_k in layers_map[intent]: + if not self._layer_matches_domains(layer_id, domains): + continue + if available and layer_id not in available: + continue + result.append(LayerQuery(layer_id=layer_id, top_k=top_k)) + if result: + return result + return [ + LayerQuery(layer_id=layer_id, top_k=top_k) + for layer_id, top_k in layers_map[intent] + if self._layer_matches_domains(layer_id, domains) + ] + + def _layer_matches_domains(self, layer_id: str, domains: list[str]) -> bool: + if domains == ["CODE"]: + return layer_id.startswith("C") + if domains == ["DOCS"]: + return layer_id.startswith("D") + return layer_id.startswith("C") or layer_id.startswith("D") diff --git a/app/modules/rag/intent_router_v2/local_runner.py b/app/modules/rag/intent_router_v2/local_runner.py new file mode 100644 index 0000000..96cc484 --- /dev/null +++ b/app/modules/rag/intent_router_v2/local_runner.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import logging + +from app.modules.rag.intent_router_v2.models import ConversationState, IntentRouterResult, RepoContext +from app.modules.rag.intent_router_v2.router import IntentRouterV2 + +LOGGER = logging.getLogger(__name__) + + +class IntentRouterScenarioRunner: + def __init__(self, router: IntentRouterV2) -> None: + self._router = router + + def run(self, queries: list[str], repo_context: RepoContext | None = None) -> list[IntentRouterResult]: + state = ConversationState() + context = repo_context or RepoContext() + results: list[IntentRouterResult] = [] + for index, user_query in enumerate(queries, start=1): + LOGGER.warning("intent router local input: turn=%s user_query=%s", index, user_query) + result = self._router.route(user_query, state, context) + LOGGER.warning("intent router local output: turn=%s result=%s", index, result.model_dump_json(ensure_ascii=False)) + results.append(result) + state = state.advance(result) + return results diff --git a/app/modules/rag/intent_router_v2/logger.py b/app/modules/rag/intent_router_v2/logger.py new file mode 100644 index 0000000..2775bf8 --- /dev/null +++ b/app/modules/rag/intent_router_v2/logger.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +import logging + +from app.modules.rag.intent_router_v2.models import ConversationState, IntentRouterResult, RepoContext + +LOGGER = logging.getLogger(__name__) + + +class IntentRouterLogger: + def log_request(self, user_query: str, conversation_state: ConversationState, repo_context: RepoContext) -> None: + LOGGER.warning( + "intent router v2 request: turn=%s active_intent=%s user_query=%s languages=%s domains=%s", + conversation_state.turn_index + 1, + conversation_state.active_intent, + " ".join((user_query or "").split()), + repo_context.languages, + repo_context.available_domains, + ) + + def log_result(self, result: IntentRouterResult) -> None: + LOGGER.warning("intent router v2 result: %s", result.model_dump_json(ensure_ascii=False)) diff --git a/app/modules/rag/intent_router_v2/models.py b/app/modules/rag/intent_router_v2/models.py new file mode 100644 index 0000000..d5897d5 --- /dev/null +++ b/app/modules/rag/intent_router_v2/models.py @@ -0,0 +1,182 @@ +from __future__ import annotations + +import re +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +IntentType = Literal["CODE_QA", "DOCS_QA", "GENERATE_DOCS_FROM_CODE", "PROJECT_MISC"] +ConversationMode = Literal["START", "CONTINUE", "SWITCH"] +AnchorType = Literal["FILE_PATH", "SYMBOL", "DOC_REF", "KEY_TERM"] +AnchorSource = Literal["user_text", "conversation_state", "heuristic"] +_INLINE_CODE_RE = re.compile(r"`([^`]*)`") +_CODE_SYMBOL_RE = re.compile(r"\b([A-Za-z_][A-Za-z0-9_]{2,})\b") + + +class AnchorSpan(BaseModel): + model_config = ConfigDict(extra="forbid") + + start: int = 0 + end: int = 0 + + +class QueryAnchor(BaseModel): + model_config = ConfigDict(extra="forbid") + + type: AnchorType + value: str + source: AnchorSource = "user_text" + subtype: str | None = None + span: AnchorSpan | None = None + confidence: float = 0.0 + + @field_validator("confidence") + @classmethod + def clamp_confidence(cls, value: float) -> float: + return max(0.0, min(1.0, float(value))) + + +class QueryPlan(BaseModel): + model_config = ConfigDict(extra="forbid") + + raw: str + normalized: str + sub_intent: str = "EXPLAIN" + negations: list[str] = Field(default_factory=list) + expansions: list[str] = Field(default_factory=list) + keyword_hints: list[str] = Field(default_factory=list) + anchors: list[QueryAnchor] = Field(default_factory=list) + + +class LayerQuery(BaseModel): + model_config = ConfigDict(extra="forbid") + + layer_id: str + top_k: int + + +class CodeRetrievalFilters(BaseModel): + model_config = ConfigDict(extra="forbid") + + test_policy: str = "EXCLUDE" + path_scope: list[str] = Field(default_factory=list) + language: list[str] = Field(default_factory=list) + + +class DocsRetrievalFilters(BaseModel): + model_config = ConfigDict(extra="forbid") + + path_scope: list[str] = Field(default_factory=list) + doc_kinds: list[str] = Field(default_factory=list) + doc_language: list[str] = Field(default_factory=list) + + +class HybridRetrievalFilters(BaseModel): + model_config = ConfigDict(extra="forbid") + + test_policy: str = "EXCLUDE" + path_scope: list[str] = Field(default_factory=list) + language: list[str] = Field(default_factory=list) + doc_kinds: list[str] = Field(default_factory=list) + doc_language: list[str] = Field(default_factory=list) + + +class RetrievalSpec(BaseModel): + model_config = ConfigDict(extra="forbid") + + domains: list[str] = Field(default_factory=list) + layer_queries: list[LayerQuery] = Field(default_factory=list) + filters: CodeRetrievalFilters | DocsRetrievalFilters | HybridRetrievalFilters = Field(default_factory=CodeRetrievalFilters) + rerank_profile: str = "" + + +class EvidencePolicy(BaseModel): + model_config = ConfigDict(extra="forbid") + + require_def: bool = False + require_flow: bool = False + require_spec: bool = False + allow_answer_without_evidence: bool = False + + +class IntentRouterResult(BaseModel): + model_config = ConfigDict(extra="forbid") + + schema_version: str = "1.1" + intent: IntentType + graph_id: str + conversation_mode: ConversationMode + query_plan: QueryPlan + retrieval_spec: RetrievalSpec + evidence_policy: EvidencePolicy + + +class ConversationState(BaseModel): + model_config = ConfigDict(extra="forbid") + + active_intent: IntentType | None = None + active_domain: str | None = None + active_anchors: list[QueryAnchor] = Field(default_factory=list) + active_symbol: str | None = None + active_path_scope: list[str] = Field(default_factory=list) + active_code_span_symbols: list[str] = Field(default_factory=list) + last_query: str = "" + turn_index: int = 0 + + def advance(self, result: IntentRouterResult) -> "ConversationState": + user_anchors = [anchor for anchor in result.query_plan.anchors if anchor.source == "user_text"] + symbol_candidates = [anchor.value for anchor in user_anchors if anchor.type == "SYMBOL"] + has_user_file_anchor = any(anchor.type == "FILE_PATH" for anchor in user_anchors) + if symbol_candidates: + active_symbol = symbol_candidates[-1] + elif has_user_file_anchor: + active_symbol = None + else: + active_symbol = self.active_symbol + raw_code_symbols = _extract_code_symbols(result.query_plan.raw) + active_code_span_symbols = raw_code_symbols or list(self.active_code_span_symbols) + path_scope = list(getattr(result.retrieval_spec.filters, "path_scope", []) or []) + active_domains = list(result.retrieval_spec.domains or []) + active_domain = active_domains[0] if len(active_domains) == 1 else self.active_domain + return ConversationState( + active_intent=result.intent, + active_domain=active_domain, + active_anchors=list(user_anchors), + active_symbol=active_symbol, + active_path_scope=path_scope or list(self.active_path_scope), + active_code_span_symbols=active_code_span_symbols, + last_query=result.query_plan.raw, + turn_index=self.turn_index + 1, + ) + + +class RepoContext(BaseModel): + model_config = ConfigDict(extra="forbid") + + languages: list[str] = Field(default_factory=list) + available_domains: list[str] = Field(default_factory=lambda: ["CODE", "DOCS"]) + available_layers: list[str] = Field(default_factory=list) + + +class IntentDecision(BaseModel): + model_config = ConfigDict(extra="forbid") + + intent: IntentType + confidence: float = 0.0 + reason: str = "" + + @field_validator("confidence") + @classmethod + def clamp_confidence(cls, value: float) -> float: + return max(0.0, min(1.0, float(value))) + + +def _extract_code_symbols(raw: str) -> list[str]: + symbols: list[str] = [] + for match in _INLINE_CODE_RE.finditer(raw or ""): + snippet = match.group(1) + for token in _CODE_SYMBOL_RE.findall(snippet): + if token not in symbols: + symbols.append(token) + return symbols[:8] diff --git a/app/modules/rag/intent_router_v2/negation_detector.py b/app/modules/rag/intent_router_v2/negation_detector.py new file mode 100644 index 0000000..c3fadeb --- /dev/null +++ b/app/modules/rag/intent_router_v2/negation_detector.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +import re + +_TEST_NEG_RE = re.compile( + r"(?:не\s+про\s+тест|без\s+тест|кроме\s+тест|про\s+прод\s+код|только\s+прод|production\s+code)", + re.IGNORECASE, +) + + +class NegationDetector: + def detect(self, text: str) -> set[str]: + lowered = (text or "").lower() + negations: set[str] = set() + if _TEST_NEG_RE.search(lowered): + negations.add("tests") + return negations diff --git a/app/modules/rag/intent_router_v2/normalization.py b/app/modules/rag/intent_router_v2/normalization.py new file mode 100644 index 0000000..cbbca79 --- /dev/null +++ b/app/modules/rag/intent_router_v2/normalization.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import re + +CODE_SPAN_RE = re.compile(r"`[^`]*`") +FILE_PATH_RE = re.compile( + r"(? str: + text = raw or "" + protected = _ProtectedText() + text = self._protect(text, protected) + text = self._collapse_whitespace(text) + text = text.translate(QUOTE_TRANSLATION) + text = SPACE_BEFORE_PUNCT_RE.sub(r"\1", text) + text = SPACE_AFTER_PUNCT_RE.sub(r"\1 ", text) + text = self._collapse_whitespace(text) + return protected.restore(text) + + def _protect(self, text: str, protected: "_ProtectedText") -> str: + for pattern in (CODE_SPAN_RE, FILE_PATH_RE, DOTTED_IDENT_RE, CAMEL_RE, SNAKE_RE): + text = pattern.sub(protected.replace, text) + return text + + def _collapse_whitespace(self, text: str) -> str: + return WS_RE.sub(" ", text).strip() + + +class _ProtectedText: + def __init__(self) -> None: + self._items: dict[str, str] = {} + self._index = 0 + + def replace(self, match: re.Match[str]) -> str: + placeholder = f"@@P{self._index}@@" + self._items[placeholder] = match.group(0) + self._index += 1 + return placeholder + + def restore(self, text: str) -> str: + restored = text + for placeholder, value in self._items.items(): + restored = restored.replace(placeholder, value) + return restored diff --git a/app/modules/rag/intent_router_v2/normalization_terms.py b/app/modules/rag/intent_router_v2/normalization_terms.py new file mode 100644 index 0000000..41aa85e --- /dev/null +++ b/app/modules/rag/intent_router_v2/normalization_terms.py @@ -0,0 +1,48 @@ +from __future__ import annotations + + +class KeyTermCanonicalizer: + _ALIASES: dict[str, set[str]] = { + "файл": { + "файл", + "файла", + "файле", + "файлу", + "файлом", + "файлы", + "файлов", + "файлам", + "файлами", + }, + "класс": {"класс", "класса", "классе", "классу", "классом", "классы", "классов", "классам"}, + "функция": {"функция", "функции", "функцию", "функцией", "функциях"}, + "метод": {"метод", "метода", "методе", "методу", "методом", "методы"}, + "документация": {"документация", "документации", "документацию"}, + "тест": {"тест", "тесты", "тестов", "тестам", "тестами", "юнит-тест", "юниттест"}, + "модуль": {"модуль", "модуля"}, + "пакет": {"пакет"}, + } + + def __init__(self) -> None: + self._token_to_canonical = self._build_index() + + def canonicalize(self, token: str) -> str | None: + return self._token_to_canonical.get((token or "").lower()) + + def aliases(self) -> set[str]: + values: set[str] = set() + for forms in self._ALIASES.values(): + values.update(forms) + return values + + def is_test_term(self, token: str) -> bool: + canonical = self.canonicalize(token) + return canonical == "тест" + + def _build_index(self) -> dict[str, str]: + index: dict[str, str] = {} + for canonical, forms in self._ALIASES.items(): + index[canonical] = canonical + for form in forms: + index[form] = canonical + return index diff --git a/app/modules/rag/intent_router_v2/protocols.py b/app/modules/rag/intent_router_v2/protocols.py new file mode 100644 index 0000000..4b88f20 --- /dev/null +++ b/app/modules/rag/intent_router_v2/protocols.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +from typing import Protocol + + +class TextGenerator(Protocol): + def generate(self, prompt_name: str, user_input: str, *, log_context: str | None = None) -> str: ... diff --git a/app/modules/rag/intent_router_v2/query_normalizer.py b/app/modules/rag/intent_router_v2/query_normalizer.py new file mode 100644 index 0000000..7b64914 --- /dev/null +++ b/app/modules/rag/intent_router_v2/query_normalizer.py @@ -0,0 +1,3 @@ +from app.modules.rag.intent_router_v2.normalization import QueryNormalizer + +__all__ = ["QueryNormalizer"] diff --git a/app/modules/rag/intent_router_v2/query_plan_builder.py b/app/modules/rag/intent_router_v2/query_plan_builder.py new file mode 100644 index 0000000..47f30fc --- /dev/null +++ b/app/modules/rag/intent_router_v2/query_plan_builder.py @@ -0,0 +1,223 @@ +from __future__ import annotations + +from app.modules.rag.intent_router_v2.anchor_extractor import AnchorExtractor +from app.modules.rag.intent_router_v2.anchor_span_validator import AnchorSpanValidator +from app.modules.rag.intent_router_v2.conversation_anchor_builder import ConversationAnchorBuilder +from app.modules.rag.intent_router_v2.keyword_hint_builder import KeywordHintBuilder +from app.modules.rag.intent_router_v2.keyword_hint_sanitizer import KeywordHintSanitizer +from app.modules.rag.intent_router_v2.models import ConversationState, QueryAnchor, QueryPlan +from app.modules.rag.intent_router_v2.negation_detector import NegationDetector +from app.modules.rag.intent_router_v2.normalization import QueryNormalizer +from app.modules.rag.intent_router_v2.sub_intent_detector import SubIntentDetector +from app.modules.rag.intent_router_v2.test_signals import has_test_focus, is_negative_test_request, is_test_related_token +from app.modules.rag.intent_router_v2.term_mapping import RuEnTermMapper + + +class QueryPlanBuilder: + _WHY_MARKERS = ("почему", "зачем", "откуда", "из-за чего") + _NEXT_STEP_MARKERS = ("что дальше", "дальше что", "и что теперь", "продолжай") + def __init__( + self, + normalizer: QueryNormalizer | None = None, + extractor: AnchorExtractor | None = None, + mapper: RuEnTermMapper | None = None, + keyword_hints: KeywordHintBuilder | None = None, + keyword_hint_sanitizer: KeywordHintSanitizer | None = None, + carryover: ConversationAnchorBuilder | None = None, + span_validator: AnchorSpanValidator | None = None, + sub_intent_detector: SubIntentDetector | None = None, + negation_detector: NegationDetector | None = None, + ) -> None: + self._normalizer = normalizer or QueryNormalizer() + self._extractor = extractor or AnchorExtractor() + self._mapper = mapper or RuEnTermMapper() + self._keyword_hints_builder = keyword_hints or KeywordHintBuilder() + self._keyword_hint_sanitizer = keyword_hint_sanitizer or KeywordHintSanitizer() + self._carryover = carryover or ConversationAnchorBuilder() + self._span_validator = span_validator or AnchorSpanValidator() + self._sub_intent_detector = sub_intent_detector or SubIntentDetector() + self._negation_detector = negation_detector or NegationDetector() + + def build( + self, + user_query: str, + conversation_state: ConversationState, + continue_mode: bool, + *, + conversation_mode: str = "START", + intent: str = "PROJECT_MISC", + ) -> QueryPlan: + raw = user_query or "" + normalized = self._normalizer.normalize(raw) + if not normalized and raw.strip(): + normalized = raw + negations = self._negation_detector.detect(normalized) + user_anchors = self._span_validator.sanitize(self._extractor.extract(raw), len(raw)) + has_file_path = any(anchor.type == "FILE_PATH" and anchor.source == "user_text" for anchor in user_anchors) + sub_intent = self._sub_intent_detector.detect(raw, has_file_path=has_file_path, negations=negations) + merged_anchors = self._merge_anchors( + raw, + user_anchors, + conversation_state, + continue_mode, + conversation_mode=conversation_mode, + intent=intent, + ) + skip_tests = "tests" in negations or is_negative_test_request(raw) + cleaned_anchors = self._remove_negated_test_terms(skip_tests, merged_anchors) + sub_intent = self._resolve_sub_intent(sub_intent, raw, cleaned_anchors, intent=intent, negations=negations) + if intent == "DOCS_QA": + sub_intent = "EXPLAIN" + expansions = self._expansions(normalized, cleaned_anchors, skip_tests=skip_tests) + keyword_hints = self._keyword_hints( + raw, + normalized, + cleaned_anchors, + skip_tests=skip_tests, + intent=intent, + state=conversation_state, + ) + return QueryPlan( + raw=raw, + normalized=normalized, + sub_intent=sub_intent, + negations=sorted(negations), + expansions=expansions, + keyword_hints=keyword_hints, + anchors=cleaned_anchors, + ) + + def _merge_anchors( + self, + raw: str, + anchors: list[QueryAnchor], + state: ConversationState, + continue_mode: bool, + *, + conversation_mode: str, + intent: str, + ) -> list[QueryAnchor]: + has_user_symbol = any(anchor.type == "SYMBOL" and anchor.source == "user_text" for anchor in anchors) + has_user_file = any(anchor.type == "FILE_PATH" and anchor.source == "user_text" for anchor in anchors) + inherited = self._carryover.build( + raw, + state, + continue_mode=continue_mode, + has_user_symbol=has_user_symbol, + has_user_file_path=has_user_file, + ) + if ( + conversation_mode == "SWITCH" + and intent == "DOCS_QA" + and not has_user_file + and not has_user_symbol + and state.active_symbol + ): + inherited.append( + QueryAnchor( + type="SYMBOL", + value=state.active_symbol, + source="conversation_state", + span=None, + confidence=0.62, + ) + ) + return self._dedupe(anchors + inherited) + + def _expansions(self, normalized: str, anchors: list[QueryAnchor], *, skip_tests: bool) -> list[str]: + values = self._mapper.expand(normalized) + has_symbol = any(anchor.type == "SYMBOL" for anchor in anchors) + if has_symbol: + values = [value for value in values if value.lower() not in {"def", "class"}] + if not skip_tests and has_test_focus(normalized): + for candidate in ("test", "unit test"): + if candidate not in values: + values.append(candidate) + for anchor in anchors: + if anchor.type == "SYMBOL" and anchor.value not in values: + values.append(anchor.value) + if skip_tests: + values = [value for value in values if not is_test_related_token(value)] + return values[:16] + + def _keyword_hints( + self, + raw: str, + normalized: str, + anchors: list[QueryAnchor], + *, + skip_tests: bool, + intent: str, + state: ConversationState, + ) -> list[str]: + values = self._keyword_hints_builder.build(normalized) + for anchor in anchors: + if anchor.type not in {"FILE_PATH", "SYMBOL"}: + continue + candidate = anchor.value + if candidate not in values: + values.append(candidate) + if skip_tests: + values = [value for value in values if not is_test_related_token(value)] + sanitized = self._keyword_hint_sanitizer.sanitize(raw, anchors, values) + if intent == "DOCS_QA" and not sanitized: + fallback = list(dict.fromkeys([*self._expansions(normalized, anchors, skip_tests=skip_tests)])) + sanitized = fallback[:3] + if state.active_symbol and state.active_symbol not in sanitized: + sanitized.append(state.active_symbol) + sanitized = sanitized[:5] + return sanitized + + def _remove_negated_test_terms(self, skip_tests: bool, anchors: list[QueryAnchor]) -> list[QueryAnchor]: + if not skip_tests: + return anchors + result: list[QueryAnchor] = [] + for anchor in anchors: + if anchor.type not in {"KEY_TERM", "SYMBOL"}: + result.append(anchor) + continue + if is_test_related_token(anchor.value): + continue + result.append(anchor) + return result + + def _dedupe(self, anchors: list[QueryAnchor]) -> list[QueryAnchor]: + result: list[QueryAnchor] = [] + seen: set[tuple[str, str, str | None, str]] = set() + for anchor in anchors: + key = (anchor.type, anchor.value, anchor.subtype, anchor.source) + if key in seen: + continue + seen.add(key) + result.append(anchor) + return result + + def _resolve_sub_intent( + self, + candidate: str, + raw: str, + anchors: list[QueryAnchor], + *, + intent: str, + negations: set[str], + ) -> str: + if candidate != "EXPLAIN": + return candidate + if intent != "CODE_QA": + return candidate + text = " ".join((raw or "").lower().split()) + has_symbol = any(anchor.type == "SYMBOL" and anchor.confidence >= 0.6 for anchor in anchors) + has_file = any(anchor.type == "FILE_PATH" and self._looks_like_file(anchor.value) and anchor.confidence >= 0.6 for anchor in anchors) + has_user_anchor = any(anchor.source == "user_text" for anchor in anchors) + is_why = any(marker in text for marker in self._WHY_MARKERS) + is_next_steps = any(marker in text for marker in self._NEXT_STEP_MARKERS) + is_short_generic = len(text.split()) <= 4 and text.endswith("?") + if (is_why and has_file and has_symbol) or ((is_next_steps or is_short_generic) and has_file): + return "EXPLAIN_LOCAL" + if "tests" in negations and not has_user_anchor and (has_file or has_symbol): + return "EXPLAIN_LOCAL" + return candidate + + def _looks_like_file(self, value: str) -> bool: + tail = (value or "").rsplit("/", 1)[-1] + return "." in tail diff --git a/app/modules/rag/intent_router_v2/retrieval_filter_builder.py b/app/modules/rag/intent_router_v2/retrieval_filter_builder.py new file mode 100644 index 0000000..ed86b9c --- /dev/null +++ b/app/modules/rag/intent_router_v2/retrieval_filter_builder.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +from app.modules.rag.intent_router_v2.models import ( + CodeRetrievalFilters, + ConversationState, + DocsRetrievalFilters, + HybridRetrievalFilters, + QueryAnchor, + RepoContext, +) +from app.modules.rag.intent_router_v2.test_signals import has_test_focus, is_negative_test_request, is_test_related_token + + +class RetrievalFilterBuilder: + def build( + self, + domains: list[str], + anchors: list[QueryAnchor], + repo_context: RepoContext, + *, + raw_query: str, + conversation_state: ConversationState | None, + conversation_mode: str, + sub_intent: str = "EXPLAIN", + ) -> CodeRetrievalFilters | DocsRetrievalFilters | HybridRetrievalFilters: + path_scope = self._path_scope( + anchors, + conversation_state=conversation_state, + conversation_mode=conversation_mode, + raw_query=raw_query, + sub_intent=sub_intent, + ) + if domains == ["DOCS"]: + return DocsRetrievalFilters( + path_scope=path_scope, + doc_kinds=self._doc_kinds(anchors, raw_query), + doc_language=[], + ) + if domains == ["CODE"]: + return CodeRetrievalFilters( + test_policy=self._test_policy(raw_query, anchors), + path_scope=path_scope, + language=list(repo_context.languages), + ) + return HybridRetrievalFilters( + test_policy=self._test_policy(raw_query, anchors), + path_scope=path_scope, + language=list(repo_context.languages), + doc_kinds=self._doc_kinds(anchors, raw_query), + doc_language=[], + ) + + def _test_policy(self, raw_query: str, anchors: list[QueryAnchor]) -> str: + if is_negative_test_request(raw_query): + return "EXCLUDE" + if has_test_focus(raw_query): + return "INCLUDE" + has_test_keyterm = any(anchor.type == "KEY_TERM" and is_test_related_token(anchor.value) for anchor in anchors) + return "INCLUDE" if has_test_keyterm else "EXCLUDE" + + def _path_scope( + self, + anchors: list[QueryAnchor], + *, + conversation_state: ConversationState | None, + conversation_mode: str, + raw_query: str, + sub_intent: str, + ) -> list[str]: + values: list[str] = [] + has_user_file_anchor = False + file_values: list[str] = [] + for anchor in anchors: + if anchor.type != "FILE_PATH": + continue + if anchor.source == "user_text": + has_user_file_anchor = True + if anchor.value not in values: + values.append(anchor.value) + if self._looks_like_file_path(anchor.value) and anchor.value not in file_values: + file_values.append(anchor.value) + parent = anchor.value.rsplit("/", 1)[0] if "/" in anchor.value and self._looks_like_file_path(anchor.value) else "" + if parent and parent not in values: + values.append(parent) + if sub_intent in {"OPEN_FILE", "EXPLAIN_LOCAL"} and file_values and not self._is_explicit_directory_scope(raw_query): + return file_values[:6] + if has_user_file_anchor or conversation_mode != "CONTINUE": + return values[:6] + if values: + return values[:6] + inherited = list((conversation_state.active_path_scope if conversation_state else []) or []) + return inherited[:6] + + def _doc_kinds(self, anchors: list[QueryAnchor], raw_query: str) -> list[str]: + text = (raw_query or "").lower() + kinds: list[str] = [] + has_readme = "readme" in text or any( + anchor.type in {"DOC_REF", "FILE_PATH"} and anchor.value.lower().endswith("readme.md") + for anchor in anchors + ) + if has_readme: + kinds.append("README") + return kinds + + def _looks_like_file_path(self, value: str) -> bool: + filename = value.rsplit("/", 1)[-1] + return "." in filename + + def _is_explicit_directory_scope(self, raw_query: str) -> bool: + text = (raw_query or "").lower() + return any(marker in text for marker in ("в папке", "в директории", "в каталоге")) diff --git a/app/modules/rag/intent_router_v2/retrieval_spec_factory.py b/app/modules/rag/intent_router_v2/retrieval_spec_factory.py new file mode 100644 index 0000000..69ab20d --- /dev/null +++ b/app/modules/rag/intent_router_v2/retrieval_spec_factory.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +from app.modules.rag.contracts.enums import RagLayer +from app.modules.rag.intent_router_v2.layer_query_builder import LayerQueryBuilder +from app.modules.rag.intent_router_v2.models import ConversationState, QueryAnchor, RepoContext, RetrievalSpec +from app.modules.rag.intent_router_v2.retrieval_filter_builder import RetrievalFilterBuilder + + +class RetrievalSpecFactory: + _LAYERS = { + "CODE_QA": [ + (RagLayer.CODE_ENTRYPOINTS, 6), + (RagLayer.CODE_SYMBOL_CATALOG, 8), + (RagLayer.CODE_DEPENDENCY_GRAPH, 6), + (RagLayer.CODE_SOURCE_CHUNKS, 8), + ], + "DOCS_QA": [ + (RagLayer.DOCS_MODULE_CATALOG, 5), + (RagLayer.DOCS_FACT_INDEX, 8), + (RagLayer.DOCS_SECTION_INDEX, 8), + (RagLayer.DOCS_POLICY_INDEX, 4), + ], + "GENERATE_DOCS_FROM_CODE": [ + (RagLayer.CODE_SYMBOL_CATALOG, 12), + (RagLayer.CODE_DEPENDENCY_GRAPH, 8), + (RagLayer.CODE_SOURCE_CHUNKS, 12), + (RagLayer.CODE_ENTRYPOINTS, 6), + ], + "PROJECT_MISC": [ + (RagLayer.DOCS_MODULE_CATALOG, 4), + (RagLayer.DOCS_SECTION_INDEX, 6), + (RagLayer.CODE_SYMBOL_CATALOG, 4), + (RagLayer.CODE_SOURCE_CHUNKS, 4), + ], + } + _DOMAINS = { + "CODE_QA": ["CODE"], + "DOCS_QA": ["DOCS"], + "GENERATE_DOCS_FROM_CODE": ["CODE"], + "PROJECT_MISC": ["CODE", "DOCS"], + } + _RERANK = { + "CODE_QA": "code", + "DOCS_QA": "docs", + "GENERATE_DOCS_FROM_CODE": "generate", + "PROJECT_MISC": "project", + } + _OPEN_FILE_LAYERS = [ + (RagLayer.CODE_SOURCE_CHUNKS, 12), + ] + _OPEN_FILE_WITH_SYMBOL_LAYERS = [ + (RagLayer.CODE_SOURCE_CHUNKS, 12), + (RagLayer.CODE_SYMBOL_CATALOG, 6), + ] + _EXPLAIN_LOCAL_LAYERS = [ + (RagLayer.CODE_SOURCE_CHUNKS, 12), + (RagLayer.CODE_SYMBOL_CATALOG, 8), + (RagLayer.CODE_DEPENDENCY_GRAPH, 4), + ] + + def __init__( + self, + layer_builder: LayerQueryBuilder | None = None, + filter_builder: RetrievalFilterBuilder | None = None, + ) -> None: + self._layer_builder = layer_builder or LayerQueryBuilder() + self._filter_builder = filter_builder or RetrievalFilterBuilder() + + def build( + self, + intent: str, + anchors: list[QueryAnchor], + repo_context: RepoContext, + *, + raw_query: str = "", + conversation_state: ConversationState | None = None, + conversation_mode: str = "START", + sub_intent: str = "EXPLAIN", + ) -> RetrievalSpec: + domains = self._domains(intent, repo_context) + layers_map = self._with_sub_intent_layers(intent, sub_intent, anchors) + layer_queries = self._layer_builder.build(intent, repo_context, domains=domains, layers_map=layers_map) + filters = self._filter_builder.build( + domains, + anchors, + repo_context, + raw_query=raw_query, + conversation_state=conversation_state, + conversation_mode=conversation_mode, + sub_intent=sub_intent, + ) + return RetrievalSpec( + domains=domains, + layer_queries=layer_queries, + filters=filters, + rerank_profile=self._RERANK[intent], + ) + + def _domains(self, intent: str, repo_context: RepoContext) -> list[str]: + available = set(repo_context.available_domains or ["CODE", "DOCS"]) + result = [domain for domain in self._DOMAINS[intent] if domain in available] + return result or list(self._DOMAINS[intent]) + + def _with_sub_intent_layers( + self, + intent: str, + sub_intent: str, + anchors: list[QueryAnchor], + ) -> dict[str, list[tuple[str, int]]]: + if intent != "CODE_QA": + return self._LAYERS + layers_map = dict(self._LAYERS) + if sub_intent == "OPEN_FILE": + has_symbol = any(anchor.type == "SYMBOL" and anchor.source == "user_text" for anchor in anchors) + layers_map["CODE_QA"] = list(self._OPEN_FILE_WITH_SYMBOL_LAYERS if has_symbol else self._OPEN_FILE_LAYERS) + elif sub_intent == "EXPLAIN_LOCAL": + layers_map["CODE_QA"] = list(self._EXPLAIN_LOCAL_LAYERS) + return layers_map diff --git a/app/modules/rag/intent_router_v2/router.py b/app/modules/rag/intent_router_v2/router.py new file mode 100644 index 0000000..85797d5 --- /dev/null +++ b/app/modules/rag/intent_router_v2/router.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from app.modules.rag.intent_router_v2.classifier import IntentClassifierV2 +from app.modules.rag.intent_router_v2.conversation_policy import ConversationPolicy +from app.modules.rag.intent_router_v2.evidence_policy_factory import EvidencePolicyFactory +from app.modules.rag.intent_router_v2.graph_id_resolver import GraphIdResolver +from app.modules.rag.intent_router_v2.logger import IntentRouterLogger +from app.modules.rag.intent_router_v2.models import ConversationState, IntentRouterResult, RepoContext +from app.modules.rag.intent_router_v2.query_plan_builder import QueryPlanBuilder +from app.modules.rag.intent_router_v2.retrieval_spec_factory import RetrievalSpecFactory + + +class IntentRouterV2: + def __init__( + self, + classifier: IntentClassifierV2 | None = None, + conversation_policy: ConversationPolicy | None = None, + query_plan_builder: QueryPlanBuilder | None = None, + retrieval_factory: RetrievalSpecFactory | None = None, + evidence_factory: EvidencePolicyFactory | None = None, + graph_resolver: GraphIdResolver | None = None, + logger: IntentRouterLogger | None = None, + ) -> None: + self._classifier = classifier or IntentClassifierV2() + self._conversation_policy = conversation_policy or ConversationPolicy() + self._query_plan_builder = query_plan_builder or QueryPlanBuilder() + self._retrieval_factory = retrieval_factory or RetrievalSpecFactory() + self._evidence_factory = evidence_factory or EvidencePolicyFactory() + self._graph_resolver = graph_resolver or GraphIdResolver() + self._logger = logger or IntentRouterLogger() + + def route( + self, + user_query: str, + conversation_state: ConversationState | None = None, + repo_context: RepoContext | None = None, + ) -> IntentRouterResult: + state = conversation_state or ConversationState() + context = repo_context or RepoContext() + self._logger.log_request(user_query, state, context) + decision = self._classifier.classify(user_query, state) + intent, conversation_mode = self._conversation_policy.resolve(decision, user_query, state) + query_plan = self._query_plan_builder.build( + user_query, + state, + continue_mode=conversation_mode == "CONTINUE", + conversation_mode=conversation_mode, + intent=intent, + ) + result = IntentRouterResult( + intent=intent, + graph_id=self._graph_resolver.resolve(intent), + conversation_mode=conversation_mode, + query_plan=query_plan, + retrieval_spec=self._retrieval_factory.build( + intent, + query_plan.anchors, + context, + raw_query=query_plan.raw, + conversation_state=state, + conversation_mode=conversation_mode, + sub_intent=query_plan.sub_intent, + ), + evidence_policy=self._evidence_factory.build( + intent, + sub_intent=query_plan.sub_intent, + negations=query_plan.negations, + has_user_anchor=any(anchor.source == "user_text" for anchor in query_plan.anchors), + ), + ) + self._logger.log_result(result) + return result diff --git a/app/modules/rag/intent_router_v2/sub_intent_detector.py b/app/modules/rag/intent_router_v2/sub_intent_detector.py new file mode 100644 index 0000000..c9ff705 --- /dev/null +++ b/app/modules/rag/intent_router_v2/sub_intent_detector.py @@ -0,0 +1,23 @@ +from __future__ import annotations + + +class SubIntentDetector: + _OPEN_VERBS = ("открой", "посмотри", "проверь", "уточни") + _EXPLAIN_MARKERS = ("объясни", "как работает", "почему", "что делает", "зачем", "логика", "флоу", "flow") + _TEST_MARKERS = ("тест", "pytest", "unit test", "юнит") + + def detect(self, raw: str, *, has_file_path: bool, negations: set[str]) -> str: + text = " ".join((raw or "").lower().split()) + if not text: + return "EXPLAIN" + if has_file_path and self._has_open_verb(text) and not self._has_explain_markers(text): + return "OPEN_FILE" + if "tests" not in negations and any(marker in text for marker in self._TEST_MARKERS): + return "FIND_TESTS" + return "EXPLAIN" + + def _has_open_verb(self, text: str) -> bool: + return any(text.startswith(verb) or f" {verb} " in f" {text} " for verb in self._OPEN_VERBS) + + def _has_explain_markers(self, text: str) -> bool: + return any(marker in text for marker in self._EXPLAIN_MARKERS) diff --git a/app/modules/rag/intent_router_v2/symbol_rules.py b/app/modules/rag/intent_router_v2/symbol_rules.py new file mode 100644 index 0000000..e754a49 --- /dev/null +++ b/app/modules/rag/intent_router_v2/symbol_rules.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +PY_KEYWORDS: set[str] = { + "and", + "as", + "assert", + "async", + "await", + "break", + "class", + "continue", + "def", + "del", + "elif", + "else", + "except", + "false", + "finally", + "for", + "from", + "global", + "if", + "import", + "in", + "is", + "lambda", + "none", + "nonlocal", + "not", + "or", + "pass", + "raise", + "return", + "true", + "try", + "while", + "with", + "yield", +} + +COMMON_PATH_SEGMENTS: set[str] = { + "app", + "src", + "docs", + "tests", + "module", + "modules", + "core", + "pkg", + "lib", +} diff --git a/app/modules/rag/intent_router_v2/term_mapping.py b/app/modules/rag/intent_router_v2/term_mapping.py new file mode 100644 index 0000000..62aaab7 --- /dev/null +++ b/app/modules/rag/intent_router_v2/term_mapping.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import re + +from app.modules.rag.intent_router_v2.normalization_terms import KeyTermCanonicalizer + +_WORD_RE = re.compile(r"[A-Za-zА-Яа-яЁё-]+") + + +class RuEnTermMapper: + _CANONICAL_MAP = { + "класс": ["class"], + "метод": ["method"], + "функция": ["function", "def"], + "модуль": ["module"], + "пакет": ["package"], + "файл": ["file"], + "тест": ["test", "unit test"], + "документация": ["documentation", "docs"], + "readme": ["readme"], + } + _ENGLISH_SOURCES = { + "class": ["class"], + "method": ["method"], + "function": ["function", "def"], + "module": ["module"], + "package": ["package"], + "file": ["file"], + "test": ["test", "unit test"], + "tests": ["test", "unit test"], + "documentation": ["documentation", "docs"], + "docs": ["documentation", "docs"], + "readme": ["readme"], + "def": ["def"], + } + + def __init__(self, canonicalizer: KeyTermCanonicalizer | None = None) -> None: + self._canonicalizer = canonicalizer or KeyTermCanonicalizer() + + def expand(self, text: str) -> list[str]: + expansions: list[str] = [] + lowered = (text or "").lower() + for token in _WORD_RE.findall(lowered): + canonical = self._canonicalizer.canonicalize(token) or token + self._extend(expansions, self._CANONICAL_MAP.get(canonical, [])) + self._extend(expansions, self._ENGLISH_SOURCES.get(token, [])) + if "unit test" in lowered or "unit tests" in lowered: + self._extend(expansions, self._ENGLISH_SOURCES["test"]) + return expansions + + def key_terms(self) -> tuple[str, ...]: + return tuple(self._CANONICAL_MAP.keys()) + + def all_literal_terms(self) -> tuple[str, ...]: + values = set(self._canonicalizer.aliases()) + values.update(self._CANONICAL_MAP.keys()) + values.update(self._ENGLISH_SOURCES.keys()) + for targets in self._CANONICAL_MAP.values(): + values.update(target.lower() for target in targets) + for targets in self._ENGLISH_SOURCES.values(): + values.update(target.lower() for target in targets) + return tuple(sorted(values)) + + def _extend(self, result: list[str], values: list[str]) -> None: + for value in values: + if value not in result: + result.append(value) diff --git a/app/modules/rag/intent_router_v2/test_signals.py b/app/modules/rag/intent_router_v2/test_signals.py new file mode 100644 index 0000000..60c4a52 --- /dev/null +++ b/app/modules/rag/intent_router_v2/test_signals.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import re + +_NEGATIVE_TEST_RE = re.compile(r"\b(?:не|без|кроме)\b[^.?!]{0,28}\bтест", re.IGNORECASE) +_NEGATIVE_TEST_MARKERS = ("не про тест", "без тест", "кроме тест", "про прод код", "только прод", "production code") +_POSITIVE_TEST_MARKERS = ( + "тест", + "tests", + "pytest", + "unit test", + "unit tests", + "тестиру", +) +_TEST_TERMS = {"тест", "тесты", "test", "tests", "pytest", "unit", "unit test", "юнит-тест", "юниттест"} + + +def is_negative_test_request(text: str) -> bool: + lowered = (text or "").lower() + if _NEGATIVE_TEST_RE.search(lowered): + return True + return any(marker in lowered for marker in _NEGATIVE_TEST_MARKERS) + + +def has_test_focus(text: str) -> bool: + lowered = (text or "").lower() + if is_negative_test_request(lowered): + return False + return any(marker in lowered for marker in _POSITIVE_TEST_MARKERS) + + +def is_test_related_token(value: str) -> bool: + lowered = (value or "").lower().strip() + if not lowered: + return False + if lowered in _TEST_TERMS: + return True + if lowered.startswith("test"): + return True + return lowered.startswith("тест") diff --git a/app/modules/rag/persistence/__pycache__/cache_repository.cpython-312.pyc b/app/modules/rag/persistence/__pycache__/cache_repository.cpython-312.pyc new file mode 100644 index 0000000..35a64c2 Binary files /dev/null and b/app/modules/rag/persistence/__pycache__/cache_repository.cpython-312.pyc differ diff --git a/app/modules/rag/persistence/__pycache__/document_repository.cpython-312.pyc b/app/modules/rag/persistence/__pycache__/document_repository.cpython-312.pyc new file mode 100644 index 0000000..a7ca93b Binary files /dev/null and b/app/modules/rag/persistence/__pycache__/document_repository.cpython-312.pyc differ diff --git a/app/modules/rag/persistence/__pycache__/job_repository.cpython-312.pyc b/app/modules/rag/persistence/__pycache__/job_repository.cpython-312.pyc new file mode 100644 index 0000000..9e1193c Binary files /dev/null and b/app/modules/rag/persistence/__pycache__/job_repository.cpython-312.pyc differ diff --git a/app/modules/rag/persistence/__pycache__/query_repository.cpython-312.pyc b/app/modules/rag/persistence/__pycache__/query_repository.cpython-312.pyc index 8320bca..c590c35 100644 Binary files a/app/modules/rag/persistence/__pycache__/query_repository.cpython-312.pyc and b/app/modules/rag/persistence/__pycache__/query_repository.cpython-312.pyc differ diff --git a/app/modules/rag/persistence/__pycache__/repository.cpython-312.pyc b/app/modules/rag/persistence/__pycache__/repository.cpython-312.pyc new file mode 100644 index 0000000..26cf73c Binary files /dev/null and b/app/modules/rag/persistence/__pycache__/repository.cpython-312.pyc differ diff --git a/app/modules/rag/persistence/__pycache__/retrieval_statement_builder.cpython-312.pyc b/app/modules/rag/persistence/__pycache__/retrieval_statement_builder.cpython-312.pyc new file mode 100644 index 0000000..a7969b2 Binary files /dev/null and b/app/modules/rag/persistence/__pycache__/retrieval_statement_builder.cpython-312.pyc differ diff --git a/app/modules/rag/persistence/__pycache__/schema_repository.cpython-312.pyc b/app/modules/rag/persistence/__pycache__/schema_repository.cpython-312.pyc new file mode 100644 index 0000000..0bef89d Binary files /dev/null and b/app/modules/rag/persistence/__pycache__/schema_repository.cpython-312.pyc differ diff --git a/app/modules/rag/persistence/__pycache__/session_repository.cpython-312.pyc b/app/modules/rag/persistence/__pycache__/session_repository.cpython-312.pyc new file mode 100644 index 0000000..342f560 Binary files /dev/null and b/app/modules/rag/persistence/__pycache__/session_repository.cpython-312.pyc differ diff --git a/app/modules/rag/persistence/document_repository.py b/app/modules/rag/persistence/document_repository.py index b665094..9cd5631 100644 --- a/app/modules/rag/persistence/document_repository.py +++ b/app/modules/rag/persistence/document_repository.py @@ -46,7 +46,6 @@ class RagDocumentRepository: for doc in docs: row = doc.to_record() metadata = row["metadata"] - links = row["links"] emb = row["embedding"] or [] emb_str = "[" + ",".join(str(x) for x in emb) + "]" if emb else None conn.execute( @@ -55,15 +54,15 @@ class RagDocumentRepository: INSERT INTO rag_chunks ( rag_session_id, path, chunk_index, content, embedding, artifact_type, section, doc_id, doc_version, owner, system_component, last_modified, staleness_score, created_at, updated_at, - rag_doc_id, layer, lang, repo_id, commit_sha, title, metadata_json, links_json, span_start, - span_end, symbol_id, qname, kind, framework, entrypoint_type, module_id, section_path, doc_kind + layer, lang, repo_id, commit_sha, title, metadata_json, span_start, span_end, symbol_id, + qname, kind, framework, entrypoint_type, module_id, section_path, doc_kind ) VALUES ( :sid, :path, :chunk_index, :content, CAST(:emb AS vector), :artifact_type, :section, :doc_id, :doc_version, :owner, :system_component, :last_modified, :staleness_score, CURRENT_TIMESTAMP, - CURRENT_TIMESTAMP, :rag_doc_id, :layer, :lang, :repo_id, :commit_sha, :title, :metadata_json, - :links_json, :span_start, :span_end, :symbol_id, :qname, :kind, :framework, :entrypoint_type, - :module_id, :section_path, :doc_kind + CURRENT_TIMESTAMP, :layer, :lang, :repo_id, :commit_sha, :title, :metadata_json, + :span_start, :span_end, :symbol_id, :qname, :kind, :framework, :entrypoint_type, :module_id, + :section_path, :doc_kind ) """ ), @@ -81,14 +80,12 @@ class RagDocumentRepository: "system_component": metadata.get("system_component"), "last_modified": metadata.get("last_modified"), "staleness_score": metadata.get("staleness_score"), - "rag_doc_id": row["doc_id"], "layer": row["layer"], "lang": row["lang"], "repo_id": row["repo_id"], "commit_sha": row["commit_sha"], "title": row["title"], "metadata_json": json.dumps(metadata, ensure_ascii=True), - "links_json": json.dumps(links, ensure_ascii=True), "span_start": row["span_start"], "span_end": row["span_end"], "symbol_id": metadata.get("symbol_id"), diff --git a/app/modules/rag/persistence/query_repository.py b/app/modules/rag/persistence/query_repository.py index 4e0d320..b5c0d1b 100644 --- a/app/modules/rag/persistence/query_repository.py +++ b/app/modules/rag/persistence/query_repository.py @@ -4,11 +4,14 @@ import json from sqlalchemy import text -from app.modules.rag.retrieval.query_terms import extract_query_terms +from app.modules.rag.persistence.retrieval_statement_builder import RetrievalStatementBuilder from app.modules.shared.db import get_engine class RagQueryRepository: + def __init__(self) -> None: + self._builder = RetrievalStatementBuilder() + def retrieve( self, rag_session_id: str, @@ -18,89 +21,47 @@ class RagQueryRepository: limit: int = 5, layers: list[str] | None = None, path_prefixes: list[str] | None = None, + exclude_path_prefixes: list[str] | None = None, + exclude_like_patterns: list[str] | None = None, prefer_non_tests: bool = False, ) -> list[dict]: - emb = "[" + ",".join(str(x) for x in query_embedding) + "]" - filters = ["rag_session_id = :sid"] - params: dict = {"sid": rag_session_id, "emb": emb, "lim": limit} - if layers: - filters.append("layer = ANY(:layers)") - params["layers"] = layers - if path_prefixes: - or_filters = [] - for idx, prefix in enumerate(path_prefixes): - key = f"path_{idx}" - params[key] = f"{prefix}%" - or_filters.append(f"path LIKE :{key}") - filters.append("(" + " OR ".join(or_filters) + ")") - term_filters = [] - terms = extract_query_terms(query_text) - for idx, term in enumerate(terms): - exact_key = f"term_exact_{idx}" - prefix_key = f"term_prefix_{idx}" - contains_key = f"term_contains_{idx}" - params[exact_key] = term - params[prefix_key] = f"{term}%" - params[contains_key] = f"%{term}%" - term_filters.append( - "CASE " - f"WHEN lower(COALESCE(qname, '')) = :{exact_key} THEN 0 " - f"WHEN lower(COALESCE(symbol_id, '')) = :{exact_key} THEN 1 " - f"WHEN lower(COALESCE(title, '')) = :{exact_key} THEN 2 " - f"WHEN lower(COALESCE(qname, '')) LIKE :{prefix_key} THEN 3 " - f"WHEN lower(COALESCE(title, '')) LIKE :{prefix_key} THEN 4 " - f"WHEN lower(COALESCE(path, '')) LIKE :{contains_key} THEN 5 " - f"WHEN lower(COALESCE(content, '')) LIKE :{contains_key} THEN 6 " - "ELSE 100 END" - ) - lexical_sql = "LEAST(" + ", ".join(term_filters) + ")" if term_filters else "100" - test_penalty_sql = ( - "CASE " - "WHEN lower(path) LIKE 'tests/%' OR lower(path) LIKE '%/tests/%' OR lower(path) LIKE 'test_%' OR lower(path) LIKE '%/test_%' " - "THEN 1 ELSE 0 END" - if prefer_non_tests - else "0" + sql, params = self._builder.build_retrieve( + rag_session_id, + query_embedding, + query_text=query_text, + limit=limit, + layers=layers, + path_prefixes=path_prefixes, + exclude_path_prefixes=exclude_path_prefixes, + exclude_like_patterns=exclude_like_patterns, + prefer_non_tests=prefer_non_tests, ) - layer_rank_sql = ( - "CASE " - "WHEN layer = 'C3_ENTRYPOINTS' THEN 0 " - "WHEN layer = 'C1_SYMBOL_CATALOG' THEN 1 " - "WHEN layer = 'C2_DEPENDENCY_GRAPH' THEN 2 " - "WHEN layer = 'C0_SOURCE_CHUNKS' THEN 3 " - "WHEN layer = 'D1_MODULE_CATALOG' THEN 0 " - "WHEN layer = 'D2_FACT_INDEX' THEN 1 " - "WHEN layer = 'D3_SECTION_INDEX' THEN 2 " - "WHEN layer = 'D4_POLICY_INDEX' THEN 3 " - "ELSE 10 END" - ) - sql = f""" - SELECT path, content, layer, title, metadata_json, span_start, span_end, - {lexical_sql} AS lexical_rank, - {test_penalty_sql} AS test_penalty, - {layer_rank_sql} AS layer_rank, - (embedding <=> CAST(:emb AS vector)) AS distance - FROM rag_chunks - WHERE {' AND '.join(filters)} - ORDER BY lexical_rank ASC, test_penalty ASC, layer_rank ASC, embedding <=> CAST(:emb AS vector) - LIMIT :lim - """ with get_engine().connect() as conn: rows = conn.execute(text(sql), params).mappings().fetchall() return [self._row_to_dict(row) for row in rows] - def fallback_chunks(self, rag_session_id: str, *, limit: int = 5, layers: list[str] | None = None) -> list[dict]: - filters = ["rag_session_id = :sid"] - params: dict = {"sid": rag_session_id, "lim": limit} - if layers: - filters.append("layer = ANY(:layers)") - params["layers"] = layers - sql = f""" - SELECT path, content, layer, title, metadata_json, span_start, span_end - FROM rag_chunks - WHERE {' AND '.join(filters)} - ORDER BY id DESC - LIMIT :lim - """ + def retrieve_lexical_code( + self, + rag_session_id: str, + *, + query_text: str, + limit: int = 5, + path_prefixes: list[str] | None = None, + exclude_path_prefixes: list[str] | None = None, + exclude_like_patterns: list[str] | None = None, + prefer_non_tests: bool = False, + ) -> list[dict]: + sql, params = self._builder.build_lexical_code( + rag_session_id, + query_text=query_text, + limit=limit, + path_prefixes=path_prefixes, + exclude_path_prefixes=exclude_path_prefixes, + exclude_like_patterns=exclude_like_patterns, + prefer_non_tests=prefer_non_tests, + ) + if sql is None: + return [] with get_engine().connect() as conn: rows = conn.execute(text(sql), params).mappings().fetchall() return [self._row_to_dict(row) for row in rows] diff --git a/app/modules/rag/persistence/repository.py b/app/modules/rag/persistence/repository.py index a8418f5..54baefa 100644 --- a/app/modules/rag/persistence/repository.py +++ b/app/modules/rag/persistence/repository.py @@ -67,6 +67,9 @@ class RagRepository: query_text: str = "", limit: int = 5, layers: list[str] | None = None, + path_prefixes: list[str] | None = None, + exclude_path_prefixes: list[str] | None = None, + exclude_like_patterns: list[str] | None = None, prefer_non_tests: bool = False, ) -> list[dict]: return self._query.retrieve( @@ -75,8 +78,29 @@ class RagRepository: query_text=query_text, limit=limit, layers=layers, + path_prefixes=path_prefixes, + exclude_path_prefixes=exclude_path_prefixes, + exclude_like_patterns=exclude_like_patterns, prefer_non_tests=prefer_non_tests, ) - def fallback_chunks(self, rag_session_id: str, limit: int = 5, layers: list[str] | None = None) -> list[dict]: - return self._query.fallback_chunks(rag_session_id, limit=limit, layers=layers) + def retrieve_lexical_code( + self, + rag_session_id: str, + query_text: str, + *, + limit: int = 5, + path_prefixes: list[str] | None = None, + exclude_path_prefixes: list[str] | None = None, + exclude_like_patterns: list[str] | None = None, + prefer_non_tests: bool = False, + ) -> list[dict]: + return self._query.retrieve_lexical_code( + rag_session_id, + query_text=query_text, + limit=limit, + path_prefixes=path_prefixes, + exclude_path_prefixes=exclude_path_prefixes, + exclude_like_patterns=exclude_like_patterns, + prefer_non_tests=prefer_non_tests, + ) diff --git a/app/modules/rag/persistence/retrieval_statement_builder.py b/app/modules/rag/persistence/retrieval_statement_builder.py new file mode 100644 index 0000000..a9972bf --- /dev/null +++ b/app/modules/rag/persistence/retrieval_statement_builder.py @@ -0,0 +1,201 @@ +from __future__ import annotations + +from app.modules.rag.retrieval.query_terms import extract_query_terms + +_LIKE_ESCAPE_SQL = " ESCAPE E'\\\\'" + + +class RetrievalStatementBuilder: + def build_retrieve( + self, + rag_session_id: str, + query_embedding: list[float], + *, + query_text: str = "", + limit: int = 5, + layers: list[str] | None = None, + path_prefixes: list[str] | None = None, + exclude_path_prefixes: list[str] | None = None, + exclude_like_patterns: list[str] | None = None, + prefer_non_tests: bool = False, + ) -> tuple[str, dict]: + emb = "[" + ",".join(str(x) for x in query_embedding) + "]" + filters = ["rag_session_id = :sid"] + params: dict = {"sid": rag_session_id, "emb": emb, "lim": limit} + self._append_prefix_group(filters, params, "path", path_prefixes) + self._append_prefix_group(filters, params, "exclude_prefix", exclude_path_prefixes, negate=True) + self._append_like_group(filters, params, "exclude_like", exclude_like_patterns, negate=True) + if layers: + filters.append("layer = ANY(:layers)") + params["layers"] = layers + lexical_sql = self._lexical_rank_sql(query_text, params) + test_penalty_sql = self._test_penalty_sql( + prefer_non_tests, + params, + base_key="penalty", + path_prefixes=exclude_path_prefixes, + like_patterns=exclude_like_patterns, + ) + layer_rank_sql = ( + "CASE " + "WHEN layer = 'C3_ENTRYPOINTS' THEN 0 " + "WHEN layer = 'C1_SYMBOL_CATALOG' THEN 1 " + "WHEN layer = 'C2_DEPENDENCY_GRAPH' THEN 2 " + "WHEN layer = 'C0_SOURCE_CHUNKS' THEN 3 " + "WHEN layer = 'D1_MODULE_CATALOG' THEN 0 " + "WHEN layer = 'D2_FACT_INDEX' THEN 1 " + "WHEN layer = 'D3_SECTION_INDEX' THEN 2 " + "WHEN layer = 'D4_POLICY_INDEX' THEN 3 " + "ELSE 10 END" + ) + sql = f""" + SELECT path, content, layer, title, metadata_json, span_start, span_end, + {lexical_sql} AS lexical_rank, + {test_penalty_sql} AS test_penalty, + {layer_rank_sql} AS layer_rank, + (embedding <=> CAST(:emb AS vector)) AS distance + FROM rag_chunks + WHERE {' AND '.join(filters)} + ORDER BY lexical_rank ASC, test_penalty ASC, layer_rank ASC, embedding <=> CAST(:emb AS vector) + LIMIT :lim + """ + return sql, params + + def build_lexical_code( + self, + rag_session_id: str, + *, + query_text: str, + limit: int = 5, + path_prefixes: list[str] | None = None, + exclude_path_prefixes: list[str] | None = None, + exclude_like_patterns: list[str] | None = None, + prefer_non_tests: bool = False, + ) -> tuple[str | None, dict]: + terms = extract_query_terms(query_text) + if not terms: + return None, {} + filters = ["rag_session_id = :sid", "layer = 'C0_SOURCE_CHUNKS'"] + params: dict = {"sid": rag_session_id, "lim": limit} + self._append_prefix_group(filters, params, "path", path_prefixes) + self._append_prefix_group(filters, params, "exclude_prefix", exclude_path_prefixes, negate=True) + self._append_like_group(filters, params, "exclude_like", exclude_like_patterns, negate=True) + lexical_filters: list[str] = [] + lexical_ranks: list[str] = [] + for idx, term in enumerate(terms): + exact_key = f"lex_exact_{idx}" + prefix_key = f"lex_prefix_{idx}" + contains_key = f"lex_contains_{idx}" + params[exact_key] = term + params[prefix_key] = f"{term}%" + params[contains_key] = f"%{term}%" + lexical_filters.append( + f"(lower(COALESCE(qname, '')) = :{exact_key} " + f"OR lower(COALESCE(title, '')) = :{exact_key} " + f"OR lower(COALESCE(path, '')) LIKE :{contains_key} " + f"OR lower(COALESCE(title, '')) LIKE :{prefix_key} " + f"OR lower(COALESCE(content, '')) LIKE :{contains_key})" + ) + lexical_ranks.append( + "CASE " + f"WHEN lower(COALESCE(qname, '')) = :{exact_key} THEN 0 " + f"WHEN lower(COALESCE(title, '')) = :{exact_key} THEN 1 " + f"WHEN lower(COALESCE(title, '')) LIKE :{prefix_key} THEN 2 " + f"WHEN lower(COALESCE(path, '')) LIKE :{contains_key} THEN 3 " + f"WHEN lower(COALESCE(content, '')) LIKE :{contains_key} THEN 4 " + "ELSE 100 END" + ) + filters.append("(" + " OR ".join(lexical_filters) + ")") + lexical_sql = "LEAST(" + ", ".join(lexical_ranks) + ")" + test_penalty_sql = self._test_penalty_sql( + prefer_non_tests, + params, + base_key="lex_penalty", + path_prefixes=exclude_path_prefixes, + like_patterns=exclude_like_patterns, + ) + sql = f""" + SELECT path, content, layer, title, metadata_json, span_start, span_end, + {lexical_sql} AS lexical_rank, + {test_penalty_sql} AS test_penalty + FROM rag_chunks + WHERE {' AND '.join(filters)} + ORDER BY lexical_rank ASC, test_penalty ASC, path ASC, span_start ASC + LIMIT :lim + """ + return sql, params + + def _lexical_rank_sql(self, query_text: str, params: dict) -> str: + term_filters: list[str] = [] + for idx, term in enumerate(extract_query_terms(query_text)): + exact_key = f"term_exact_{idx}" + prefix_key = f"term_prefix_{idx}" + contains_key = f"term_contains_{idx}" + params[exact_key] = term + params[prefix_key] = f"{term}%" + params[contains_key] = f"%{term}%" + term_filters.append( + "CASE " + f"WHEN lower(COALESCE(qname, '')) = :{exact_key} THEN 0 " + f"WHEN lower(COALESCE(symbol_id, '')) = :{exact_key} THEN 1 " + f"WHEN lower(COALESCE(title, '')) = :{exact_key} THEN 2 " + f"WHEN lower(COALESCE(qname, '')) LIKE :{prefix_key} THEN 3 " + f"WHEN lower(COALESCE(title, '')) LIKE :{prefix_key} THEN 4 " + f"WHEN lower(COALESCE(path, '')) LIKE :{contains_key} THEN 5 " + f"WHEN lower(COALESCE(content, '')) LIKE :{contains_key} THEN 6 " + "ELSE 100 END" + ) + return "LEAST(" + ", ".join(term_filters) + ")" if term_filters else "100" + + def _append_prefix_group(self, filters: list[str], params: dict, base_key: str, prefixes: list[str] | None, *, negate: bool = False) -> None: + if not prefixes: + return + items: list[str] = [] + for idx, prefix in enumerate(prefixes): + key = f"{base_key}_{idx}" + params[key] = self._escape_like_value(prefix) + "%" + items.append(f"path LIKE :{key}{_LIKE_ESCAPE_SQL}") + self._append_group(filters, items, negate=negate) + + def _append_like_group(self, filters: list[str], params: dict, base_key: str, patterns: list[str] | None, *, negate: bool = False) -> None: + if not patterns: + return + items: list[str] = [] + for idx, pattern in enumerate(patterns): + key = f"{base_key}_{idx}" + params[key] = pattern + items.append(f"lower(path) LIKE :{key}{_LIKE_ESCAPE_SQL}") + self._append_group(filters, items, negate=negate) + + def _append_group(self, filters: list[str], parts: list[str], *, negate: bool) -> None: + if not parts: + return + joined = " OR ".join(parts) + filters.append(f"NOT ({joined})" if negate else f"({joined})") + + def _test_penalty_sql( + self, + enabled: bool, + params: dict, + *, + base_key: str, + path_prefixes: list[str] | None, + like_patterns: list[str] | None, + ) -> str: + if not enabled: + return "0" + parts: list[str] = [] + for idx, prefix in enumerate(path_prefixes or []): + key = f"{base_key}_prefix_{idx}" + params[key] = self._escape_like_value(prefix) + "%" + parts.append(f"lower(path) LIKE :{key}{_LIKE_ESCAPE_SQL}") + for idx, pattern in enumerate(like_patterns or []): + key = f"{base_key}_like_{idx}" + params[key] = pattern + parts.append(f"lower(path) LIKE :{key}{_LIKE_ESCAPE_SQL}") + if not parts: + return "0" + return "CASE WHEN " + " OR ".join(parts) + " THEN 1 ELSE 0 END" + + def _escape_like_value(self, value: str) -> str: + return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") diff --git a/app/modules/rag/persistence/schema_repository.py b/app/modules/rag/persistence/schema_repository.py index 5648165..db03ae7 100644 --- a/app/modules/rag/persistence/schema_repository.py +++ b/app/modules/rag/persistence/schema_repository.py @@ -106,6 +106,7 @@ class RagSchemaRepository: ) self._ensure_columns(conn) self._ensure_indexes(conn) + self._drop_unused_rag_chunk_columns(conn) conn.commit() def _ensure_columns(self, conn) -> None: @@ -118,14 +119,12 @@ class RagSchemaRepository: "ALTER TABLE rag_chunks ADD COLUMN IF NOT EXISTS system_component TEXT NULL", "ALTER TABLE rag_chunks ADD COLUMN IF NOT EXISTS last_modified TIMESTAMPTZ NULL", "ALTER TABLE rag_chunks ADD COLUMN IF NOT EXISTS staleness_score DOUBLE PRECISION NULL", - "ALTER TABLE rag_chunks ADD COLUMN IF NOT EXISTS rag_doc_id VARCHAR(128) NULL", "ALTER TABLE rag_chunks ADD COLUMN IF NOT EXISTS layer VARCHAR(64) NULL", "ALTER TABLE rag_chunks ADD COLUMN IF NOT EXISTS lang VARCHAR(32) NULL", "ALTER TABLE rag_chunks ADD COLUMN IF NOT EXISTS repo_id VARCHAR(512) NULL", "ALTER TABLE rag_chunks ADD COLUMN IF NOT EXISTS commit_sha VARCHAR(128) NULL", "ALTER TABLE rag_chunks ADD COLUMN IF NOT EXISTS title TEXT NULL", "ALTER TABLE rag_chunks ADD COLUMN IF NOT EXISTS metadata_json TEXT NULL", - "ALTER TABLE rag_chunks ADD COLUMN IF NOT EXISTS links_json TEXT NULL", "ALTER TABLE rag_chunks ADD COLUMN IF NOT EXISTS span_start INTEGER NULL", "ALTER TABLE rag_chunks ADD COLUMN IF NOT EXISTS span_end INTEGER NULL", "ALTER TABLE rag_chunks ADD COLUMN IF NOT EXISTS symbol_id TEXT NULL", @@ -162,6 +161,13 @@ class RagSchemaRepository: ): conn.execute(text(statement)) + def _drop_unused_rag_chunk_columns(self, conn) -> None: + for statement in ( + "ALTER TABLE rag_chunks DROP COLUMN IF EXISTS rag_doc_id", + "ALTER TABLE rag_chunks DROP COLUMN IF EXISTS links_json", + ): + conn.execute(text(statement)) + def _ensure_indexes(self, conn) -> None: for statement in ( "CREATE INDEX IF NOT EXISTS idx_rag_chunks_session ON rag_chunks (rag_session_id)", diff --git a/app/modules/rag/retrieval/__pycache__/test_filter.cpython-312-pytest-9.0.2.pyc b/app/modules/rag/retrieval/__pycache__/test_filter.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000..c0e2f50 Binary files /dev/null and b/app/modules/rag/retrieval/__pycache__/test_filter.cpython-312-pytest-9.0.2.pyc differ diff --git a/app/modules/rag/retrieval/__pycache__/test_filter.cpython-312.pyc b/app/modules/rag/retrieval/__pycache__/test_filter.cpython-312.pyc new file mode 100644 index 0000000..3a127eb Binary files /dev/null and b/app/modules/rag/retrieval/__pycache__/test_filter.cpython-312.pyc differ diff --git a/app/modules/rag/retrieval/query_router.py b/app/modules/rag/retrieval/query_router.py deleted file mode 100644 index eaa153c..0000000 --- a/app/modules/rag/retrieval/query_router.py +++ /dev/null @@ -1,43 +0,0 @@ -from __future__ import annotations - -from app.modules.rag.contracts import RagLayer, RetrievalMode - - -class RagQueryRouter: - _CODE_HINTS = ( - "как работает код", - "explain code", - "explain the code", - "по коду", - "из кода", - "построй документацию по коду", - "документацию по коду", - "where is implemented", - "где реализовано", - "endpoint", - "handler", - "symbol", - "function", - "class", - "method", - ) - - _DOCS_LAYERS = [ - RagLayer.DOCS_MODULE_CATALOG, - RagLayer.DOCS_FACT_INDEX, - RagLayer.DOCS_SECTION_INDEX, - RagLayer.DOCS_POLICY_INDEX, - ] - _CODE_LAYERS = [ - RagLayer.CODE_ENTRYPOINTS, - RagLayer.CODE_SYMBOL_CATALOG, - RagLayer.CODE_DEPENDENCY_GRAPH, - RagLayer.CODE_SOURCE_CHUNKS, - ] - - def resolve_mode(self, query: str) -> str: - lowered = query.lower() - return RetrievalMode.CODE if any(hint in lowered for hint in self._CODE_HINTS) else RetrievalMode.DOCS - - def layers_for_mode(self, mode: str) -> list[str]: - return list(self._CODE_LAYERS if mode == RetrievalMode.CODE else self._DOCS_LAYERS) diff --git a/app/modules/rag/retrieval/test_filter.py b/app/modules/rag/retrieval/test_filter.py new file mode 100644 index 0000000..2e6118a --- /dev/null +++ b/app/modules/rag/retrieval/test_filter.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import os +import re +from dataclasses import dataclass +from fnmatch import fnmatch +from typing import Iterable + +DEFAULT_TEST_PATH_PATTERNS = ( + "tests/", + "test/", + "__tests__/", + "mocks/", + "fixtures/", + "stubs/", + "conftest.py", + "*_test.*", + "*.test.*", + "*.spec.*", +) + +_TRUE_VALUES = {"1", "true", "yes", "on"} +_SAFE_PATTERN_RE = re.compile(r"^[A-Za-z0-9_./*?-]+$") + + +@dataclass(slots=True) +class RetrievalPathFilter: + exclude_path_prefixes: list[str] + exclude_like_patterns: list[str] + + +def exclude_tests_default() -> bool: + return os.getenv("RAG_EXCLUDE_TESTS_DEFAULT", "true").strip().lower() in _TRUE_VALUES + + +def debug_disable_test_filter() -> bool: + return os.getenv("RAG_DEBUG_DISABLE_TEST_FILTER", "false").strip().lower() in _TRUE_VALUES + + +def configured_test_patterns() -> list[str]: + raw = os.getenv("RAG_TEST_PATH_PATTERNS", "") + if not raw.strip(): + return list(DEFAULT_TEST_PATH_PATTERNS) + return [item.strip() for item in raw.split(",") if item.strip()] + + +def build_test_filters(patterns: Iterable[str] | None = None) -> RetrievalPathFilter: + prefixes: list[str] = [] + like_patterns: list[str] = [] + for pattern in _validated_patterns(patterns or configured_test_patterns()): + if pattern.endswith("/"): + _append(prefixes, pattern) + _append(like_patterns, f"%/{pattern}%") + continue + sql_like = _glob_to_sql_like(pattern) + _append(like_patterns, sql_like) + if "/" not in pattern: + _append(like_patterns, f"%/{sql_like}") + return RetrievalPathFilter(exclude_path_prefixes=prefixes, exclude_like_patterns=like_patterns) + + +def is_test_path(path: str, patterns: Iterable[str] | None = None) -> bool: + normalized = (path or "").strip().lower() + if not normalized: + return False + for pattern in _validated_patterns(patterns or configured_test_patterns()): + if pattern.endswith("/"): + token = pattern.rstrip("/") + if normalized.startswith(pattern) or f"/{token}/" in normalized: + return True + continue + if fnmatch(normalized, pattern) or fnmatch(normalized, f"*/{pattern}"): + return True + return False + + +def _validated_patterns(patterns: Iterable[str]) -> list[str]: + result: list[str] = [] + for raw_pattern in patterns: + pattern = (raw_pattern or "").strip().lower() + if not pattern: + continue + if not _SAFE_PATTERN_RE.fullmatch(pattern): + continue + if pattern not in result: + result.append(pattern) + return result + + +def _glob_to_sql_like(pattern: str) -> str: + escaped = pattern.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + return escaped.replace("*", "%").replace("?", "_") + + +def _append(values: list[str], item: str) -> None: + if item and item not in values: + values.append(item) diff --git a/app/modules/rag/services/__pycache__/rag_service.cpython-312.pyc b/app/modules/rag/services/__pycache__/rag_service.cpython-312.pyc new file mode 100644 index 0000000..423cded Binary files /dev/null and b/app/modules/rag/services/__pycache__/rag_service.cpython-312.pyc differ diff --git a/app/modules/rag/services/rag_service.py b/app/modules/rag/services/rag_service.py index ae2c996..3d9b31f 100644 --- a/app/modules/rag/services/rag_service.py +++ b/app/modules/rag/services/rag_service.py @@ -2,6 +2,7 @@ from __future__ import annotations import asyncio import hashlib +import logging import os from collections.abc import Awaitable, Callable from inspect import isawaitable @@ -11,9 +12,10 @@ from app.modules.rag.indexing.code.pipeline import CodeIndexingPipeline from app.modules.rag.indexing.common.report import IndexReport from app.modules.rag.indexing.docs.pipeline import DocsIndexingPipeline from app.modules.rag.persistence.repository import RagRepository -from app.modules.rag.retrieval.query_router import RagQueryRouter from app.modules.rag_session.embedding.gigachat_embedder import GigaChatEmbedder +LOGGER = logging.getLogger(__name__) + class RagService: def __init__( @@ -26,7 +28,6 @@ class RagService: self._repo = repository self._docs = DocsIndexingPipeline() self._code = CodeIndexingPipeline() - self._queries = RagQueryRouter() async def index_snapshot( self, @@ -55,36 +56,6 @@ class RagService: self._repo.apply_document_changes(rag_session_id, delete_paths, report.documents_list) return report.as_tuple() - async def retrieve(self, rag_session_id: str, query: str) -> list[dict]: - mode = self._queries.resolve_mode(query) - layers = self._queries.layers_for_mode(mode) - prefer_non_tests = mode == "code" and "test" not in query.lower() and "тест" not in query.lower() - try: - query_embedding = self._embedder.embed([query])[0] - rows = self._repo.retrieve( - rag_session_id, - query_embedding, - query_text=query, - limit=8, - layers=layers, - prefer_non_tests=prefer_non_tests, - ) - except Exception: - rows = self._repo.fallback_chunks(rag_session_id, limit=8, layers=layers) - if not rows and mode != "docs": - rows = self._repo.fallback_chunks(rag_session_id, limit=8, layers=self._queries.layers_for_mode("docs")) - return [ - { - "source": row["path"], - "content": row["content"], - "layer": row.get("layer"), - "title": row.get("title"), - "metadata": row.get("metadata", {}), - "score": row.get("distance"), - } - for row in rows - ] - async def _index_files( self, rag_session_id: str, @@ -99,15 +70,28 @@ class RagService: try: blob_sha = self._blob_sha(file) cached = await asyncio.to_thread(self._repo.get_cached_documents, repo_id, blob_sha) + pipelines = self._resolve_pipeline_names(path) if cached: report.documents_list.extend(self._with_file_metadata(cached, file, repo_id, blob_sha)) report.cache_hit_files += 1 + LOGGER.warning( + "rag ingest file: rag_session_id=%s path=%s processing=cache pipeline=%s", + rag_session_id, + path, + ",".join(pipelines), + ) else: built = self._build_documents(repo_id, path, file) embedded = await asyncio.to_thread(self._embed_documents, built, file, repo_id, blob_sha) report.documents_list.extend(embedded) await asyncio.to_thread(self._repo.cache_documents, repo_id, path, blob_sha, embedded) report.cache_miss_files += 1 + LOGGER.warning( + "rag ingest file: rag_session_id=%s path=%s processing=embed pipeline=%s", + rag_session_id, + path, + ",".join(pipelines), + ) report.indexed_files += 1 except Exception as exc: report.failed_files += 1 @@ -128,6 +112,16 @@ class RagService: docs.extend(self._docs.index_file(repo_id=repo_id, commit_sha=commit_sha, path=path, content=content)) return docs + def _resolve_pipeline_names(self, path: str) -> list[str]: + names: list[str] = [] + if self._docs.supports(path): + names.append("DOCS") + if self._code.supports(path): + names.append("CODE") + if not names: + names.append("DOCS") + return names + def _embed_documents(self, docs: list[RagDocument], file: dict, repo_id: str, blob_sha: str) -> list[RagDocument]: if not docs: return [] @@ -190,7 +184,6 @@ class RagService: if isawaitable(result): await result - class _PipelineReport(IndexReport): def __init__(self) -> None: super().__init__() diff --git a/app/modules/rag_session/module.py b/app/modules/rag_session/module.py index b44f953..63ea414 100644 --- a/app/modules/rag_session/module.py +++ b/app/modules/rag_session/module.py @@ -1,5 +1,5 @@ from fastapi import APIRouter -from fastapi.responses import StreamingResponse +from fastapi.responses import JSONResponse, StreamingResponse from app.core.exceptions import AppError from app.modules.rag_session.embedding.gigachat_embedder import GigaChatEmbedder @@ -37,6 +37,7 @@ class RagModule: token_provider = GigaChatTokenProvider(settings) client = GigaChatClient(settings, token_provider) embedder = GigaChatEmbedder(client) + self.embedder = embedder self.rag = RagService(embedder=embedder, repository=repository, chunker=TextChunker()) self.sessions = RagSessionStore(repository) self.jobs = IndexJobStore(repository) @@ -252,12 +253,13 @@ class RagModule: } @router.post("/retrieve") - async def retrieve(payload: dict) -> dict: - rag_session_id = payload.get("rag_session_id") or payload.get("project_id", "") - ctx = await self.rag.retrieve( - rag_session_id=rag_session_id, - query=payload.get("query", ""), + async def retrieve() -> JSONResponse: + return JSONResponse( + status_code=410, + content={ + "error": "deprecated", + "message": "POST /internal/rag/retrieve is deprecated.", + }, ) - return {"items": ctx} return router diff --git a/app/modules/shared/__pycache__/env_loader.cpython-312.pyc b/app/modules/shared/__pycache__/env_loader.cpython-312.pyc new file mode 100644 index 0000000..50fbdf9 Binary files /dev/null and b/app/modules/shared/__pycache__/env_loader.cpython-312.pyc differ diff --git a/app/modules/shared/env_loader.py b/app/modules/shared/env_loader.py new file mode 100644 index 0000000..6f04104 --- /dev/null +++ b/app/modules/shared/env_loader.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import os +from pathlib import Path + +_ENV_FILES = (".env", ".env.local") + + +def load_workspace_env(start_dir: str | Path | None = None) -> list[Path]: + base = Path(start_dir or Path.cwd()).resolve() + loaded: list[Path] = [] + for directory in reversed((base, *base.parents)): + for file_name in _ENV_FILES: + path = directory / file_name + if not path.is_file(): + continue + _load_env_file(path) + loaded.append(path) + return loaded + + +def _load_env_file(path: Path) -> None: + for raw_line in path.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, raw_value = line.split("=", 1) + name = key.removeprefix("export ").strip() + if not name or name in os.environ: + continue + os.environ[name] = _normalize_value(raw_value.strip()) + + +def _normalize_value(value: str) -> str: + if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: + return value[1:-1] + return value diff --git a/app/modules/shared/gigachat/__pycache__/client.cpython-312.pyc b/app/modules/shared/gigachat/__pycache__/client.cpython-312.pyc index 6a0c283..3de6534 100644 Binary files a/app/modules/shared/gigachat/__pycache__/client.cpython-312.pyc and b/app/modules/shared/gigachat/__pycache__/client.cpython-312.pyc differ diff --git a/app/modules/shared/gigachat/client.py b/app/modules/shared/gigachat/client.py index 2b00def..5096bbe 100644 --- a/app/modules/shared/gigachat/client.py +++ b/app/modules/shared/gigachat/client.py @@ -1,5 +1,8 @@ +import time + import requests +from app.core.constants import MAX_RETRIES from app.modules.shared.gigachat.errors import GigaChatError from app.modules.shared.gigachat.settings import GigaChatSettings from app.modules.shared.gigachat.token_provider import GigaChatTokenProvider @@ -19,23 +22,7 @@ class GigaChatClient: {"role": "user", "content": user_prompt}, ], } - try: - response = requests.post( - f"{self._settings.api_url.rstrip('/')}/chat/completions", - json=payload, - headers={ - "Authorization": f"Bearer {token}", - "Content-Type": "application/json", - }, - timeout=90, - verify=self._settings.ssl_verify, - ) - except requests.RequestException as exc: - raise GigaChatError(f"GigaChat completion request failed: {exc}") from exc - - if response.status_code >= 400: - raise GigaChatError(f"GigaChat completion error {response.status_code}: {response.text}") - + response = self._post_with_retry("/chat/completions", payload, token=token, timeout=90, operation_name="completion") data = response.json() choices = data.get("choices") or [] if not choices: @@ -49,25 +36,49 @@ class GigaChatClient: "model": self._settings.embedding_model, "input": texts, } - try: - response = requests.post( - f"{self._settings.api_url.rstrip('/')}/embeddings", - json=payload, - headers={ - "Authorization": f"Bearer {token}", - "Content-Type": "application/json", - }, - timeout=90, - verify=self._settings.ssl_verify, - ) - except requests.RequestException as exc: - raise GigaChatError(f"GigaChat embeddings request failed: {exc}") from exc - - if response.status_code >= 400: - raise GigaChatError(f"GigaChat embeddings error {response.status_code}: {response.text}") - + response = self._post_with_retry("/embeddings", payload, token=token, timeout=90, operation_name="embeddings") data = response.json() items = data.get("data") if not isinstance(items, list): raise GigaChatError("Unexpected GigaChat embeddings response") return [list(map(float, x.get("embedding") or [])) for x in items] + + def _post_with_retry( + self, + path: str, + payload: dict, + *, + token: str, + timeout: int, + operation_name: str, + ): + last_error: Exception | None = None + for attempt in range(1, MAX_RETRIES + 1): + try: + response = requests.post( + f"{self._settings.api_url.rstrip('/')}{path}", + json=payload, + headers={ + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + }, + timeout=timeout, + verify=self._settings.ssl_verify, + ) + except requests.RequestException as exc: + last_error = GigaChatError(f"GigaChat {operation_name} request failed: {exc}") + else: + if response.status_code < 400: + return response + last_error = GigaChatError(f"GigaChat {operation_name} error {response.status_code}: {response.text}") + if not self._is_retryable_status(response.status_code): + raise last_error + if attempt == MAX_RETRIES: + break + time.sleep(0.1 * attempt) + if last_error is None: + raise GigaChatError(f"GigaChat {operation_name} failed without response") + raise last_error + + def _is_retryable_status(self, status_code: int) -> bool: + return status_code == 429 or status_code >= 500 diff --git a/app/schemas/__pycache__/rag_sessions.cpython-312.pyc b/app/schemas/__pycache__/rag_sessions.cpython-312.pyc index 5ad9da4..e2c2452 100644 Binary files a/app/schemas/__pycache__/rag_sessions.cpython-312.pyc and b/app/schemas/__pycache__/rag_sessions.cpython-312.pyc differ diff --git a/docs/architecture/contracts_retrieval.json b/docs/architecture/contracts_retrieval.json new file mode 100644 index 0000000..5d0e918 --- /dev/null +++ b/docs/architecture/contracts_retrieval.json @@ -0,0 +1,380 @@ +{ + "layers": { + "C0_SOURCE_CHUNKS": { + "retriever": { + "class": "RagService", + "file": "app/modules/rag/services/rag_service.py", + "method": "retrieve" + }, + "indexer": { + "class": "CodeTextDocumentBuilder", + "file": "app/modules/rag/indexing/code/code_text/document_builder.py", + "method": "build" + }, + "input": { + "type": "observed shape", + "fields": { + "rag_session_id": { + "type": "string", + "required": true + }, + "query": { + "type": "string", + "required": true + }, + "layers": { + "type": "implicit list[string]", + "required": false, + "source": "RagQueryRouter.layers_for_mode('code')" + } + } + }, + "output": { + "type": "list[dict]", + "fields": { + "source": "string", + "content": "string", + "layer": "\"C0_SOURCE_CHUNKS\"", + "title": "string", + "metadata": { + "chunk_index": "int", + "chunk_type": "\"symbol_block\" | \"window\"", + "module_or_unit": "string", + "artifact_type": "\"CODE\"" + }, + "score": "float | null" + } + }, + "examples": { + "input": { + "rag_session_id": "rag-123", + "query": "where is implemented get_user" + }, + "output": { + "source": "app/api/users.py", + "content": "async def get_user(user_id: str):\n service = UserService()\n return service.get_user(user_id)", + "layer": "C0_SOURCE_CHUNKS", + "title": "app/api/users.py:get_user", + "metadata": { + "chunk_index": 0, + "chunk_type": "symbol_block", + "module_or_unit": "app.api.users", + "artifact_type": "CODE" + }, + "score": 0.07 + } + }, + "defaults": { + "retrieve_limit": 8, + "embed_batch_size_env": "RAG_EMBED_BATCH_SIZE", + "embed_batch_size_default": 16, + "window_chunk_size_lines": 80, + "window_overlap_lines": 15 + }, + "limitations": [ + "Line spans are stored but not returned in the public retrieval item shape.", + "No direct path or namespace filter is exposed through the retrieval endpoint." + ] + }, + "C1_SYMBOL_CATALOG": { + "retriever": { + "class": "RagService", + "file": "app/modules/rag/services/rag_service.py", + "method": "retrieve" + }, + "indexer": { + "class": "SymbolDocumentBuilder", + "file": "app/modules/rag/indexing/code/symbols/document_builder.py", + "method": "build" + }, + "input": { + "type": "observed shape", + "fields": { + "rag_session_id": { + "type": "string", + "required": true + }, + "query": { + "type": "string", + "required": true + }, + "query_term_expansion": { + "type": "list[string]", + "required": false, + "source": "extract_query_terms(query_text)", + "max_items": 6 + } + } + }, + "output": { + "type": "list[dict]", + "fields": { + "source": "string", + "content": "string", + "layer": "\"C1_SYMBOL_CATALOG\"", + "title": "string", + "metadata": { + "symbol_id": "string", + "qname": "string", + "kind": "\"class\" | \"function\" | \"method\" | \"const\"", + "signature": "string", + "decorators_or_annotations": "list[string]", + "docstring_or_javadoc": "string | null", + "parent_symbol_id": "string | null", + "package_or_module": "string", + "is_entry_candidate": "bool", + "lang_payload": "object", + "artifact_type": "\"CODE\"" + }, + "score": "float | null" + } + }, + "examples": { + "input": { + "rag_session_id": "rag-123", + "query": "where is implemented get_user" + }, + "output": { + "source": "app/api/users.py", + "content": "function get_user\nget_user(user_id)", + "layer": "C1_SYMBOL_CATALOG", + "title": "get_user", + "metadata": { + "symbol_id": "sha256(...)", + "qname": "get_user", + "kind": "function", + "signature": "get_user(user_id)", + "decorators_or_annotations": [ + "router.get" + ], + "docstring_or_javadoc": null, + "parent_symbol_id": null, + "package_or_module": "app.api.users", + "is_entry_candidate": true, + "lang_payload": { + "async": true + }, + "artifact_type": "CODE" + }, + "score": 0.07 + } + }, + "defaults": { + "retrieve_limit": 8, + "layer_rank": 1 + }, + "limitations": [ + "Only Python AST symbols are indexed.", + "Cross-file resolution is not implemented.", + "parent_symbol_id is an observed qname-like value, not guaranteed to be a symbol hash." + ] + }, + "C2_DEPENDENCY_GRAPH": { + "retriever": { + "class": "RagService", + "file": "app/modules/rag/services/rag_service.py", + "method": "retrieve" + }, + "indexer": { + "class": "EdgeDocumentBuilder", + "file": "app/modules/rag/indexing/code/edges/document_builder.py", + "method": "build" + }, + "input": { + "type": "observed shape", + "fields": { + "rag_session_id": { + "type": "string", + "required": true + }, + "query": { + "type": "string", + "required": true + } + } + }, + "output": { + "type": "list[dict]", + "fields": { + "source": "string", + "content": "string", + "layer": "\"C2_DEPENDENCY_GRAPH\"", + "title": "string", + "metadata": { + "edge_id": "string", + "edge_type": "\"calls\" | \"imports\" | \"inherits\"", + "src_symbol_id": "string", + "src_qname": "string", + "dst_symbol_id": "string | null", + "dst_ref": "string | null", + "resolution": "\"resolved\" | \"partial\"", + "lang_payload": "object", + "artifact_type": "\"CODE\"" + }, + "score": "float | null" + } + }, + "examples": { + "input": { + "rag_session_id": "rag-123", + "query": "how get_user calls service" + }, + "output": { + "source": "app/api/users.py", + "content": "get_user calls UserService", + "layer": "C2_DEPENDENCY_GRAPH", + "title": "get_user:calls", + "metadata": { + "edge_id": "sha256(...)", + "edge_type": "calls", + "src_symbol_id": "sha256(...)", + "src_qname": "get_user", + "dst_symbol_id": null, + "dst_ref": "UserService", + "resolution": "partial", + "lang_payload": { + "callsite_kind": "function_call" + }, + "artifact_type": "CODE" + }, + "score": 0.11 + } + }, + "defaults": { + "retrieve_limit": 8, + "layer_rank": 2, + "graph_build_mode": "static_python_ast" + }, + "limitations": [ + "No traversal API exists.", + "Edges are stored as retrievable rows, not as a graph-native store.", + "Destination resolution is local to one indexed file." + ] + }, + "C3_ENTRYPOINTS": { + "retriever": { + "class": "RagService", + "file": "app/modules/rag/services/rag_service.py", + "method": "retrieve" + }, + "indexer": { + "class": "EntrypointDocumentBuilder", + "file": "app/modules/rag/indexing/code/entrypoints/document_builder.py", + "method": "build" + }, + "input": { + "type": "observed shape", + "fields": { + "rag_session_id": { + "type": "string", + "required": true + }, + "query": { + "type": "string", + "required": true + } + } + }, + "output": { + "type": "list[dict]", + "fields": { + "source": "string", + "content": "string", + "layer": "\"C3_ENTRYPOINTS\"", + "title": "string", + "metadata": { + "entry_id": "string", + "entry_type": "\"http\" | \"cli\"", + "framework": "\"fastapi\" | \"flask\" | \"typer\" | \"click\"", + "route_or_command": "string", + "handler_symbol_id": "string", + "lang_payload": "object", + "artifact_type": "\"CODE\"" + }, + "score": "float | null" + } + }, + "examples": { + "input": { + "rag_session_id": "rag-123", + "query": "which endpoint handles get user" + }, + "output": { + "source": "app/api/users.py", + "content": "fastapi http \"/users/{user_id}\"", + "layer": "C3_ENTRYPOINTS", + "title": "\"/users/{user_id}\"", + "metadata": { + "entry_id": "sha256(...)", + "entry_type": "http", + "framework": "fastapi", + "route_or_command": "\"/users/{user_id}\"", + "handler_symbol_id": "sha256(...)", + "lang_payload": { + "methods": [ + "GET" + ] + }, + "artifact_type": "CODE" + }, + "score": 0.05 + } + }, + "defaults": { + "retrieve_limit": 8, + "layer_rank": 0 + }, + "limitations": [ + "Detection is decorator-string based.", + "No Django, Celery, RQ, or cron entrypoints were found.", + "Returned payload does not expose line spans." + ] + } + }, + "retrieval_endpoint": { + "entrypoint": { + "file": "app/modules/rag_session/module.py", + "method": "internal_router.retrieve" + }, + "request": { + "type": "dict", + "fields": { + "rag_session_id": "string | optional if project_id provided", + "project_id": "string | optional fallback for rag_session_id", + "query": "string" + } + }, + "response": { + "type": "dict", + "fields": { + "items": "list[retrieval item]" + } + }, + "defaults": { + "mode": "docs unless RagQueryRouter detects code hints", + "limit": 8, + "embedding_provider": "GigaChat embeddings", + "fallback_after_embedding_error": true, + "fallback_to_docs_when_code_empty": true + } + }, + "ranking": { + "storage": "PostgreSQL rag_chunks + pgvector", + "query_repository": { + "class": "RagQueryRepository", + "file": "app/modules/rag/persistence/query_repository.py", + "method": "retrieve" + }, + "order_by": [ + "lexical_rank ASC", + "test_penalty ASC", + "layer_rank ASC", + "embedding <=> query_embedding ASC" + ], + "notes": [ + "lexical_rank is derived from qname/symbol_id/title/path/content matching extracted query terms", + "test_penalty is applied only when prefer_non_tests=true", + "layer priority is C3 > C1 > C2 > C0 for code retrieval" + ] + } +} diff --git a/docs/architecture/llm_inventory.md b/docs/architecture/llm_inventory.md new file mode 100644 index 0000000..1156e69 --- /dev/null +++ b/docs/architecture/llm_inventory.md @@ -0,0 +1,270 @@ +# LLM Inventory + +## Provider and SDK + +- Provider in code: GigaChat / Sber +- Local SDK style: custom thin HTTP client over `requests` +- Core files: + - `app/modules/shared/gigachat/client.py` + - `app/modules/shared/gigachat/settings.py` + - `app/modules/shared/gigachat/token_provider.py` + - `app/modules/agent/llm/service.py` + +There is no OpenAI SDK, Azure SDK, or local model runtime in the current implementation. + +## Configuration + +Model and endpoint configuration are read from environment in `GigaChatSettings.from_env()`: + +- `GIGACHAT_AUTH_URL` + - default: `https://ngw.devices.sberbank.ru:9443/api/v2/oauth` +- `GIGACHAT_API_URL` + - default: `https://gigachat.devices.sberbank.ru/api/v1` +- `GIGACHAT_SCOPE` + - default: `GIGACHAT_API_PERS` +- `GIGACHAT_TOKEN` + - required for auth +- `GIGACHAT_SSL_VERIFY` + - default: `true` +- `GIGACHAT_MODEL` + - default: `GigaChat` +- `GIGACHAT_EMBEDDING_MODEL` + - default: `Embeddings` +- `AGENT_PROMPTS_DIR` + - optional prompt directory override + +PostgreSQL config for retrieval storage is separate: + +- `DATABASE_URL` + - default: `postgresql+psycopg://agent:agent@db:5432/agent` + +## Default models + +- Chat/completions model default: `GigaChat` +- Embedding model default: `Embeddings` + +## Completion payload + +Observed payload sent by `GigaChatClient.complete(...)`: + +```json +{ + "model": "GigaChat", + "messages": [ + {"role": "system", "content": ""}, + {"role": "user", "content": ""} + ] +} +``` + +Endpoint: + +- `POST {GIGACHAT_API_URL}/chat/completions` + +Observed response handling: + +- reads `choices[0].message.content` +- if no choices: returns empty string + +## Embeddings payload + +Observed payload sent by `GigaChatClient.embed(...)`: + +```json +{ + "model": "Embeddings", + "input": [ + "", + "" + ] +} +``` + +Endpoint: + +- `POST {GIGACHAT_API_URL}/embeddings` + +Observed response handling: + +- expects `data` list +- maps each `item.embedding` to `list[float]` + +## Parameters + +### Explicitly implemented + +- `model` +- `messages` +- `input` +- HTTP timeout: + - completions: `90s` + - embeddings: `90s` + - auth: `30s` +- TLS verification flag: + - `verify=settings.ssl_verify` + +### Not implemented in payload + +- `temperature` +- `top_p` +- `max_tokens` +- `response_format` +- tools/function calling +- streaming +- seed +- stop sequences + +`ASSUMPTION:` the service uses provider defaults for sampling and output length because these fields are not sent in the request payload. + +## Context and budget limits + +There is no centralized token budget manager in the current code. + +Observed practical limits instead: + +- prompt file text is loaded as-is from disk +- user input is passed as-is +- RAG context shaping happens outside the LLM client +- docs indexing summary truncation: + - docs module catalog summary: `4000` chars + - docs policy text: `4000` chars +- project QA source bundle caps: + - top `12` rag items + - top `10` file candidates +- logging truncation only: + - LLM input/output logs capped at `1500` chars for logs + +`ASSUMPTION:` there is no explicit max-context enforcement before chat completion requests. The current system relies on upstream graph logic to keep inputs small enough. + +## Retry, backoff, timeout + +### Timeouts + +- auth: `30s` +- chat completion: `90s` +- embeddings: `90s` + +### Retry + +- Generic async retry wrapper exists in `app/modules/shared/retry_executor.py` +- It retries only: + - `TimeoutError` + - `ConnectionError` + - `OSError` +- Retry constants: + - `MAX_RETRIES = 5` + - backoff: `0.1 * attempt` seconds + +### Important current limitation + +- `GigaChatClient` raises `GigaChatError` on HTTP and request failures. +- `RetryExecutor` does not catch `GigaChatError`. +- Result: LLM and embeddings calls are effectively not retried by this generic retry helper unless errors are converted upstream. + +## Prompt formation + +Prompt loading is handled by `PromptLoader`: + +- base dir: `app/modules/agent/prompts` +- override: `AGENT_PROMPTS_DIR` +- file naming convention: `.txt` + +Prompt composition model today: + +- system prompt: + - full contents of selected prompt file +- user prompt: + - raw runtime input string passed by the caller +- no separate developer prompt layer in the application payload + +If a prompt file is missing: + +- fallback system prompt: `You are a helpful assistant.` + +## Prompt templates present + +- `router_intent` +- `general_answer` +- `project_answer` +- `docs_detect` +- `docs_strategy` +- `docs_plan_sections` +- `docs_generation` +- `docs_self_check` +- `docs_execution_summary` +- `project_edits_plan` +- `project_edits_hunks` +- `project_edits_self_check` + +## Key LLM call entrypoints + +### Composition roots + +- `app/modules/agent/module.py` + - builds `GigaChatSettings` + - builds `GigaChatTokenProvider` + - builds `GigaChatClient` + - builds `PromptLoader` + - builds `AgentLlmService` +- `app/modules/rag_session/module.py` + - builds the same provider stack for embeddings used by RAG + +### Main abstraction + +- `AgentLlmService.generate(prompt_name, user_input, log_context=None)` + +### Current generate callsites + +- `app/modules/agent/engine/router/intent_classifier.py` + - `router_intent` +- `app/modules/agent/engine/graphs/base_graph.py` + - `general_answer` +- `app/modules/agent/engine/graphs/project_qa_graph.py` + - `project_answer` +- `app/modules/agent/engine/graphs/docs_graph_logic.py` + - `docs_detect` + - `docs_strategy` + - `docs_plan_sections` + - `docs_generation` + - `docs_self_check` + - `docs_execution_summary`-like usage via summary step +- `app/modules/agent/engine/graphs/project_edits_logic.py` + - `project_edits_plan` + - `project_edits_self_check` + - `project_edits_hunks` + +## Logging and observability + +`AgentLlmService` logs: + +- input: + - `graph llm input: context=... prompt=... user_input=...` +- output: + - `graph llm output: context=... prompt=... output=...` + +Log truncation: + +- 1500 chars + +RAG retrieval logs separately in `RagService`, but without embedding vectors. + +## Integration with retrieval + +There are two distinct GigaChat usages: + +1. Chat/completion path for agent reasoning and generation +2. Embedding path for RAG indexing and retrieval + +The embedding adapter is `GigaChatEmbedder`, used by: + +- `app/modules/rag/services/rag_service.py` + +## Notable limitations + +- Single provider coupling: chat and embeddings both depend on GigaChat-specific endpoints. +- No model routing by scenario. +- No tool/function calling. +- No centralized prompt token budgeting. +- No explicit retry for `GigaChatError`. +- No streaming completions. +- No structured response mode beyond prompt conventions and downstream parsing. diff --git a/docs/architecture/rag_chunks_column_audit.md b/docs/architecture/rag_chunks_column_audit.md new file mode 100644 index 0000000..68aaf26 --- /dev/null +++ b/docs/architecture/rag_chunks_column_audit.md @@ -0,0 +1,13 @@ +| column | used_by | safe_to_drop | notes | +| --- | --- | --- | --- | +| `layer` | `USED_BY_CODE_V2`, `USED_BY_DOCS_INDEXING` | no | Core selector for C0-C3 and D1-D4 queries. | +| `title` | `USED_BY_CODE_V2`, `USED_BY_DOCS_INDEXING` | no | Used in lexical ranking and prompt evidence labels. | +| `metadata_json` | `USED_BY_CODE_V2`, `USED_BY_DOCS_INDEXING` | no | C2/C0 graph lookups and docs metadata depend on it. | +| `span_start`, `span_end` | `USED_BY_CODE_V2` | no | Needed for symbol-to-chunk resolution and locations. | +| `symbol_id`, `qname`, `kind`, `lang` | `USED_BY_CODE_V2` | no | Used by code indexing, ranking, trace building, and diagnostics. | +| `repo_id`, `commit_sha` | `USED_BY_CODE_V2`, `USED_BY_DOCS_INDEXING` | no | Used by indexing/cache and retained for provenance. | +| `entrypoint_type`, `framework` | `USED_BY_CODE_V2` | no | Used by C3 filtering and entrypoint diagnostics. | +| `doc_kind`, `module_id`, `section_path` | `USED_BY_DOCS_INDEXING` | no | Still written by docs indexing and covered by docs tests. | +| `artifact_type`, `section`, `doc_version`, `owner`, `system_component`, `last_modified`, `staleness_score` | `USED_BY_DOCS_INDEXING` | no | File metadata still flows through indexing/cache; left intact for now. | +| `rag_doc_id` | `UNUSED` | yes | Written into `rag_chunks` only; no reads in runtime/indexing code. | +| `links_json` | `UNUSED` | yes | Stored in `rag_chunks` only; reads exist for `rag_chunk_cache`, not `rag_chunks`. | diff --git a/docs/architecture/retrieval_callgraph.mmd b/docs/architecture/retrieval_callgraph.mmd new file mode 100644 index 0000000..95e3795 --- /dev/null +++ b/docs/architecture/retrieval_callgraph.mmd @@ -0,0 +1,31 @@ +flowchart TD + A["HTTP: POST /internal/rag/retrieve"] --> B["RagModule.internal_router.retrieve(payload)"] + B --> C["RagService.retrieve(rag_session_id, query)"] + C --> D["RagQueryRouter.resolve_mode(query)"] + D --> E["RagQueryRouter.layers_for_mode(mode)"] + C --> F["GigaChatEmbedder.embed([query])"] + F --> G["GigaChatClient.embed(payload)"] + G --> H["POST /embeddings"] + C --> I["RagRepository.retrieve(...)"] + I --> J["RagQueryRepository.retrieve(...)"] + J --> K["PostgreSQL rag_chunks + pgvector"] + K --> L["ORDER BY lexical_rank, test_penalty, layer_rank, vector distance"] + L --> M["rows: path/content/layer/title/metadata/span/distance"] + M --> N["normalize to {source, content, layer, title, metadata, score}"] + N --> O["response: {items: [...]}"] + + C --> P["embedding error?"] + P -->|yes| Q["RagRepository.fallback_chunks(...)"] + Q --> R["latest rows by id DESC"] + R --> N + + C --> S["no rows and mode != docs?"] + S -->|yes| T["fallback to docs layers"] + T --> I + + U["GraphAgentRuntime for project/qa"] --> V["ProjectQaRetrievalGraphFactory._retrieve_context"] + V --> C + V --> W["ProjectQaSupport.build_source_bundle(...)"] + W --> X["source_bundle"] + X --> Y["context_analysis"] + Y --> Z["answer_composition"] diff --git a/docs/architecture/retrieval_inventory.md b/docs/architecture/retrieval_inventory.md new file mode 100644 index 0000000..ed1a709 --- /dev/null +++ b/docs/architecture/retrieval_inventory.md @@ -0,0 +1,457 @@ +# Retrieval Inventory + +## Scope and method + +This document describes the retrieval and indexing pipeline as implemented in code today. The inventory is based primarily on: + +- `app/modules/rag/services/rag_service.py` +- `app/modules/rag/persistence/*.py` +- `app/modules/rag/indexing/code/**/*.py` +- `app/modules/rag/indexing/docs/**/*.py` +- `app/modules/rag_session/module.py` +- `app/modules/agent/engine/graphs/project_qa_step_graphs.py` +- `app/modules/agent/engine/orchestrator/*.py` + +`ASSUMPTION:` the intended layer semantics are the ones implied by code and tests, not by future architecture plans. This matters because only `C0` through `C3` are materially implemented today; `C4+` exist only as enum constants. + +## Current retrieval pipeline + +1. Retrieval entrypoint is `POST /internal/rag/retrieve` in `app/modules/rag_session/module.py`. +2. The endpoint calls `RagService.retrieve(rag_session_id, query)`. +3. `RagQueryRouter` chooses `docs` or `code` mode from the raw query text. +4. `RagService` computes a single embedding for the full query via `GigaChatEmbedder`. +5. `RagQueryRepository.retrieve(...)` runs one SQL query against `rag_chunks` in PostgreSQL with `pgvector`. +6. Ranking order is: + - lexical rank + - test-file penalty + - layer rank + - vector distance `embedding <=> query_embedding` +7. Response items are normalized to `{source, content, layer, title, metadata, score}`. +8. If embeddings fail, retrieval falls back to latest chunks from the same layers. +9. If code retrieval returns nothing, service falls back to docs layers. + +## Storage and indices + +- Primary store: PostgreSQL from `DATABASE_URL`, configured in `app/modules/shared/db.py`. +- Vector extension: `CREATE EXTENSION IF NOT EXISTS vector` in `app/modules/rag/persistence/schema_repository.py`. +- Primary table: `rag_chunks`. +- Cache tables: + - `rag_blob_cache` + - `rag_chunk_cache` + - `rag_session_chunk_map` +- SQL indexes currently created: + - `(rag_session_id)` + - `(rag_session_id, layer)` + - `(rag_session_id, layer, path)` + - `(qname)` + - `(symbol_id)` + - `(module_id)` + - `(doc_kind)` + - `(entrypoint_type, framework)` + +`ASSUMPTION:` there is no explicit ANN index for the vector column in schema code. The code creates general SQL indexes, but no `ivfflat`/`hnsw` index is defined here. + +## Layer: C0_SOURCE_CHUNKS + +### Implementation + +- Produced by `CodeIndexingPipeline.index_file(...)` in `app/modules/rag/indexing/code/pipeline.py`. +- Chunking logic: `CodeTextChunker.chunk(...)` in `app/modules/rag/indexing/code/code_text/chunker.py`. +- Document builder: `CodeTextDocumentBuilder.build(...)` in `app/modules/rag/indexing/code/code_text/document_builder.py`. +- Persisted via `RagDocumentRepository.insert_documents(...)` into `rag_chunks`. + +### Input contract + +This is an indexing layer, not a direct public retriever. The observed upstream indexing input is a file dict with at least: + +- required: + - `path: str` + - `content: str` +- optional: + - `commit_sha: str | None` + - `content_hash: str` + - metadata fields copied through by `RagService._document_metadata(...)` + +For retrieval, the layer is queried only indirectly through: + +- `rag_session_id: str` +- `query: str` +- inferred mode/layers from `RagQueryRouter` +- fixed `limit=8` + +### Output contract + +Stored document shape: + +- top-level: + - `layer = "C0_SOURCE_CHUNKS"` + - `lang = "python"` + - `source.repo_id` + - `source.commit_sha` + - `source.path` + - `title` + - `text` + - `span.start_line` + - `span.end_line` + - `embedding` +- metadata: + - `chunk_index` + - `chunk_type`: `symbol_block` or `window` + - `module_or_unit` + - `artifact_type = "CODE"` + - plus file-level metadata injected by `RagService` + +Returned retrieval item shape: + +- `source` +- `content` +- `layer` +- `title` +- `metadata` +- `score` + +No `line_start` / `line_end` are returned to the caller directly; they remain in DB columns `span_start` / `span_end` and are only used in logs. + +### Defaults & limits + +- AST chunking prefers one chunk per top-level class/function/async function. +- Fallback window chunking: + - `size = 80` lines + - `overlap = 15` lines +- Global retrieval limit from `RagService.retrieve(...)`: `8` +- Embedding batch size from env: + - `RAG_EMBED_BATCH_SIZE` + - default `16` + +### Known issues + +- Nested methods/functions are not emitted as C0 chunks unless represented inside a selected top-level block. +- Returned API payload omits line spans even though storage has them. +- No direct filter by path, namespace, symbol, or `top_k` is exposed through the current endpoint. + +## Layer: C1_SYMBOL_CATALOG + +### Implementation + +- Symbol extraction: `SymbolExtractor.extract(...)` in `app/modules/rag/indexing/code/symbols/extractor.py`. +- AST parsing: `PythonAstParser.parse_module(...)`. +- Document builder: `SymbolDocumentBuilder.build(...)`. +- Retrieval reads rows from `rag_chunks`; there is no dedicated symbol table. + +### Input contract + +Indexing input is the same per-file payload as C0. + +Observed symbol extraction source: + +- Python AST only +- supported symbol kinds: + - `class` + - `function` + - `method` + - `const` for top-level imports/import aliases + +Retrieval input is still the generic text query endpoint. Query terms are enriched by `extract_query_terms(...)`: + +- extracts identifier-like tokens from query text +- normalizes camelCase/PascalCase to snake_case +- adds special intent terms for management/control-related queries +- max observed query terms: `6` + +### Output contract + +Stored document shape: + +- top-level: + - `layer = "C1_SYMBOL_CATALOG"` + - `title = qname` + - `text = " \n\n"` + - `span.start_line` + - `span.end_line` +- metadata: + - `symbol_id` + - `qname` + - `kind` + - `signature` + - `decorators_or_annotations` + - `docstring_or_javadoc` + - `parent_symbol_id` + - `package_or_module` + - `is_entry_candidate` + - `lang_payload` + - `artifact_type = "CODE"` + +Observed `lang_payload` variants: + +- class: + - `bases` +- function/method: + - `async` +- import alias: + - `imported_from` + - `import_alias` + +### Defaults & limits + +- Only Python source files are indexed into C-layers. +- Import and import-from declarations are materialized as `const` symbols only at module top level. +- Retrieval ranking gives C1 priority rank `1`, after C3 and before C2/C0. + +### Known issues + +- No explicit visibility/public-private model. +- `parent_symbol_id` currently stores the parent qname string from the stack, not the parent symbol hash. This is an observed implementation detail. +- Cross-file symbol resolution is not implemented; `dst_symbol_id` in edges resolves only against symbols extracted from the same file. + +## Layer: C2_DEPENDENCY_GRAPH + +### Implementation + +- Edge extraction: `EdgeExtractor.extract(...)` in `app/modules/rag/indexing/code/edges/extractor.py`. +- Document builder: `EdgeDocumentBuilder.build(...)`. +- Built during `CodeIndexingPipeline.index_file(...)`. + +### Input contract + +Indexing input is the same per-file source payload as C0/C1. + +Graph construction method: + +- static analysis only +- Python AST walk only +- no runtime tracing +- no tree-sitter + +Observed edge types: + +- `calls` +- `imports` +- `inherits` + +### Output contract + +Stored document shape: + +- top-level: + - `layer = "C2_DEPENDENCY_GRAPH"` + - `title = ":"` + - `text = " "` + - `span.start_line` + - `span.end_line` + - `links` contains one evidence link of type `EDGE` +- metadata: + - `edge_id` + - `edge_type` + - `src_symbol_id` + - `src_qname` + - `dst_symbol_id` + - `dst_ref` + - `resolution`: `resolved` or `partial` + - `lang_payload` + - `artifact_type = "CODE"` + +Observed `lang_payload` usage: + +- for calls: may include `callsite_kind = "function_call"` + +### Defaults & limits + +- Edge extraction is per-file only. +- `imports` edges are emitted only while visiting a class/function scope; top-level imports do not become C2 edges. +- Layer rank in retrieval SQL: `2` + +### Known issues + +- There is no traversal API, graph repository, or query language over C2. Retrieval only treats edges as text/vector rows in `rag_chunks`. +- Destination resolution is local to the file-level qname map. +- Top-level module import relationships are incompletely represented because `visit_Import` / `visit_ImportFrom` skip when there is no current scope. + +## Layer: C3_ENTRYPOINTS + +### Implementation + +- Detection registry: `EntrypointDetectorRegistry.detect_all(...)`. +- Detectors: + - `FastApiEntrypointDetector` + - `FlaskEntrypointDetector` + - `TyperClickEntrypointDetector` +- Document builder: `EntrypointDocumentBuilder.build(...)`. + +### Input contract + +Indexing input is the same per-file source payload as other C-layers. + +Detected entrypoint families today: + +- HTTP: + - FastAPI decorators such as `.get`, `.post`, `.put`, `.patch`, `.delete`, `.route` + - Flask `.route` +- CLI: + - Typer/Click `.command` + - Typer/Click `.callback` + +Not detected: + +- Django routes +- Celery tasks +- RQ jobs +- cron jobs / scheduler entries + +### Output contract + +Stored document shape: + +- top-level: + - `layer = "C3_ENTRYPOINTS"` + - `title = route_or_command` + - `text = " "` + - `span.start_line` + - `span.end_line` + - `links` contains one evidence link of type `CODE_SPAN` +- metadata: + - `entry_id` + - `entry_type`: observed `http` or `cli` + - `framework`: observed `fastapi`, `flask`, `typer`, `click` + - `route_or_command` + - `handler_symbol_id` + - `lang_payload` + - `artifact_type = "CODE"` + +FastAPI-specific observed payload: + +- `lang_payload.methods = [HTTP_METHOD]` for `.get/.post/...` + +### Defaults & limits + +- Retrieval layer rank: `0` highest among code layers. +- Entrypoint mapping is handler-symbol centric: + - decorator match -> symbol -> `handler_symbol_id` + - physical location comes from symbol span + +### Known issues + +- Route parsing is string-based from decorator text, not semantic AST argument parsing. +- No dedicated entrypoint tags beyond `entry_type`, `framework`, and raw decorator-derived payload. +- Background jobs and non-decorator entrypoints are not indexed. + +## Dependency graph / trace current state + +### Exists or stub? + +- C2 exists and is populated. +- It is not a stub. +- It is also not a full-project dependency graph service; it is a set of per-edge documents stored in `rag_chunks`. + +### How the graph is built + +- static Python AST analysis +- no runtime instrumentation +- no import graph resolver across modules +- no tree-sitter + +### Edge types in data + +- `calls` +- `imports` +- `inherits` + +### Traversal API + +- No traversal API was found in `app/modules/rag/*` or `app/modules/agent/*`. +- No method accepts graph traversal parameters such as depth, start node, edge filters, or BFS/DFS strategy. +- Current access path is only retrieval over indexed edge documents. + +## Entrypoints current state + +### Implemented extraction + +- HTTP routes: + - FastAPI + - Flask +- CLI: + - Typer + - Click + +### Mapping model + +- `entrypoint -> handler_symbol_id -> symbol span/path` +- The entrypoint record itself stores: + - framework + - entry type + - raw route/command string + - handler symbol id + +### Tags/types + +- `entry_type` is the main normalized tag. +- Observed values: `http`, `cli`. +- `framework` is the second discriminator. +- There are no richer endpoint taxonomies such as `job`, `worker`, `webhook`, `scheduler`. + +## Defaults and operational limits + +- Query mode default: `docs` +- Code mode is enabled by keyword heuristics in `RagQueryRouter` +- Retrieval hard limit: `8` +- Fallback limit: `8` +- Query term extraction limit: `6` +- Ranked source bundle for project QA: + - top `12` RAG items + - top `10` file candidates +- No exposed `namespace`, `path_prefixes`, `top_k`, `max_chars`, `max_chunks`, `max_depth` in the public/internal retrieval endpoint + +`ASSUMPTION:` the absence of these controls in endpoint and service signatures means they are not part of the current supported contract, even though `RagQueryRepository.retrieve(...)` has an internal `path_prefixes` parameter. + +## Known cross-cutting issues + +- Retrieval contract is effectively text-only at API level; structured retrieval exists only as internal SQL parameters. +- Response payload drops explicit line spans even though spans are stored. +- Vector retrieval is coupled to a single provider-specific embedder. +- Docs mode is the default, so code retrieval depends on heuristic query phrasing unless the project/qa graph prepends `по коду`. +- There is no separate retrieval contract per layer exposed over API; all layer selection is implicit. + +## Where to plug ExplainPack pipeline + +### Option 1: replace or extend `project_qa/context_analysis` + +- Code location: + - `app/modules/agent/engine/graphs/project_qa_step_graphs.py` +- Why: + - retrieval is already complete at this step + - input bundle already contains ranked `rag_items` and `file_candidates` + - output is already a structured `analysis_brief` +- Risk: + - low + - minimal invasion if ExplainPack consumes `source_bundle` and emits the same `analysis_brief` shape + +### Option 2: insert a new orchestrator step between `context_retrieval` and `context_analysis` + +- Code location: + - `app/modules/agent/engine/orchestrator/template_registry.py` + - `app/modules/agent/engine/orchestrator/step_registry.py` +- Why: + - preserves current retrieval behavior + - makes ExplainPack an explicit pipeline stage with its own artifact + - cleanest for observability and future A/B migration +- Risk: + - low to medium + - requires one new artifact contract and one extra orchestration step, but no change to retrieval storage + +### Option 3: introduce ExplainPack inside `ExplainActions.extract_logic` + +- Code location: + - `app/modules/agent/engine/orchestrator/actions/explain_actions.py` +- Why: + - useful if ExplainPack is meant only for explain-style scenarios + - keeps general project QA untouched +- Risk: + - medium + - narrower integration point; may create duplicate reasoning logic separate from project QA analysis path + +## Bottom line + +- C0-C3 are implemented and persisted in one physical store: `rag_chunks`. +- Retrieval is a hybrid SQL ranking over lexical heuristics plus pgvector distance. +- C2 exists, but only as retrievable edge documents, not as a traversable graph subsystem. +- C3 covers FastAPI/Flask/Typer/Click only. +- The least invasive ExplainPack integration point is after retrieval and before answer composition, preferably as a new explicit orchestrator artifact or as a replacement for `context_analysis`. diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..904d42b --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +markers = + intent_router: intent router v2 suite (routing, normalization, anchors, retrieval spec invariants) diff --git a/tests/agent/__pycache__/test_gigachat_client_retry.cpython-312-pytest-9.0.2.pyc b/tests/agent/__pycache__/test_gigachat_client_retry.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000..91f3df6 Binary files /dev/null and b/tests/agent/__pycache__/test_gigachat_client_retry.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/agent/__pycache__/test_llm_service_logging.cpython-312-pytest-9.0.2.pyc b/tests/agent/__pycache__/test_llm_service_logging.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000..8f1aaf5 Binary files /dev/null and b/tests/agent/__pycache__/test_llm_service_logging.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/agent/__pycache__/test_logging_setup.cpython-312-pytest-9.0.2.pyc b/tests/agent/__pycache__/test_logging_setup.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000..22cb4ad Binary files /dev/null and b/tests/agent/__pycache__/test_logging_setup.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/agent/engine/router/__pycache__/test_router_service_intent_policy.cpython-312-pytest-9.0.2.pyc b/tests/agent/engine/router/__pycache__/test_router_service_intent_policy.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000..c85e675 Binary files /dev/null and b/tests/agent/engine/router/__pycache__/test_router_service_intent_policy.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/agent/engine/router/test_router_service_intent_policy.py b/tests/agent/engine/router/test_router_service_intent_policy.py new file mode 100644 index 0000000..7a7ec95 --- /dev/null +++ b/tests/agent/engine/router/test_router_service_intent_policy.py @@ -0,0 +1,181 @@ +import sys +import types + +sqlalchemy = types.ModuleType("sqlalchemy") +sqlalchemy.text = lambda value: value +sqlalchemy.create_engine = lambda *args, **kwargs: object() +sys.modules.setdefault("sqlalchemy", sqlalchemy) + +sqlalchemy_engine = types.ModuleType("sqlalchemy.engine") +sqlalchemy_engine.Engine = object +sys.modules.setdefault("sqlalchemy.engine", sqlalchemy_engine) + +sqlalchemy_orm = types.ModuleType("sqlalchemy.orm") +sqlalchemy_orm.sessionmaker = lambda *args, **kwargs: object() +sys.modules.setdefault("sqlalchemy.orm", sqlalchemy_orm) + +sqlalchemy_pool = types.ModuleType("sqlalchemy.pool") +sqlalchemy_pool.NullPool = object +sys.modules.setdefault("sqlalchemy.pool", sqlalchemy_pool) + +from app.modules.agent.engine.router.router_service import RouterService +from app.modules.agent.engine.router.schemas import RouteDecision, RouterContext + + +class _FakeRegistry: + def is_valid(self, domain_id: str, process_id: str) -> bool: + return (domain_id, process_id) in { + ("default", "general"), + ("project", "qa"), + ("project", "edits"), + ("docs", "generation"), + } + + def get_factory(self, domain_id: str, process_id: str): + return object() + + +class _FakeClassifier: + def __init__(self, decision: RouteDecision | None = None, forced: RouteDecision | None = None) -> None: + self._decision = decision or RouteDecision(domain_id="project", process_id="qa", confidence=0.95, reason="new_intent") + self._forced = forced + self.calls = 0 + + def from_mode(self, mode: str) -> RouteDecision | None: + return self._forced if mode != "auto" else None + + def classify_new_intent(self, user_message: str, context: RouterContext) -> RouteDecision: + self.calls += 1 + return self._decision + + +class _FakeContextStore: + def __init__(self, context: RouterContext) -> None: + self._context = context + self.updated: list[dict] = [] + + def get(self, conversation_key: str) -> RouterContext: + return self._context + + def update(self, conversation_key: str, **kwargs) -> None: + self.updated.append({"conversation_key": conversation_key, **kwargs}) + + +class _FakeSwitchDetector: + def __init__(self, should_switch: bool) -> None: + self._should_switch = should_switch + + def should_switch(self, user_message: str, context: RouterContext) -> bool: + return self._should_switch + + +def test_router_service_classifies_first_message() -> None: + service = RouterService( + registry=_FakeRegistry(), + classifier=_FakeClassifier(), + context_store=_FakeContextStore(RouterContext()), + switch_detector=_FakeSwitchDetector(False), + ) + + route = service.resolve("Объясни как работает endpoint", "dialog-1") + + assert route.domain_id == "project" + assert route.process_id == "qa" + assert route.decision_type == "start" + + +def test_router_service_keeps_current_intent_for_follow_up() -> None: + context = RouterContext( + active_intent={"domain_id": "project", "process_id": "qa"}, + last_routing={"domain_id": "project", "process_id": "qa"}, + dialog_started=True, + turn_index=1, + ) + classifier = _FakeClassifier( + decision=RouteDecision(domain_id="docs", process_id="generation", confidence=0.99, reason="should_not_run") + ) + service = RouterService( + registry=_FakeRegistry(), + classifier=classifier, + context_store=_FakeContextStore(context), + switch_detector=_FakeSwitchDetector(False), + ) + + route = service.resolve("Покажи подробнее", "dialog-1") + + assert route.domain_id == "project" + assert route.process_id == "qa" + assert route.decision_type == "continue" + assert classifier.calls == 0 + + +def test_router_service_switches_only_on_explicit_new_intent() -> None: + context = RouterContext( + active_intent={"domain_id": "project", "process_id": "qa"}, + last_routing={"domain_id": "project", "process_id": "qa"}, + dialog_started=True, + turn_index=2, + ) + classifier = _FakeClassifier( + decision=RouteDecision(domain_id="project", process_id="edits", confidence=0.96, reason="explicit_edit") + ) + service = RouterService( + registry=_FakeRegistry(), + classifier=classifier, + context_store=_FakeContextStore(context), + switch_detector=_FakeSwitchDetector(True), + ) + + route = service.resolve("Теперь измени файл README.md", "dialog-1") + + assert route.domain_id == "project" + assert route.process_id == "edits" + assert route.decision_type == "switch" + assert route.explicit_switch is True + assert classifier.calls == 1 + + +def test_router_service_keeps_current_when_explicit_switch_is_unresolved() -> None: + context = RouterContext( + active_intent={"domain_id": "project", "process_id": "qa"}, + last_routing={"domain_id": "project", "process_id": "qa"}, + dialog_started=True, + turn_index=2, + ) + classifier = _FakeClassifier( + decision=RouteDecision(domain_id="docs", process_id="generation", confidence=0.2, reason="low_confidence") + ) + service = RouterService( + registry=_FakeRegistry(), + classifier=classifier, + context_store=_FakeContextStore(context), + switch_detector=_FakeSwitchDetector(True), + ) + + route = service.resolve("Теперь сделай что-то другое", "dialog-1") + + assert route.domain_id == "project" + assert route.process_id == "qa" + assert route.decision_type == "continue" + assert route.reason == "explicit_switch_unresolved_keep_current" + + +def test_router_service_persists_decision_type() -> None: + store = _FakeContextStore(RouterContext()) + service = RouterService( + registry=_FakeRegistry(), + classifier=_FakeClassifier(), + context_store=store, + switch_detector=_FakeSwitchDetector(False), + ) + + service.persist_context( + "dialog-1", + domain_id="project", + process_id="qa", + user_message="Объясни", + assistant_message="Ответ", + decision_type="continue", + ) + + assert store.updated[0]["decision_type"] == "continue" diff --git a/tests/agent/orchestrator/__pycache__/test_code_explain_actions.cpython-312-pytest-9.0.2.pyc b/tests/agent/orchestrator/__pycache__/test_code_explain_actions.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000..e83a0da Binary files /dev/null and b/tests/agent/orchestrator/__pycache__/test_code_explain_actions.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/agent/orchestrator/__pycache__/test_orchestrator_service.cpython-312-pytest-9.0.2.pyc b/tests/agent/orchestrator/__pycache__/test_orchestrator_service.cpython-312-pytest-9.0.2.pyc index b9b202a..1b41965 100644 Binary files a/tests/agent/orchestrator/__pycache__/test_orchestrator_service.cpython-312-pytest-9.0.2.pyc and b/tests/agent/orchestrator/__pycache__/test_orchestrator_service.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/agent/orchestrator/__pycache__/test_project_qa_actions.cpython-312-pytest-9.0.2.pyc b/tests/agent/orchestrator/__pycache__/test_project_qa_actions.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000..562c374 Binary files /dev/null and b/tests/agent/orchestrator/__pycache__/test_project_qa_actions.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/agent/orchestrator/__pycache__/test_project_qa_answer_graph_v2.cpython-312-pytest-9.0.2.pyc b/tests/agent/orchestrator/__pycache__/test_project_qa_answer_graph_v2.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000..e0fcc48 Binary files /dev/null and b/tests/agent/orchestrator/__pycache__/test_project_qa_answer_graph_v2.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/agent/orchestrator/__pycache__/test_project_qa_retrieval_graph_v2_only.cpython-312-pytest-9.0.2.pyc b/tests/agent/orchestrator/__pycache__/test_project_qa_retrieval_graph_v2_only.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000..608cd1d Binary files /dev/null and b/tests/agent/orchestrator/__pycache__/test_project_qa_retrieval_graph_v2_only.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/agent/orchestrator/__pycache__/test_template_registry.cpython-312-pytest-9.0.2.pyc b/tests/agent/orchestrator/__pycache__/test_template_registry.cpython-312-pytest-9.0.2.pyc index fb434ec..a4df06e 100644 Binary files a/tests/agent/orchestrator/__pycache__/test_template_registry.cpython-312-pytest-9.0.2.pyc and b/tests/agent/orchestrator/__pycache__/test_template_registry.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/agent/orchestrator/test_code_explain_actions.py b/tests/agent/orchestrator/test_code_explain_actions.py new file mode 100644 index 0000000..052df7b --- /dev/null +++ b/tests/agent/orchestrator/test_code_explain_actions.py @@ -0,0 +1,59 @@ +from app.modules.agent.engine.orchestrator.actions.code_explain_actions import CodeExplainActions +from app.modules.agent.engine.orchestrator.execution_context import ExecutionContext +from app.modules.agent.engine.orchestrator.models import ( + ArtifactType, + ExecutionPlan, + OutputContract, + RoutingMeta, + Scenario, + TaskConstraints, + TaskSpec, +) +from app.modules.rag.explain.models import ExplainIntent, ExplainPack + + +class _FakeRetriever: + def build_pack(self, rag_session_id: str, user_query: str, *, file_candidates: list[dict] | None = None) -> ExplainPack: + assert rag_session_id == "rag-1" + assert "endpoint" in user_query + assert file_candidates == [{"path": "app/api/users.py", "content": "..." }] + return ExplainPack(intent=ExplainIntent(raw_query=user_query, normalized_query=user_query)) + + +def _ctx() -> ExecutionContext: + task = TaskSpec( + task_id="task-1", + dialog_session_id="dialog-1", + rag_session_id="rag-1", + user_message="Explain endpoint get_user", + scenario=Scenario.EXPLAIN_PART, + routing=RoutingMeta(domain_id="project", process_id="qa", confidence=0.9, reason="test"), + constraints=TaskConstraints(), + output_contract=OutputContract(result_type="answer"), + metadata={"rag_context": "", "confluence_context": "", "files_map": {}}, + ) + plan = ExecutionPlan( + plan_id="plan-1", + task_id="task-1", + scenario=Scenario.EXPLAIN_PART, + template_id="tpl", + template_version="1", + steps=[], + ) + ctx = ExecutionContext(task=task, plan=plan, graph_resolver=lambda *_: None, graph_invoker=lambda *_: {}) + ctx.artifacts.put( + key="source_bundle", + artifact_type=ArtifactType.STRUCTURED_JSON, + content={"file_candidates": [{"path": "app/api/users.py", "content": "..."}]}, + ) + return ctx + + +def test_code_explain_actions_store_explain_pack() -> None: + ctx = _ctx() + actions = CodeExplainActions(_FakeRetriever()) + + actions.build_code_explain_pack(ctx) + + stored = ctx.artifacts.get_content("explain_pack", {}) + assert stored["intent"]["raw_query"] == "Explain endpoint get_user" diff --git a/tests/agent/orchestrator/test_orchestrator_service.py b/tests/agent/orchestrator/test_orchestrator_service.py index d1cdce2..5b33578 100644 --- a/tests/agent/orchestrator/test_orchestrator_service.py +++ b/tests/agent/orchestrator/test_orchestrator_service.py @@ -14,7 +14,7 @@ class DummyGraph: pass -def _task(scenario: Scenario) -> TaskSpec: +def _task(scenario: Scenario, *, domain_id: str = "project", process_id: str = "qa") -> TaskSpec: allow_writes = scenario in {Scenario.DOCS_FROM_ANALYTICS, Scenario.TARGETED_EDIT, Scenario.GHERKIN_MODEL} return TaskSpec( task_id="task-1", @@ -23,7 +23,7 @@ def _task(scenario: Scenario) -> TaskSpec: mode="auto", user_message="Explain this module", scenario=scenario, - routing=RoutingMeta(domain_id="project", process_id="qa", confidence=0.95, reason="unit-test"), + routing=RoutingMeta(domain_id=domain_id, process_id=process_id, confidence=0.95, reason="unit-test"), constraints=TaskConstraints(allow_writes=allow_writes, max_steps=16, max_retries_per_step=2, step_timeout_sec=90), output_contract=OutputContract(result_type="answer"), metadata={ @@ -38,8 +38,8 @@ def test_orchestrator_service_returns_answer() -> None: service = OrchestratorService() def graph_resolver(domain_id: str, process_id: str): - assert domain_id == "project" - assert process_id == "qa" + assert domain_id == "default" + assert process_id == "general" return DummyGraph() def graph_invoker(_graph, state: dict, dialog_session_id: str): @@ -47,7 +47,13 @@ def test_orchestrator_service_returns_answer() -> None: assert dialog_session_id == "dialog-1" return {"answer": "It works.", "changeset": []} - result = asyncio.run(service.run(task=_task(Scenario.GENERAL_QA), graph_resolver=graph_resolver, graph_invoker=graph_invoker)) + result = asyncio.run( + service.run( + task=_task(Scenario.GENERAL_QA, domain_id="default", process_id="general"), + graph_resolver=graph_resolver, + graph_invoker=graph_invoker, + ) + ) assert result.answer == "It works." assert result.meta["plan"]["status"] == "completed" @@ -70,3 +76,100 @@ def test_orchestrator_service_generates_changeset_for_docs_scenario() -> None: ) assert result.meta["plan"]["status"] == "completed" assert len(result.changeset) > 0 + + +def test_orchestrator_service_uses_project_qa_reasoning_without_graph() -> None: + service = OrchestratorService() + requested_graphs: list[tuple[str, str]] = [] + + def graph_resolver(domain_id: str, process_id: str): + requested_graphs.append((domain_id, process_id)) + return DummyGraph() + + def graph_invoker(_graph, state: dict, _dialog_session_id: str): + if "resolved_request" not in state: + return { + "resolved_request": { + "original_message": state["message"], + "normalized_message": state["message"], + "subject_hint": "", + "source_hint": "code", + "russian": True, + } + } + if "question_profile" not in state: + return { + "question_profile": { + "domain": "code", + "intent": "inventory", + "terms": ["control", "channel"], + "entities": [], + "russian": True, + } + } + if "source_bundle" not in state: + return { + "source_bundle": { + "profile": state["question_profile"], + "rag_items": [], + "file_candidates": [ + {"path": "src/config_manager/v2/control/base.py", "content": "class ControlChannel: pass"}, + {"path": "src/config_manager/v2/control/http_channel.py", "content": "class HttpControlChannel(ControlChannel): pass # http api"}, + ], + "rag_total": 0, + "files_total": 2, + } + } + if "analysis_brief" not in state: + return { + "analysis_brief": { + "subject": "management channels", + "findings": ["В коде найдены конкретные реализации каналов управления: http channel (`src/config_manager/v2/control/http_channel.py`)."], + "evidence": ["src/config_manager/v2/control/http_channel.py"], + "gaps": [], + "answer_mode": "inventory", + } + } + return { + "answer_brief": { + "question_profile": state["question_profile"], + "resolved_subject": "management channels", + "key_findings": ["В коде найдены конкретные реализации каналов управления: http channel (`src/config_manager/v2/control/http_channel.py`)."], + "supporting_evidence": ["src/config_manager/v2/control/http_channel.py"], + "missing_evidence": [], + "answer_mode": "inventory", + }, + "final_answer": "## Кратко\n### Что реализовано\n- В коде найдены конкретные реализации каналов управления: http channel (`src/config_manager/v2/control/http_channel.py`).", + } + + task = _task(Scenario.GENERAL_QA).model_copy( + update={ + "user_message": "Какие каналы управления уже реализованы?", + "metadata": { + "rag_context": "", + "confluence_context": "", + "files_map": { + "src/config_manager/v2/control/base.py": { + "content": "class ControlChannel:\n async def start(self):\n ..." + }, + "src/config_manager/v2/control/http_channel.py": { + "content": "class HttpControlChannel(ControlChannel):\n async def start(self):\n ...\n# http api" + }, + }, + "rag_items": [], + }, + } + ) + + result = asyncio.run(service.run(task=task, graph_resolver=graph_resolver, graph_invoker=graph_invoker)) + + assert "Что реализовано" in result.answer + assert "http channel" in result.answer.lower() + assert result.meta["plan"]["status"] == "completed" + assert requested_graphs == [ + ("project_qa", "conversation_understanding"), + ("project_qa", "question_classification"), + ("project_qa", "context_retrieval"), + ("project_qa", "context_analysis"), + ("project_qa", "answer_composition"), + ] diff --git a/tests/agent/orchestrator/test_project_qa_actions.py b/tests/agent/orchestrator/test_project_qa_actions.py new file mode 100644 index 0000000..ca35e1f --- /dev/null +++ b/tests/agent/orchestrator/test_project_qa_actions.py @@ -0,0 +1,71 @@ +from app.modules.agent.engine.orchestrator.actions.project_qa_actions import ProjectQaActions +from app.modules.agent.engine.orchestrator.execution_context import ExecutionContext +from app.modules.agent.engine.orchestrator.models import ( + ExecutionPlan, + OutputContract, + RoutingMeta, + Scenario, + TaskConstraints, + TaskSpec, +) + + +def _ctx(message: str, rag_items: list[dict], files_map: dict[str, dict]) -> ExecutionContext: + task = TaskSpec( + task_id="task-1", + dialog_session_id="dialog-1", + rag_session_id="rag-1", + user_message=message, + scenario=Scenario.GENERAL_QA, + routing=RoutingMeta(domain_id="project", process_id="qa", confidence=0.9, reason="test"), + constraints=TaskConstraints(), + output_contract=OutputContract(result_type="answer"), + metadata={ + "rag_items": rag_items, + "rag_context": "", + "confluence_context": "", + "files_map": files_map, + }, + ) + plan = ExecutionPlan( + plan_id="plan-1", + task_id="task-1", + scenario=Scenario.GENERAL_QA, + template_id="tpl", + template_version="1", + steps=[], + ) + return ExecutionContext(task=task, plan=plan, graph_resolver=lambda *_: None, graph_invoker=lambda *_: {}) + + +def test_project_qa_actions_build_inventory_answer_from_code_sources() -> None: + ctx = _ctx( + "Какие каналы управления уже реализованы?", + [], + { + "src/config_manager/v2/control/base.py": {"content": "class ControlChannel:\n async def start(self):\n ..."}, + "src/config_manager/v2/core/control_bridge.py": { + "content": "class ControlChannelBridge:\n async def on_start(self):\n ...\n async def on_status(self):\n ..." + }, + "src/config_manager/v2/control/http_channel.py": { + "content": "class HttpControlChannel(ControlChannel):\n async def start(self):\n ...\n# http api" + }, + "src/config_manager/v2/control/telegram_channel.py": { + "content": "class TelegramControlChannel(ControlChannel):\n async def start(self):\n ...\n# telegram bot" + }, + }, + ) + actions = ProjectQaActions() + + actions.classify_project_question(ctx) + actions.collect_project_sources(ctx) + actions.analyze_project_sources(ctx) + actions.build_project_answer_brief(ctx) + actions.compose_project_answer(ctx) + + answer = str(ctx.artifacts.get_content("final_answer", "")) + assert "### Что реализовано" in answer + assert "http channel" in answer.lower() + assert "telegram channel" in answer.lower() + assert "### Где смотреть в проекте" in answer + diff --git a/tests/agent/orchestrator/test_project_qa_answer_graph_v2.py b/tests/agent/orchestrator/test_project_qa_answer_graph_v2.py new file mode 100644 index 0000000..769557d --- /dev/null +++ b/tests/agent/orchestrator/test_project_qa_answer_graph_v2.py @@ -0,0 +1,74 @@ +import sys +import types + +langgraph = types.ModuleType("langgraph") +langgraph_graph = types.ModuleType("langgraph.graph") +langgraph_graph.END = "END" +langgraph_graph.START = "START" +langgraph_graph.StateGraph = object +sys.modules.setdefault("langgraph", langgraph) +sys.modules.setdefault("langgraph.graph", langgraph_graph) + +from app.modules.agent.engine.graphs.project_qa_step_graphs import ProjectQaAnswerGraphFactory + + +class _FakeLlm: + def __init__(self) -> None: + self.calls: list[tuple[str, str, str | None]] = [] + + def generate(self, prompt_name: str, user_input: str, *, log_context: str | None = None) -> str: + self.calls.append((prompt_name, user_input, log_context)) + return "## Summary\n[entrypoint_1] [excerpt_1]" + + +def test_project_qa_answer_graph_uses_v2_prompt_when_explain_pack_present() -> None: + llm = _FakeLlm() + factory = ProjectQaAnswerGraphFactory(llm) + + result = factory._compose_answer( + { + "message": "Explain endpoint get_user", + "question_profile": {"russian": False}, + "analysis_brief": {"findings": [], "evidence": [], "gaps": [], "answer_mode": "summary"}, + "explain_pack": { + "intent": { + "raw_query": "Explain endpoint get_user", + "normalized_query": "Explain endpoint get_user", + "keywords": ["get_user"], + "hints": {"paths": [], "symbols": [], "endpoints": [], "commands": []}, + "expected_entry_types": ["http"], + "depth": "medium", + }, + "selected_entrypoints": [], + "seed_symbols": [], + "trace_paths": [], + "evidence_index": { + "entrypoint_1": { + "evidence_id": "entrypoint_1", + "kind": "entrypoint", + "summary": "/users/{id}", + "location": {"path": "app/api/users.py", "start_line": 10, "end_line": 10}, + "supports": ["handler-1"], + } + }, + "code_excerpts": [ + { + "evidence_id": "excerpt_1", + "symbol_id": "handler-1", + "title": "get_user", + "path": "app/api/users.py", + "start_line": 10, + "end_line": 18, + "content": "async def get_user():\n return 1", + "focus": "overview", + } + ], + "missing": [], + "conflicts": [], + }, + } + ) + + assert result["final_answer"].startswith("## Summary") + assert llm.calls[0][0] == "code_explain_answer_v2" + assert '"evidence_id": "excerpt_1"' in llm.calls[0][1] diff --git a/tests/agent/orchestrator/test_project_qa_retrieval_graph_v2_only.py b/tests/agent/orchestrator/test_project_qa_retrieval_graph_v2_only.py new file mode 100644 index 0000000..739f8d9 --- /dev/null +++ b/tests/agent/orchestrator/test_project_qa_retrieval_graph_v2_only.py @@ -0,0 +1,49 @@ +import sys +import types + +langgraph = types.ModuleType("langgraph") +langgraph_graph = types.ModuleType("langgraph.graph") +langgraph_graph.END = "END" +langgraph_graph.START = "START" +langgraph_graph.StateGraph = object +sys.modules.setdefault("langgraph", langgraph) +sys.modules.setdefault("langgraph.graph", langgraph_graph) + +from app.modules.agent.engine.graphs.project_qa_step_graphs import ProjectQaRetrievalGraphFactory + + +class _FailingRag: + async def retrieve(self, rag_session_id: str, query: str): + raise AssertionError("legacy rag should not be called for explain_part") + + +def test_project_qa_retrieval_skips_legacy_rag_for_explain_part() -> None: + factory = ProjectQaRetrievalGraphFactory(_FailingRag()) + + result = factory._retrieve_context( + { + "scenario": "explain_part", + "project_id": "rag-1", + "resolved_request": { + "original_message": "Explain how ConfigManager works", + "normalized_message": "Explain how ConfigManager works", + }, + "question_profile": { + "domain": "code", + "intent": "explain", + "terms": ["configmanager"], + "entities": ["ConfigManager"], + "russian": False, + }, + "files_map": { + "src/config_manager/__init__.py": { + "content": "from .v2 import ConfigManagerV2 as ConfigManager", + "content_hash": "hash-1", + } + }, + } + ) + + bundle = result["source_bundle"] + assert bundle["rag_items"] == [] + assert bundle["files_total"] >= 1 diff --git a/tests/agent/orchestrator/test_template_registry.py b/tests/agent/orchestrator/test_template_registry.py index 30878fd..8572107 100644 --- a/tests/agent/orchestrator/test_template_registry.py +++ b/tests/agent/orchestrator/test_template_registry.py @@ -36,3 +36,13 @@ def test_template_registry_has_multi_step_review_docs_edit_gherkin() -> None: assert len(docs_steps) >= 9 assert len(edit_steps) >= 7 assert len(gherkin_steps) >= 8 + + +def test_template_registry_adds_code_explain_pack_step_for_project_explain() -> None: + registry = ScenarioTemplateRegistry() + + steps = [step.step_id for step in registry.build(_task(Scenario.EXPLAIN_PART)).steps] + + assert "code_explain_pack_step" in steps + assert steps.index("code_explain_pack_step") > steps.index("context_retrieval") + assert steps.index("code_explain_pack_step") < steps.index("context_analysis") diff --git a/tests/agent/test_gigachat_client_retry.py b/tests/agent/test_gigachat_client_retry.py new file mode 100644 index 0000000..288dc46 --- /dev/null +++ b/tests/agent/test_gigachat_client_retry.py @@ -0,0 +1,48 @@ +import requests + +from app.modules.shared.gigachat.client import GigaChatClient +from app.modules.shared.gigachat.settings import GigaChatSettings + + +class _FakeTokenProvider: + def get_access_token(self) -> str: + return "token" + + +class _FakeResponse: + def __init__(self, status_code: int, payload: dict, text: str = "") -> None: + self.status_code = status_code + self._payload = payload + self.text = text + + def json(self) -> dict: + return self._payload + + +def test_gigachat_client_retries_transient_http_errors(monkeypatch) -> None: + calls = {"count": 0} + + def fake_post(*args, **kwargs): + calls["count"] += 1 + if calls["count"] == 1: + return _FakeResponse(503, {}, "temporary") + return _FakeResponse(200, {"choices": [{"message": {"content": "ok"}}]}) + + monkeypatch.setattr(requests, "post", fake_post) + client = GigaChatClient( + GigaChatSettings( + auth_url="https://auth.example.test", + api_url="https://api.example.test", + scope="scope", + credentials="secret", + ssl_verify=True, + model="model", + embedding_model="embed", + ), + _FakeTokenProvider(), + ) + + result = client.complete("system", "user") + + assert result == "ok" + assert calls["count"] == 2 diff --git a/tests/agent/test_llm_service_logging.py b/tests/agent/test_llm_service_logging.py new file mode 100644 index 0000000..6cb3fa3 --- /dev/null +++ b/tests/agent/test_llm_service_logging.py @@ -0,0 +1,30 @@ +import logging + +from app.modules.agent.llm.service import AgentLlmService + + +class _FakeClient: + def complete(self, *, system_prompt: str, user_prompt: str) -> str: + assert system_prompt == "System prompt" + assert user_prompt == "User input" + return "LLM output" + + +class _FakePrompts: + def load(self, prompt_name: str) -> str: + assert prompt_name == "general_answer" + return "System prompt" + + +def test_llm_service_logs_input_and_output_for_graph_context(caplog) -> None: + service = AgentLlmService(_FakeClient(), _FakePrompts()) + + with caplog.at_level(logging.WARNING, logger="app.modules.agent.llm.service"): + result = service.generate("general_answer", "User input", log_context="graph.default.answer") + + assert result == "LLM output" + messages = [record.getMessage() for record in caplog.records] + assert any("graph llm input: context=graph.default.answer" in message for message in messages) + assert any("graph llm output: context=graph.default.answer" in message for message in messages) + assert any("User input" in message for message in messages) + assert any("LLM output" in message for message in messages) diff --git a/tests/agent/test_logging_setup.py b/tests/agent/test_logging_setup.py new file mode 100644 index 0000000..0521b56 --- /dev/null +++ b/tests/agent/test_logging_setup.py @@ -0,0 +1,24 @@ +import logging + +from app.core.logging_setup import ScrubbingFormatter + + +def test_scrubbing_formatter_redacts_identifiers_and_adds_blank_line() -> None: + formatter = ScrubbingFormatter("%(levelname)s:%(name)s:%(message)s") + record = logging.LogRecord( + name="test.logger", + level=logging.WARNING, + pathname=__file__, + lineno=1, + msg="router decision: task_id=task-1 dialog_session_id=dialog-1 graph_id=project_qa/context_retrieval", + args=(), + exc_info=None, + ) + + rendered = formatter.format(record) + + assert "task_id=" in rendered + assert "dialog_session_id=" in rendered + assert "graph_id=" in rendered + assert "task-1" not in rendered + assert rendered.endswith("\n") diff --git a/tests/chat/__pycache__/test_chat_api_simple_code_explain.cpython-312-pytest-9.0.2.pyc b/tests/chat/__pycache__/test_chat_api_simple_code_explain.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000..405eda9 Binary files /dev/null and b/tests/chat/__pycache__/test_chat_api_simple_code_explain.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/chat/__pycache__/test_chat_api_simple_code_explain.cpython-312.pyc b/tests/chat/__pycache__/test_chat_api_simple_code_explain.cpython-312.pyc new file mode 100644 index 0000000..ded8368 Binary files /dev/null and b/tests/chat/__pycache__/test_chat_api_simple_code_explain.cpython-312.pyc differ diff --git a/tests/chat/__pycache__/test_direct_service.cpython-312-pytest-9.0.2.pyc b/tests/chat/__pycache__/test_direct_service.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000..a29d09a Binary files /dev/null and b/tests/chat/__pycache__/test_direct_service.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/chat/__pycache__/test_direct_service.cpython-312.pyc b/tests/chat/__pycache__/test_direct_service.cpython-312.pyc new file mode 100644 index 0000000..d59db51 Binary files /dev/null and b/tests/chat/__pycache__/test_direct_service.cpython-312.pyc differ diff --git a/tests/chat/test_chat_api_simple_code_explain.py b/tests/chat/test_chat_api_simple_code_explain.py new file mode 100644 index 0000000..bb7afad --- /dev/null +++ b/tests/chat/test_chat_api_simple_code_explain.py @@ -0,0 +1,70 @@ +import asyncio + +from app.modules.chat.module import ChatModule +from app.modules.chat.task_store import TaskStore +from app.schemas.chat import ChatMessageRequest +from app.schemas.chat import TaskQueuedResponse +from app.modules.shared.event_bus import EventBus +from app.modules.shared.retry_executor import RetryExecutor + + +class _FakeRuntime: + async def run(self, **kwargs): + raise AssertionError("legacy runtime must not be called") + + +class _FakeDirectChat: + def __init__(self) -> None: + self.calls = 0 + + async def handle_message(self, request): + self.calls += 1 + return TaskQueuedResponse( + task_id="task-1", + status="done", + ) + + +class _FakeRagSessions: + def get(self, rag_session_id: str): + return {"rag_session_id": rag_session_id} + + +class _FakeRepository: + def create_dialog(self, dialog_session_id: str, rag_session_id: str) -> None: + return None + + def get_dialog(self, dialog_session_id: str): + return None + + def add_message(self, dialog_session_id: str, role: str, content: str, task_id: str | None = None, payload: dict | None = None) -> None: + return None + + +def test_chat_messages_endpoint_uses_direct_service(monkeypatch) -> None: + monkeypatch.setenv("SIMPLE_CODE_EXPLAIN_ONLY", "true") + direct_chat = _FakeDirectChat() + module = ChatModule( + agent_runner=_FakeRuntime(), + event_bus=EventBus(), + retry=RetryExecutor(), + rag_sessions=_FakeRagSessions(), + repository=_FakeRepository(), + direct_chat=direct_chat, + task_store=TaskStore(), + ) + router = module.public_router() + endpoint = next(route.endpoint for route in router.routes if getattr(route, "path", "") == "/api/chat/messages") + response = asyncio.run( + endpoint( + ChatMessageRequest( + session_id="dialog-1", + project_id="rag-1", + message="Explain get_user", + ), + None, + ) + ) + + assert response.task_id == "task-1" + assert direct_chat.calls == 1 diff --git a/tests/chat/test_direct_service.py b/tests/chat/test_direct_service.py new file mode 100644 index 0000000..96d071c --- /dev/null +++ b/tests/chat/test_direct_service.py @@ -0,0 +1,61 @@ +import asyncio + +from app.modules.chat.direct_service import CodeExplainChatService +from app.modules.chat.session_resolver import ChatSessionResolver +from app.modules.chat.task_store import TaskStore +from app.modules.rag.explain.models import ExplainIntent, ExplainPack +from app.schemas.chat import ChatFileContext, ChatMessageRequest + + +class _FakeRetriever: + def build_pack(self, rag_session_id: str, user_query: str, *, file_candidates: list[dict] | None = None) -> ExplainPack: + return ExplainPack( + intent=ExplainIntent(raw_query=user_query, normalized_query=user_query), + missing=["code_excerpts"], + ) + + +class _FakeLlm: + def __init__(self) -> None: + self.calls = 0 + + def generate(self, prompt_name: str, user_input: str, *, log_context: str | None = None) -> str: + self.calls += 1 + return "should not be called" + + +class _FakeDialogs: + def get(self, dialog_session_id: str): + return None + + +def test_direct_service_skips_llm_when_evidence_is_insufficient() -> None: + messages: list[tuple[str, str, str, str | None]] = [] + llm = _FakeLlm() + task_store = TaskStore() + service = CodeExplainChatService( + retriever=_FakeRetriever(), + llm=llm, + session_resolver=ChatSessionResolver(_FakeDialogs(), lambda rag_session_id: rag_session_id == "rag-1"), + task_store=task_store, + message_sink=lambda dialog_session_id, role, content, task_id=None: messages.append((dialog_session_id, role, content, task_id)), + ) + + result = asyncio.run( + service.handle_message( + ChatMessageRequest( + session_id="dialog-1", + project_id="rag-1", + message="Explain get_user", + files=[ChatFileContext(path="app/api/users.py", content="", content_hash="x")], + ) + ) + ) + + task = task_store.get(result.task_id) + assert task is not None + assert task.answer is not None + assert "Недостаточно опоры в коде" in task.answer + assert result.status == "done" + assert llm.calls == 0 + assert [item[1] for item in messages] == ["user", "assistant"] diff --git a/tests/rag/__pycache__/asserts_intent_router.cpython-312.pyc b/tests/rag/__pycache__/asserts_intent_router.cpython-312.pyc new file mode 100644 index 0000000..3628608 Binary files /dev/null and b/tests/rag/__pycache__/asserts_intent_router.cpython-312.pyc differ diff --git a/tests/rag/__pycache__/intent_router_testkit.cpython-312.pyc b/tests/rag/__pycache__/intent_router_testkit.cpython-312.pyc new file mode 100644 index 0000000..4262ead Binary files /dev/null and b/tests/rag/__pycache__/intent_router_testkit.cpython-312.pyc differ diff --git a/tests/rag/__pycache__/test_code_indexing_pipeline.cpython-312-pytest-9.0.2.pyc b/tests/rag/__pycache__/test_code_indexing_pipeline.cpython-312-pytest-9.0.2.pyc index 7db10f9..f883d4d 100644 Binary files a/tests/rag/__pycache__/test_code_indexing_pipeline.cpython-312-pytest-9.0.2.pyc and b/tests/rag/__pycache__/test_code_indexing_pipeline.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/rag/__pycache__/test_code_indexing_pipeline.cpython-312.pyc b/tests/rag/__pycache__/test_code_indexing_pipeline.cpython-312.pyc new file mode 100644 index 0000000..60ae30d Binary files /dev/null and b/tests/rag/__pycache__/test_code_indexing_pipeline.cpython-312.pyc differ diff --git a/tests/rag/__pycache__/test_docs_indexing_pipeline.cpython-312.pyc b/tests/rag/__pycache__/test_docs_indexing_pipeline.cpython-312.pyc new file mode 100644 index 0000000..9ec6c7c Binary files /dev/null and b/tests/rag/__pycache__/test_docs_indexing_pipeline.cpython-312.pyc differ diff --git a/tests/rag/__pycache__/test_explain_intent_builder.cpython-312-pytest-9.0.2.pyc b/tests/rag/__pycache__/test_explain_intent_builder.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000..6bcbf1c Binary files /dev/null and b/tests/rag/__pycache__/test_explain_intent_builder.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/rag/__pycache__/test_explain_intent_builder.cpython-312.pyc b/tests/rag/__pycache__/test_explain_intent_builder.cpython-312.pyc new file mode 100644 index 0000000..4433987 Binary files /dev/null and b/tests/rag/__pycache__/test_explain_intent_builder.cpython-312.pyc differ diff --git a/tests/rag/__pycache__/test_intent_router_e2e_flows.cpython-312-pytest-9.0.2.pyc b/tests/rag/__pycache__/test_intent_router_e2e_flows.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000..5047641 Binary files /dev/null and b/tests/rag/__pycache__/test_intent_router_e2e_flows.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/rag/__pycache__/test_intent_router_invariants.cpython-312-pytest-9.0.2.pyc b/tests/rag/__pycache__/test_intent_router_invariants.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000..6b50790 Binary files /dev/null and b/tests/rag/__pycache__/test_intent_router_invariants.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/rag/__pycache__/test_intent_router_phrase_matrix.cpython-312-pytest-9.0.2.pyc b/tests/rag/__pycache__/test_intent_router_phrase_matrix.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000..6baa842 Binary files /dev/null and b/tests/rag/__pycache__/test_intent_router_phrase_matrix.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/rag/__pycache__/test_intent_router_v2.cpython-312-pytest-9.0.2.pyc b/tests/rag/__pycache__/test_intent_router_v2.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000..d320cab Binary files /dev/null and b/tests/rag/__pycache__/test_intent_router_v2.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/rag/__pycache__/test_intent_router_v2.cpython-312.pyc b/tests/rag/__pycache__/test_intent_router_v2.cpython-312.pyc new file mode 100644 index 0000000..e7c0380 Binary files /dev/null and b/tests/rag/__pycache__/test_intent_router_v2.cpython-312.pyc differ diff --git a/tests/rag/__pycache__/test_intent_router_v2_local_flow.cpython-312-pytest-9.0.2.pyc b/tests/rag/__pycache__/test_intent_router_v2_local_flow.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000..50f714a Binary files /dev/null and b/tests/rag/__pycache__/test_intent_router_v2_local_flow.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/rag/__pycache__/test_intent_router_v2_local_flow.cpython-312.pyc b/tests/rag/__pycache__/test_intent_router_v2_local_flow.cpython-312.pyc new file mode 100644 index 0000000..e5b7221 Binary files /dev/null and b/tests/rag/__pycache__/test_intent_router_v2_local_flow.cpython-312.pyc differ diff --git a/tests/rag/__pycache__/test_layered_gateway.cpython-312-pytest-9.0.2.pyc b/tests/rag/__pycache__/test_layered_gateway.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000..49c720a Binary files /dev/null and b/tests/rag/__pycache__/test_layered_gateway.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/rag/__pycache__/test_query_normalization.cpython-312-pytest-9.0.2.pyc b/tests/rag/__pycache__/test_query_normalization.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000..59302d0 Binary files /dev/null and b/tests/rag/__pycache__/test_query_normalization.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/rag/__pycache__/test_query_normalization.cpython-312.pyc b/tests/rag/__pycache__/test_query_normalization.cpython-312.pyc new file mode 100644 index 0000000..363477c Binary files /dev/null and b/tests/rag/__pycache__/test_query_normalization.cpython-312.pyc differ diff --git a/tests/rag/__pycache__/test_query_terms.cpython-312.pyc b/tests/rag/__pycache__/test_query_terms.cpython-312.pyc new file mode 100644 index 0000000..ec3f5be Binary files /dev/null and b/tests/rag/__pycache__/test_query_terms.cpython-312.pyc differ diff --git a/tests/rag/__pycache__/test_rag_service_logging.cpython-312-pytest-9.0.2.pyc b/tests/rag/__pycache__/test_rag_service_logging.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000..2232cfa Binary files /dev/null and b/tests/rag/__pycache__/test_rag_service_logging.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/rag/__pycache__/test_retrieval_statement_builder.cpython-312-pytest-9.0.2.pyc b/tests/rag/__pycache__/test_retrieval_statement_builder.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000..d79cbb2 Binary files /dev/null and b/tests/rag/__pycache__/test_retrieval_statement_builder.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/rag/__pycache__/test_retriever_v2_no_fallback.cpython-312-pytest-9.0.2.pyc b/tests/rag/__pycache__/test_retriever_v2_no_fallback.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000..32afabf Binary files /dev/null and b/tests/rag/__pycache__/test_retriever_v2_no_fallback.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/rag/__pycache__/test_retriever_v2_no_fallback.cpython-312.pyc b/tests/rag/__pycache__/test_retriever_v2_no_fallback.cpython-312.pyc new file mode 100644 index 0000000..6719c03 Binary files /dev/null and b/tests/rag/__pycache__/test_retriever_v2_no_fallback.cpython-312.pyc differ diff --git a/tests/rag/__pycache__/test_retriever_v2_pack.cpython-312-pytest-9.0.2.pyc b/tests/rag/__pycache__/test_retriever_v2_pack.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000..795c69e Binary files /dev/null and b/tests/rag/__pycache__/test_retriever_v2_pack.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/rag/__pycache__/test_retriever_v2_pack.cpython-312.pyc b/tests/rag/__pycache__/test_retriever_v2_pack.cpython-312.pyc new file mode 100644 index 0000000..bfd2857 Binary files /dev/null and b/tests/rag/__pycache__/test_retriever_v2_pack.cpython-312.pyc differ diff --git a/tests/rag/__pycache__/test_retriever_v2_production_first.cpython-312-pytest-9.0.2.pyc b/tests/rag/__pycache__/test_retriever_v2_production_first.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000..7bb833e Binary files /dev/null and b/tests/rag/__pycache__/test_retriever_v2_production_first.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/rag/__pycache__/test_trace_builder.cpython-312-pytest-9.0.2.pyc b/tests/rag/__pycache__/test_trace_builder.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000..a0eb654 Binary files /dev/null and b/tests/rag/__pycache__/test_trace_builder.cpython-312-pytest-9.0.2.pyc differ diff --git a/tests/rag/__pycache__/test_trace_builder.cpython-312.pyc b/tests/rag/__pycache__/test_trace_builder.cpython-312.pyc new file mode 100644 index 0000000..f5354ec Binary files /dev/null and b/tests/rag/__pycache__/test_trace_builder.cpython-312.pyc differ diff --git a/tests/rag/asserts_intent_router.py b/tests/rag/asserts_intent_router.py new file mode 100644 index 0000000..6df58c8 --- /dev/null +++ b/tests/rag/asserts_intent_router.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import re + +from app.modules.rag.intent_router_v2.models import IntentRouterResult + + +def assert_intent(out: IntentRouterResult, expected: str) -> None: + assert out.intent == expected + + +def assert_domains(out: IntentRouterResult, expected: list[str]) -> None: + assert out.retrieval_spec.domains == expected + + +def assert_has_file_path(out: IntentRouterResult, path: str) -> None: + assert any(anchor.type == "FILE_PATH" and anchor.value == path for anchor in out.query_plan.anchors) + + +def assert_path_scope(out: IntentRouterResult, file_path: str, dir_path: str | None = None) -> None: + scope = list(getattr(out.retrieval_spec.filters, "path_scope", []) or []) + assert file_path in scope + if dir_path is not None: + assert dir_path in scope + + +def assert_file_only_scope(out: IntentRouterResult, file_path: str) -> None: + scope = list(getattr(out.retrieval_spec.filters, "path_scope", []) or []) + assert scope == [file_path] + + +def assert_spans_valid(out: IntentRouterResult) -> None: + raw_len = len(out.query_plan.raw) + for anchor in out.query_plan.anchors: + if anchor.source == "conversation_state": + assert anchor.span is None + continue + assert anchor.span is not None + assert 0 <= anchor.span.start < anchor.span.end <= raw_len + + +def assert_test_policy(out: IntentRouterResult, expected: str) -> None: + assert getattr(out.retrieval_spec.filters, "test_policy", None) == expected + + +def assert_sub_intent(out: IntentRouterResult, expected: str) -> None: + assert out.query_plan.sub_intent == expected + + +def assert_no_symbol_keyword(out: IntentRouterResult, forbidden: set[str] | None = None) -> None: + denied = forbidden or {"def", "class", "return", "import", "from"} + symbols = {anchor.value.lower() for anchor in out.query_plan.anchors if anchor.type == "SYMBOL"} + assert symbols.isdisjoint({token.lower() for token in denied}) + + +def assert_domain_layer_prefixes(out: IntentRouterResult) -> None: + prefixes = {layer.layer_id[0] for layer in out.retrieval_spec.layer_queries if layer.layer_id} + if out.retrieval_spec.domains == ["CODE"]: + assert prefixes <= {"C"} + elif out.retrieval_spec.domains == ["DOCS"]: + assert prefixes <= {"D"} + else: + assert prefixes <= {"C", "D"} + + +def assert_no_symbol_leakage_from_paths(out: IntentRouterResult) -> None: + file_values = [anchor.value for anchor in out.query_plan.anchors if anchor.type == "FILE_PATH"] + if not file_values: + return + parts: set[str] = set() + for value in file_values: + for token in re.split(r"[/.]+", value.lower()): + if token: + parts.add(token) + for anchor in out.query_plan.anchors: + if anchor.type == "SYMBOL": + assert anchor.value.lower() not in parts diff --git a/tests/rag/intent_router_testkit.py b/tests/rag/intent_router_testkit.py new file mode 100644 index 0000000..257df03 --- /dev/null +++ b/tests/rag/intent_router_testkit.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from app.modules.rag.contracts.enums import RagLayer +from app.modules.rag.intent_router_v2 import ConversationState, IntentRouterV2, RepoContext + + +def repo_context() -> RepoContext: + return RepoContext( + languages=["python"], + available_domains=["CODE", "DOCS"], + available_layers=[ + RagLayer.CODE_ENTRYPOINTS, + RagLayer.CODE_SYMBOL_CATALOG, + RagLayer.CODE_DEPENDENCY_GRAPH, + RagLayer.CODE_SOURCE_CHUNKS, + RagLayer.DOCS_MODULE_CATALOG, + RagLayer.DOCS_FACT_INDEX, + RagLayer.DOCS_SECTION_INDEX, + RagLayer.DOCS_POLICY_INDEX, + ], + ) + + +def run_sequence(queries: list[str], *, router: IntentRouterV2 | None = None, trace_label: str = "intent-router") -> list: + active_router = router or IntentRouterV2() + state = ConversationState() + results = [] + for index, query in enumerate(queries, start=1): + result = active_router.route(query, state, repo_context()) + print_trace(index, query, result, label=trace_label) + results.append(result) + state = state.advance(result) + return results + + +def run_single(query: str, *, router: IntentRouterV2 | None = None, trace_label: str = "intent-router"): + result = run_sequence([query], router=router, trace_label=trace_label)[0] + return result + + +def print_trace(index: int, query: str, result, *, label: str = "intent-router") -> None: + print(f"[{label}][turn {index}] input: {query}") + print() + print(f"[{label}][turn {index}] output: {result.model_dump_json(ensure_ascii=False)}") + print("=" * 50) diff --git a/tests/rag/test_code_indexing_pipeline.py b/tests/rag/test_code_indexing_pipeline.py index 9ba4bf9..2f1a35b 100644 --- a/tests/rag/test_code_indexing_pipeline.py +++ b/tests/rag/test_code_indexing_pipeline.py @@ -55,3 +55,21 @@ def test_code_pipeline_indexes_import_alias_as_symbol() -> None: alias_doc = next(doc for doc in docs if doc.layer == RagLayer.CODE_SYMBOL_CATALOG and doc.metadata["qname"] == "ConfigManager") assert alias_doc.metadata["kind"] == "const" assert alias_doc.metadata["lang_payload"]["import_alias"] is True + + +def test_code_pipeline_marks_test_documents() -> None: + pipeline = CodeIndexingPipeline() + content = """ +def test_user_service(): + assert True +""" + + docs = pipeline.index_file( + repo_id="acme/proj", + commit_sha="abc123", + path="tests/test_users.py", + content=content, + ) + + assert docs + assert all(doc.metadata["is_test"] is True for doc in docs) diff --git a/tests/rag/test_explain_intent_builder.py b/tests/rag/test_explain_intent_builder.py new file mode 100644 index 0000000..f386561 --- /dev/null +++ b/tests/rag/test_explain_intent_builder.py @@ -0,0 +1,22 @@ +from app.modules.rag.explain.intent_builder import ExplainIntentBuilder + + +def test_explain_intent_builder_extracts_route_symbol_and_file_hints() -> None: + builder = ExplainIntentBuilder() + + intent = builder.build("Explain how /users/{user_id} reaches UserService.get_user in app/api/users.py") + + assert "/users/{user_id}" in intent.hints.endpoints + assert "UserService.get_user" in intent.hints.symbols + assert "app/api/users.py" in intent.hints.paths + assert intent.expected_entry_types == ["http"] + assert intent.include_tests is False + assert intent.depth == "medium" + + +def test_explain_intent_builder_enables_tests_when_user_asks_for_them() -> None: + builder = ExplainIntentBuilder() + + intent = builder.build("Покажи как это тестируется в pytest и какие tests покрывают UserService") + + assert intent.include_tests is True diff --git a/tests/rag/test_intent_router_e2e_flows.py b/tests/rag/test_intent_router_e2e_flows.py new file mode 100644 index 0000000..8f76ad7 --- /dev/null +++ b/tests/rag/test_intent_router_e2e_flows.py @@ -0,0 +1,126 @@ +import os + +import pytest + +from app.modules.rag.intent_router_v2 import GigaChatIntentRouterFactory +from app.modules.shared.env_loader import load_workspace_env +from tests.rag.asserts_intent_router import ( + assert_domains, + assert_file_only_scope, + assert_intent, + assert_test_policy, +) +from tests.rag.intent_router_testkit import run_sequence + +pytestmark = pytest.mark.intent_router + + +def _live_gigachat_enabled() -> bool: + load_workspace_env() + return os.getenv("RUN_INTENT_ROUTER_V2_LIVE", "").strip() == "1" and bool(os.getenv("GIGACHAT_TOKEN", "").strip()) + + +def test_e2e_path_carryover_flow() -> None: + first, second, third = run_sequence( + [ + "Посмотри файл app/core/config.py", + "Теперь объясни функцию load_config", + "Почему так?", + ] + ) + + assert_file_only_scope(first, "app/core/config.py") + assert "app/core/config.py" in second.retrieval_spec.filters.path_scope + assert "app/core/config.py" in third.retrieval_spec.filters.path_scope + second_file_anchors = [anchor.value for anchor in second.query_plan.anchors if anchor.type == "FILE_PATH" and anchor.source == "conversation_state"] + assert second_file_anchors == ["app/core/config.py"] + assert "app/core/config.py" in second.query_plan.keyword_hints + assert "app/core" not in second.query_plan.keyword_hints + assert any(anchor.type == "FILE_PATH" and anchor.source == "conversation_state" and anchor.span is None for anchor in third.query_plan.anchors) + carried_symbols = [anchor.value for anchor in third.query_plan.anchors if anchor.type == "SYMBOL" and anchor.source == "conversation_state"] + assert carried_symbols in ([], ["load_config"]) + assert third.query_plan.sub_intent == "EXPLAIN_LOCAL" + layer_ids = [item.layer_id for item in third.retrieval_spec.layer_queries] + assert "C3_ENTRYPOINTS" not in layer_ids + + +def test_e2e_docs_switch_from_code_topic() -> None: + first, second = run_sequence( + [ + "Объясни как работает ConfigManager", + "А что про это сказано в документации?", + ] + ) + + assert_intent(first, "CODE_QA") + assert_intent(second, "DOCS_QA") + assert second.conversation_mode == "SWITCH" + assert_domains(second, ["DOCS"]) + carried = [ + anchor + for anchor in second.query_plan.anchors + if anchor.type == "SYMBOL" and anchor.value == "ConfigManager" and anchor.source == "conversation_state" + ] + assert carried + assert carried[0].span is None + assert "ConfigManager" in second.query_plan.expansions + assert "ConfigManager" in second.query_plan.keyword_hints + + +def test_e2e_tests_toggle_flow() -> None: + first, second = run_sequence( + [ + "Покажи тесты для ConfigManager", + "А теперь не про тесты, а про прод код", + ] + ) + + assert_intent(first, "CODE_QA") + assert_intent(second, "CODE_QA") + assert_test_policy(first, "INCLUDE") + assert_test_policy(second, "EXCLUDE") + assert first.query_plan.sub_intent == "FIND_TESTS" + assert second.query_plan.sub_intent == "EXPLAIN" + assert "tests" in second.query_plan.negations + assert not second.query_plan.expansions + assert second.evidence_policy.require_flow is False + + +def test_e2e_open_file_then_generic_next_steps_is_lightweight() -> None: + first, second = run_sequence( + [ + "Открой файл app/core/config.py", + "Что дальше?", + ] + ) + + assert_file_only_scope(first, "app/core/config.py") + assert_file_only_scope(second, "app/core/config.py") + assert second.query_plan.sub_intent in {"EXPLAIN_LOCAL", "NEXT_STEPS"} + layer_ids = [item.layer_id for item in second.retrieval_spec.layer_queries] + assert "C3_ENTRYPOINTS" not in layer_ids + assert second.evidence_policy.require_flow is False + assert "app/core/config.py" in second.query_plan.keyword_hints + + +@pytest.mark.skipif( + not _live_gigachat_enabled(), + reason="requires RUN_INTENT_ROUTER_V2_LIVE=1 and GIGACHAT_TOKEN in environment or .env", +) +def test_intent_router_live_smoke_path_carryover() -> None: + router = GigaChatIntentRouterFactory().build() + first, second = run_sequence( + [ + "Открой файл app/core/config.py", + "Что дальше?", + ], + router=router, + trace_label="intent-router-live", + ) + + assert_file_only_scope(first, "app/core/config.py") + assert "app/core/config.py" in second.retrieval_spec.filters.path_scope + assert second.query_plan.sub_intent in {"EXPLAIN_LOCAL", "NEXT_STEPS"} + layer_ids = [item.layer_id for item in second.retrieval_spec.layer_queries] + assert "C3_ENTRYPOINTS" not in layer_ids + assert second.evidence_policy.require_flow is False diff --git a/tests/rag/test_intent_router_invariants.py b/tests/rag/test_intent_router_invariants.py new file mode 100644 index 0000000..361fd44 --- /dev/null +++ b/tests/rag/test_intent_router_invariants.py @@ -0,0 +1,120 @@ +import pytest + +from tests.rag.asserts_intent_router import ( + assert_domain_layer_prefixes, + assert_domains, + assert_file_only_scope, + assert_has_file_path, + assert_intent, + assert_no_symbol_keyword, + assert_no_symbol_leakage_from_paths, + assert_spans_valid, + assert_sub_intent, + assert_test_policy, +) +from tests.rag.intent_router_testkit import run_sequence + +pytestmark = pytest.mark.intent_router + + +def test_invariant_code_file_path_with_canonical_key_term() -> None: + result = run_sequence(["Уточни по файлу app/core/config.py"])[0] + + assert_intent(result, "CODE_QA") + assert_has_file_path(result, "app/core/config.py") + assert_file_only_scope(result, "app/core/config.py") + key_terms = [anchor.value for anchor in result.query_plan.anchors if anchor.type == "KEY_TERM"] + assert "файл" in key_terms + assert "файлу" not in key_terms + assert_spans_valid(result) + assert_domain_layer_prefixes(result) + + +def test_invariant_open_file_for_specified_file_phrase_uses_narrow_layers() -> None: + result = run_sequence(["Уточни по файлу app/core/config.py"])[0] + + assert_intent(result, "CODE_QA") + assert_sub_intent(result, "OPEN_FILE") + assert_file_only_scope(result, "app/core/config.py") + layer_ids = [item.layer_id for item in result.retrieval_spec.layer_queries] + assert layer_ids == ["C0_SOURCE_CHUNKS"] + assert result.evidence_policy.require_flow is False + + +def test_invariant_inline_code_span_routes_to_code_and_extracts_symbol() -> None: + result = run_sequence(["Уточни по коду `def build(x): return x`"])[0] + + assert_intent(result, "CODE_QA") + assert_spans_valid(result) + assert_no_symbol_keyword(result) + symbols = [anchor.value for anchor in result.query_plan.anchors if anchor.type == "SYMBOL"] + key_terms = [anchor.value for anchor in result.query_plan.anchors if anchor.type == "KEY_TERM"] + assert "build" in symbols + assert "def" in key_terms + + +def test_invariant_docs_cyrillic_path_with_quotes() -> None: + result = run_sequence(["Что сказано в «docs/архитектура.md»?"])[0] + + assert_intent(result, "DOCS_QA") + assert_sub_intent(result, "EXPLAIN") + assert_domains(result, ["DOCS"]) + assert "docs/архитектура.md" in result.query_plan.normalized + assert_has_file_path(result, "docs/архитектура.md") + assert any(anchor.type == "DOC_REF" for anchor in result.query_plan.anchors) + assert result.retrieval_spec.filters.doc_kinds == [] + assert_spans_valid(result) + assert_domain_layer_prefixes(result) + + +def test_invariant_file_check_phrase_not_project_misc() -> None: + result = run_sequence(["Проверь app/modules/rag/explain/intent_builder.py и объясни"])[0] + + assert_intent(result, "CODE_QA") + assert_domains(result, ["CODE"]) + assert_no_symbol_leakage_from_paths(result) + assert_domain_layer_prefixes(result) + + +def test_invariant_tests_include_routing() -> None: + result = run_sequence(["Где тесты на ConfigManager?"])[0] + + assert_intent(result, "CODE_QA") + assert_test_policy(result, "INCLUDE") + symbols = [anchor.value for anchor in result.query_plan.anchors if anchor.type == "SYMBOL"] + key_terms = [anchor.value for anchor in result.query_plan.anchors if anchor.type == "KEY_TERM"] + assert "ConfigManager" in symbols + assert "тест" in key_terms + + +def test_invariant_keyword_hints_and_expansions_for_function_identifier() -> None: + result = run_sequence(["Теперь объясни функцию load_config"])[0] + + assert_intent(result, "CODE_QA") + assert "load_config" in result.query_plan.keyword_hints + assert "функция" not in result.query_plan.keyword_hints + assert "def" not in result.query_plan.expansions + + +def test_invariant_open_file_sub_intent_uses_narrow_retrieval_profile() -> None: + result = run_sequence(["Открой файл app/core/config.py"])[0] + + assert_intent(result, "CODE_QA") + assert_sub_intent(result, "OPEN_FILE") + assert_file_only_scope(result, "app/core/config.py") + layer_ids = [item.layer_id for item in result.retrieval_spec.layer_queries] + assert "C0_SOURCE_CHUNKS" in layer_ids + assert "C1_SYMBOL_CATALOG" not in layer_ids + assert "C2_DEPENDENCY_GRAPH" not in layer_ids + assert "C3_ENTRYPOINTS" not in layer_ids + assert result.evidence_policy.require_flow is False + + +def test_invariant_docs_question_routes_to_docs() -> None: + result = run_sequence(["Что сказано в документации?"])[0] + + assert_intent(result, "DOCS_QA") + assert_domains(result, ["DOCS"]) + assert_domain_layer_prefixes(result) + assert result.query_plan.keyword_hints + assert any(item in result.query_plan.expansions for item in result.query_plan.keyword_hints) diff --git a/tests/rag/test_layered_gateway.py b/tests/rag/test_layered_gateway.py new file mode 100644 index 0000000..49cf6ce --- /dev/null +++ b/tests/rag/test_layered_gateway.py @@ -0,0 +1,78 @@ +from app.modules.rag.explain.layered_gateway import LayeredRetrievalGateway + + +class _Embedder: + def embed(self, texts: list[str]) -> list[list[float]]: + return [[0.1, 0.2]] + + +class _RetryingRepository: + def __init__(self) -> None: + self.calls: list[dict] = [] + + def retrieve(self, *args, **kwargs): + self.calls.append(kwargs) + if kwargs.get("exclude_path_prefixes"): + raise RuntimeError("syntax error at or near ')'") + return [ + { + "path": "app/users/service.py", + "content": "def get_user(): pass", + "layer": "C1_SYMBOL_CATALOG", + "title": "get_user", + "metadata": {"symbol_id": "symbol-1"}, + "distance": 0.1, + "span_start": 10, + "span_end": 11, + } + ] + + def retrieve_lexical_code(self, *args, **kwargs): + self.calls.append(kwargs) + if kwargs.get("exclude_path_prefixes"): + raise RuntimeError("broken lexical filter") + return [ + { + "path": "app/users/service.py", + "content": "def get_user(): pass", + "layer": "C0_SOURCE_CHUNKS", + "title": "get_user", + "metadata": {"symbol_id": "symbol-1"}, + "span_start": 10, + "span_end": 11, + } + ] + + +class _RecordingRepository: + def __init__(self) -> None: + self.calls: list[dict] = [] + + def retrieve(self, *args, **kwargs): + self.calls.append(kwargs) + return [] + + def retrieve_lexical_code(self, *args, **kwargs): + self.calls.append(kwargs) + return [] + + +def test_gateway_retries_without_test_filter_on_vector_failure() -> None: + gateway = LayeredRetrievalGateway(_RetryingRepository(), _Embedder()) + + result = gateway.retrieve_layer("rag-1", "Explain get_user", "C1_SYMBOL_CATALOG", limit=3, exclude_tests=True) + + assert len(result.items) == 1 + assert "layer:C1_SYMBOL_CATALOG retrieval_failed:retried_without_test_filter" in result.missing + + +def test_gateway_honors_debug_disable_test_filter(monkeypatch) -> None: + monkeypatch.setenv("RAG_DEBUG_DISABLE_TEST_FILTER", "true") + repository = _RecordingRepository() + gateway = LayeredRetrievalGateway(repository, _Embedder()) + + gateway.retrieve_layer("rag-1", "Explain get_user", "C1_SYMBOL_CATALOG", limit=3, exclude_tests=True) + + assert repository.calls + assert repository.calls[0]["exclude_path_prefixes"] is None + assert repository.calls[0]["exclude_like_patterns"] is None diff --git a/tests/rag/test_query_normalization.py b/tests/rag/test_query_normalization.py new file mode 100644 index 0000000..828c948 --- /dev/null +++ b/tests/rag/test_query_normalization.py @@ -0,0 +1,63 @@ +import pytest + +from app.modules.rag.intent_router_v2.normalization import QueryNormalizer + +pytestmark = pytest.mark.intent_router + + +def test_query_normalizer_collapses_whitespace() -> None: + normalizer = QueryNormalizer() + + normalized = normalizer.normalize(" Объясни как работает \n класс X ") + + assert normalized == "Объясни как работает класс X" + + +def test_query_normalizer_canonicalizes_quotes() -> None: + normalizer = QueryNormalizer() + + normalized = normalizer.normalize('Уточни «текст» и “текст”') + + assert normalized == 'Уточни "текст" и "текст"' + + +def test_query_normalizer_preserves_backticks_verbatim() -> None: + normalizer = QueryNormalizer() + + normalized = normalizer.normalize("Уточни по коду `def build(x):` ") + + assert normalized == "Уточни по коду `def build(x):`" + + +def test_query_normalizer_preserves_latin_and_cyrillic_file_paths() -> None: + normalizer = QueryNormalizer() + + normalized = normalizer.normalize("Сверь app/core/config.py и «docs/руководство.md»") + + assert "app/core/config.py" in normalized + assert "docs/руководство.md" in normalized + assert "config. py" not in normalized + assert "руководство. md" not in normalized + + +def test_query_normalizer_punctuation_spacing_does_not_break_extensions() -> None: + normalizer = QueryNormalizer() + + normalized = normalizer.normalize("Проверь docs/spec.md , затем app/main.py !") + + assert "docs/spec.md" in normalized + assert "app/main.py" in normalized + assert "spec. md" not in normalized + assert "main. py" not in normalized + + +def test_query_normalizer_idempotent_and_without_enrichment() -> None: + normalizer = QueryNormalizer() + raw = ' Прочитай «README.md» и docs/spec.md ' + + once = normalizer.normalize(raw) + twice = normalizer.normalize(once) + + assert twice == once + assert "documentation" not in once.lower() + assert "class" not in once.lower() diff --git a/tests/rag/test_query_router.py b/tests/rag/test_query_router.py deleted file mode 100644 index 8da84d7..0000000 --- a/tests/rag/test_query_router.py +++ /dev/null @@ -1,12 +0,0 @@ -from app.modules.rag.contracts.enums import RetrievalMode -from app.modules.rag.retrieval.query_router import RagQueryRouter - - -def test_query_router_uses_docs_by_default() -> None: - router = RagQueryRouter() - assert router.resolve_mode("Какие есть требования по биллингу?") == RetrievalMode.DOCS - - -def test_query_router_switches_to_code_on_explicit_code_requests() -> None: - router = RagQueryRouter() - assert router.resolve_mode("Объясни как работает код endpoint create invoice") == RetrievalMode.CODE diff --git a/tests/rag/test_retrieval_statement_builder.py b/tests/rag/test_retrieval_statement_builder.py new file mode 100644 index 0000000..f9b4cd8 --- /dev/null +++ b/tests/rag/test_retrieval_statement_builder.py @@ -0,0 +1,44 @@ +from app.modules.rag.persistence.retrieval_statement_builder import RetrievalStatementBuilder +from app.modules.rag.retrieval.test_filter import build_test_filters, is_test_path + + +def test_retrieve_builder_adds_test_exclusion_filters() -> None: + builder = RetrievalStatementBuilder() + test_filters = build_test_filters() + + sql, params = builder.build_retrieve( + "rag-1", + [0.1, 0.2], + query_text="Explain user service", + layers=["C0_SOURCE_CHUNKS"], + exclude_path_prefixes=test_filters.exclude_path_prefixes, + exclude_like_patterns=test_filters.exclude_like_patterns, + ) + + assert "NOT (" in sql + assert "path LIKE :exclude_prefix_0" in sql + assert "lower(path) LIKE :exclude_like_0" in sql + assert "ESCAPE E'\\\\'" in sql + assert params["exclude_prefix_0"] == "tests/%" + assert "%.test.%" in params.values() + assert "%\\_test.%" in params.values() + + +def test_lexical_builder_omits_test_filters_when_not_requested() -> None: + builder = RetrievalStatementBuilder() + + sql, params = builder.build_lexical_code( + "rag-1", + query_text="Explain user service", + prefer_non_tests=False, + ) + + assert sql is not None + assert "exclude_prefix" not in sql + assert "exclude_like" not in sql + assert not any(key.startswith("exclude_") for key in params) + + +def test_test_filter_does_not_treat_contest_file_as_test() -> None: + assert is_test_path("app/contest.py") is False + assert is_test_path("tests/test_users.py") is True diff --git a/tests/rag/test_retriever_v2_no_fallback.py b/tests/rag/test_retriever_v2_no_fallback.py new file mode 100644 index 0000000..dd7f0df --- /dev/null +++ b/tests/rag/test_retriever_v2_no_fallback.py @@ -0,0 +1,52 @@ +from app.modules.rag.explain import CodeExplainRetrieverV2, LayeredRetrievalGateway + + +class _ExplodingEmbedder: + def embed(self, texts: list[str]) -> list[list[float]]: + raise RuntimeError("embedding unavailable") + + +class _RepositoryWithoutFallback: + def retrieve(self, *args, **kwargs): + raise RuntimeError("vector retrieval unavailable") + + def retrieve_lexical_code( + self, + rag_session_id: str, + query_text: str, + *, + limit: int = 5, + path_prefixes: list[str] | None = None, + exclude_path_prefixes: list[str] | None = None, + exclude_like_patterns: list[str] | None = None, + prefer_non_tests: bool = False, + ): + return [] + + +class _FakeGraphRepository: + def get_symbols_by_ids(self, rag_session_id: str, symbol_ids: list[str]): + return [] + + def get_chunks_by_symbol_ids(self, rag_session_id: str, symbol_ids: list[str], prefer_chunk_type: str = "symbol_block"): + return [] + + def get_out_edges(self, rag_session_id: str, src_symbol_ids: list[str], edge_types: list[str], limit_per_src: int): + return [] + + def resolve_symbol_by_ref(self, rag_session_id: str, dst_ref: str, package_hint: str | None = None): + return None + + +def test_retriever_v2_returns_pack_without_fallback_method() -> None: + retriever = CodeExplainRetrieverV2( + gateway=LayeredRetrievalGateway(_RepositoryWithoutFallback(), _ExplodingEmbedder()), + graph_repository=_FakeGraphRepository(), + ) + + pack = retriever.build_pack("rag-1", "Explain get_user") + + assert pack.code_excerpts == [] + assert any(item.startswith("layer:C3_ENTRYPOINTS retrieval_failed") for item in pack.missing) + assert any(item.startswith("layer:C1_SYMBOL_CATALOG retrieval_failed") for item in pack.missing) + assert "layer:C0 empty" in pack.missing diff --git a/tests/rag/test_retriever_v2_pack.py b/tests/rag/test_retriever_v2_pack.py new file mode 100644 index 0000000..0fb5c91 --- /dev/null +++ b/tests/rag/test_retriever_v2_pack.py @@ -0,0 +1,105 @@ +from app.modules.rag.explain.models import CodeLocation, LayeredRetrievalItem +from app.modules.rag.explain.retriever_v2 import CodeExplainRetrieverV2 + + +class _FakeGateway: + def retrieve_layer( + self, + rag_session_id: str, + query: str, + layer: str, + *, + limit: int, + path_prefixes: list[str] | None = None, + exclude_tests: bool = True, + prefer_non_tests: bool = False, + include_spans: bool = False, + ): + if layer == "C3_ENTRYPOINTS": + return __import__("types").SimpleNamespace( + items=[ + LayeredRetrievalItem( + source="app/api/users.py", + content="GET /users/{id}", + layer=layer, + title="GET /users/{id}", + metadata={"entry_type": "http", "handler_symbol_id": "handler-1"}, + location=CodeLocation(path="app/api/users.py", start_line=10, end_line=10), + ) + ], + missing=[], + ) + if layer == "C1_SYMBOL_CATALOG": + return __import__("types").SimpleNamespace( + items=[ + LayeredRetrievalItem( + source="app/api/users.py", + content="def get_user_handler", + layer=layer, + title="get_user_handler", + metadata={"symbol_id": "handler-1"}, + location=CodeLocation(path="app/api/users.py", start_line=10, end_line=18), + ) + ], + missing=[], + ) + raise AssertionError(layer) + + def retrieve_lexical_code( + self, + rag_session_id: str, + query: str, + *, + limit: int, + path_prefixes: list[str] | None = None, + exclude_tests: bool = True, + include_spans: bool = False, + ): + return __import__("types").SimpleNamespace(items=[], missing=[]) + + +class _FakeGraphRepository: + def get_symbols_by_ids(self, rag_session_id: str, symbol_ids: list[str]): + return [ + LayeredRetrievalItem( + source="app/api/users.py", + content="def get_user_handler", + layer="C1_SYMBOL_CATALOG", + title="get_user_handler", + metadata={"symbol_id": "handler-1"}, + location=CodeLocation(path="app/api/users.py", start_line=10, end_line=18), + ) + ] + + def get_out_edges(self, rag_session_id: str, src_symbol_ids: list[str], edge_types: list[str], limit_per_src: int): + return [] + + def resolve_symbol_by_ref(self, rag_session_id: str, dst_ref: str, package_hint: str | None = None): + return None + + def get_chunks_by_symbol_ids(self, rag_session_id: str, symbol_ids: list[str], prefer_chunk_type: str = "symbol_block"): + return [ + LayeredRetrievalItem( + source="app/api/users.py", + content="async def get_user_handler(user_id: str):\n return await service.get_user(user_id)", + layer="C0_SOURCE_CHUNKS", + title="get_user_handler", + metadata={"symbol_id": "handler-1"}, + location=CodeLocation(path="app/api/users.py", start_line=10, end_line=18), + ) + ] + + +def test_retriever_v2_builds_pack_with_trace_and_excerpts() -> None: + retriever = CodeExplainRetrieverV2( + gateway=_FakeGateway(), + graph_repository=_FakeGraphRepository(), + ) + + pack = retriever.build_pack("rag-1", "Explain endpoint get_user") + + assert len(pack.selected_entrypoints) == 1 + assert len(pack.seed_symbols) == 1 + assert len(pack.trace_paths) == 1 + assert len(pack.code_excerpts) == 1 + assert pack.code_excerpts[0].path == "app/api/users.py" diff --git a/tests/rag/test_retriever_v2_production_first.py b/tests/rag/test_retriever_v2_production_first.py new file mode 100644 index 0000000..0971664 --- /dev/null +++ b/tests/rag/test_retriever_v2_production_first.py @@ -0,0 +1,142 @@ +from types import SimpleNamespace + +from app.modules.rag.explain.models import CodeLocation, LayeredRetrievalItem +from app.modules.rag.explain.retriever_v2 import CodeExplainRetrieverV2 + + +class _ProductionFirstGateway: + def __init__(self) -> None: + self.lexical_calls: list[bool] = [] + + def retrieve_layer( + self, + rag_session_id: str, + query: str, + layer: str, + *, + limit: int, + path_prefixes: list[str] | None = None, + exclude_tests: bool = True, + prefer_non_tests: bool = False, + include_spans: bool = False, + ): + if layer == "C3_ENTRYPOINTS": + return SimpleNamespace(items=[], missing=[]) + if layer == "C1_SYMBOL_CATALOG": + return SimpleNamespace(items=[], missing=[]) + raise AssertionError(layer) + + def retrieve_lexical_code( + self, + rag_session_id: str, + query: str, + *, + limit: int, + path_prefixes: list[str] | None = None, + exclude_tests: bool = True, + include_spans: bool = False, + ): + self.lexical_calls.append(exclude_tests) + if exclude_tests: + return SimpleNamespace( + items=[ + LayeredRetrievalItem( + source="app/users/service.py", + content="def get_user():\n return repo.get_user()", + layer="C0_SOURCE_CHUNKS", + title="get_user", + metadata={"symbol_id": "user-service", "is_test": False}, + location=CodeLocation(path="app/users/service.py", start_line=10, end_line=11), + ), + LayeredRetrievalItem( + source="app/users/repository.py", + content="def get_user_repo():\n return {}", + layer="C0_SOURCE_CHUNKS", + title="get_user_repo", + metadata={"symbol_id": "user-repo", "is_test": False}, + location=CodeLocation(path="app/users/repository.py", start_line=20, end_line=21), + ), + ], + missing=[], + ) + return SimpleNamespace( + items=[ + LayeredRetrievalItem( + source="tests/test_users.py", + content="def test_get_user():\n assert service.get_user()", + layer="C0_SOURCE_CHUNKS", + title="test_get_user", + metadata={"symbol_id": "test-user", "is_test": True}, + location=CodeLocation(path="tests/test_users.py", start_line=5, end_line=6), + ) + ], + missing=[], + ) + + +class _TestsOnlyGateway(_ProductionFirstGateway): + def retrieve_lexical_code( + self, + rag_session_id: str, + query: str, + *, + limit: int, + path_prefixes: list[str] | None = None, + exclude_tests: bool = True, + include_spans: bool = False, + ): + self.lexical_calls.append(exclude_tests) + if exclude_tests: + return SimpleNamespace(items=[], missing=[]) + return SimpleNamespace( + items=[ + LayeredRetrievalItem( + source="tests/test_users.py", + content="def test_get_user():\n assert service.get_user()", + layer="C0_SOURCE_CHUNKS", + title="test_get_user", + metadata={"symbol_id": "test-user", "is_test": True}, + location=CodeLocation(path="tests/test_users.py", start_line=5, end_line=6), + ) + ], + missing=[], + ) + + +class _FakeGraphRepository: + def get_symbols_by_ids(self, rag_session_id: str, symbol_ids: list[str]): + return [] + + def get_chunks_by_symbol_ids(self, rag_session_id: str, symbol_ids: list[str], prefer_chunk_type: str = "symbol_block"): + return [] + + def get_out_edges(self, rag_session_id: str, src_symbol_ids: list[str], edge_types: list[str], limit_per_src: int): + return [] + + def resolve_symbol_by_ref(self, rag_session_id: str, dst_ref: str, package_hint: str | None = None): + return None + + +def test_retriever_prefers_prod_chunks_and_skips_test_fallback_when_enough_evidence() -> None: + gateway = _ProductionFirstGateway() + retriever = CodeExplainRetrieverV2(gateway=gateway, graph_repository=_FakeGraphRepository()) + + pack = retriever.build_pack("rag-1", "Explain get_user") + + assert gateway.lexical_calls == [True] + assert [excerpt.path for excerpt in pack.code_excerpts] == [ + "app/users/service.py", + "app/users/repository.py", + ] + assert all(not excerpt.focus.startswith("test:") for excerpt in pack.code_excerpts) + + +def test_retriever_uses_test_fallback_when_production_evidence_is_missing() -> None: + gateway = _TestsOnlyGateway() + retriever = CodeExplainRetrieverV2(gateway=gateway, graph_repository=_FakeGraphRepository()) + + pack = retriever.build_pack("rag-1", "Explain get_user") + + assert gateway.lexical_calls == [True, False] + assert [excerpt.path for excerpt in pack.code_excerpts] == ["tests/test_users.py"] + assert pack.code_excerpts[0].focus == "test:lexical" diff --git a/tests/rag/test_trace_builder.py b/tests/rag/test_trace_builder.py new file mode 100644 index 0000000..0292121 --- /dev/null +++ b/tests/rag/test_trace_builder.py @@ -0,0 +1,83 @@ +from app.modules.rag.explain.models import CodeLocation, LayeredRetrievalItem +from app.modules.rag.explain.trace_builder import TraceBuilder + + +class _FakeGraphRepository: + def get_out_edges(self, rag_session_id: str, src_symbol_ids: list[str], edge_types: list[str], limit_per_src: int): + assert rag_session_id == "rag-1" + assert edge_types == ["calls", "imports", "inherits"] + if src_symbol_ids == ["handler-1"]: + return [ + LayeredRetrievalItem( + source="app/api/users.py", + content="handler calls get_user", + layer="C2_DEPENDENCY_GRAPH", + title="handler:calls", + metadata={ + "src_symbol_id": "handler-1", + "dst_symbol_id": None, + "dst_ref": "UserService.get_user", + "resolution": "partial", + "edge_type": "calls", + }, + location=CodeLocation(path="app/api/users.py", start_line=12, end_line=12), + ) + ] + return [] + + def resolve_symbol_by_ref(self, rag_session_id: str, dst_ref: str, package_hint: str | None = None): + assert rag_session_id == "rag-1" + assert dst_ref == "UserService.get_user" + assert package_hint == "app.api" + return LayeredRetrievalItem( + source="app/services/users.py", + content="method UserService.get_user", + layer="C1_SYMBOL_CATALOG", + title="UserService.get_user", + metadata={ + "symbol_id": "service-1", + "package_or_module": "app.api.users", + }, + location=CodeLocation(path="app/services/users.py", start_line=4, end_line=10), + ) + + def get_symbols_by_ids(self, rag_session_id: str, symbol_ids: list[str]): + assert rag_session_id == "rag-1" + if symbol_ids == ["service-1"]: + return [ + LayeredRetrievalItem( + source="app/services/users.py", + content="method UserService.get_user", + layer="C1_SYMBOL_CATALOG", + title="UserService.get_user", + metadata={ + "symbol_id": "service-1", + "package_or_module": "app.api.users", + }, + location=CodeLocation(path="app/services/users.py", start_line=4, end_line=10), + ) + ] + return [] + + +def test_trace_builder_resolves_partial_edges_across_files() -> None: + builder = TraceBuilder(_FakeGraphRepository()) + seeds = [ + LayeredRetrievalItem( + source="app/api/users.py", + content="function handler", + layer="C1_SYMBOL_CATALOG", + title="get_user", + metadata={ + "symbol_id": "handler-1", + "package_or_module": "app.api.users", + }, + location=CodeLocation(path="app/api/users.py", start_line=10, end_line=18), + ) + ] + + paths = builder.build_paths("rag-1", seeds, max_depth=3) + + assert len(paths) >= 1 + assert paths[0].symbol_ids == ["handler-1", "service-1"] + assert "resolved:UserService.get_user" in paths[0].notes