96 lines
3.7 KiB
Python
96 lines
3.7 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
from app.modules.agent.intent_router_v2 import ConversationState, IntentRouterV2
|
|
from app.modules.agent.intent_router_v2.intent.classifier import IntentClassifierV2
|
|
from app.modules.agent.intent_router_v2.intent.llm_disambiguator import DocsLlmDisambiguator
|
|
from tests.unit_tests.rag.intent_router_testkit import repo_context
|
|
|
|
|
|
class FakeLlm:
|
|
def __init__(self, response: str, *, fail: bool = False) -> None:
|
|
self.response = response
|
|
self.fail = fail
|
|
self.calls: list[tuple[str, str]] = []
|
|
|
|
def generate(self, prompt_name: str, user_input: str, *, log_context: str | None = None) -> str:
|
|
self.calls.append((prompt_name, user_input))
|
|
if self.fail:
|
|
raise RuntimeError("llm unavailable")
|
|
return self.response
|
|
|
|
|
|
def test_technical_query_keeps_deterministic_routing_without_llm_call() -> None:
|
|
llm = FakeLlm('{"sub_intent":"GENERIC_QA","reason":"unused","confidence":"low"}')
|
|
router = IntentRouterV2(
|
|
classifier=IntentClassifierV2(),
|
|
llm_disambiguator=DocsLlmDisambiguator(llm),
|
|
enable_llm_disambiguation=True,
|
|
)
|
|
|
|
result = router.route("Объясни endpoint /health", ConversationState(), repo_context())
|
|
|
|
assert result.query_plan.sub_intent == "API_METHOD_EXPLAIN"
|
|
assert result.is_ambiguous is False
|
|
assert result.routing_mode == "deterministic"
|
|
assert result.llm_router_used is False
|
|
assert llm.calls == []
|
|
|
|
|
|
def test_ambiguous_query_can_be_resolved_by_llm() -> None:
|
|
llm = FakeLlm('{"sub_intent":"ENTITY_EXPLAIN","reason":"runtime health is a concept/entity here","confidence":"medium"}')
|
|
router = IntentRouterV2(
|
|
classifier=IntentClassifierV2(),
|
|
llm_disambiguator=DocsLlmDisambiguator(llm),
|
|
enable_llm_disambiguation=True,
|
|
)
|
|
|
|
result = router.route("Объясни runtime health", ConversationState(), repo_context())
|
|
|
|
assert result.is_ambiguous is True
|
|
assert result.routing_mode == "llm_disambiguation"
|
|
assert result.llm_router_used is True
|
|
assert result.deterministic_selected_sub_intent
|
|
assert result.llm_router_selected_sub_intent == "ENTITY_EXPLAIN"
|
|
assert result.query_plan.sub_intent == "ENTITY_EXPLAIN"
|
|
|
|
|
|
def test_ambiguous_query_falls_back_to_deterministic_when_llm_fails() -> None:
|
|
llm = FakeLlm("{}", fail=True)
|
|
router = IntentRouterV2(
|
|
classifier=IntentClassifierV2(),
|
|
llm_disambiguator=DocsLlmDisambiguator(llm),
|
|
enable_llm_disambiguation=True,
|
|
)
|
|
|
|
result = router.route("Как работает health check runtime?", ConversationState(), repo_context())
|
|
|
|
assert result.is_ambiguous is True
|
|
assert result.routing_mode == "deterministic_fallback"
|
|
assert result.llm_router_used is False
|
|
assert result.llm_router_error == "llm unavailable"
|
|
assert result.query_plan.sub_intent == result.deterministic_selected_sub_intent
|
|
|
|
|
|
def test_overview_query_stays_in_generic_qa() -> None:
|
|
llm = FakeLlm('{"sub_intent":"GENERIC_QA","reason":"overview query","confidence":"high"}')
|
|
router = IntentRouterV2(
|
|
classifier=IntentClassifierV2(),
|
|
llm_disambiguator=DocsLlmDisambiguator(llm),
|
|
enable_llm_disambiguation=True,
|
|
)
|
|
|
|
result = router.route("Какая структура документации?", ConversationState(), repo_context())
|
|
|
|
assert result.is_ambiguous is False or result.query_plan.sub_intent == "GENERIC_QA"
|
|
assert result.intent == "GENERAL_QA"
|
|
assert result.query_plan.sub_intent == "GENERIC_QA"
|
|
|
|
|
|
def test_llm_disambiguator_rejects_unknown_labels() -> None:
|
|
llm = FakeLlm(json.dumps({"sub_intent": "MADE_UP", "reason": "bad", "confidence": "high"}))
|
|
disambiguator = DocsLlmDisambiguator(llm)
|
|
|
|
assert disambiguator.choose({"query": "test"}) is None
|