From a990e704d970299ee3d23e765aa36e11a5999cca Mon Sep 17 00:00:00 2001 From: zosimovaa Date: Sat, 31 Jan 2026 23:46:08 +0300 Subject: [PATCH] =?UTF-8?q?=D0=91=D0=BE=D1=82=20=D1=80=D0=B0=D0=B1=D0=BE?= =?UTF-8?q?=D1=82=D0=B0=D0=B5=D1=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 16 +- docker-compose.yml | 25 +++ pyproject.toml | 1 + src/rag_agent/__pycache__/cli.cpython-312.pyc | Bin 5976 -> 8404 bytes .../__pycache__/config.cpython-312.pyc | Bin 2932 -> 3438 bytes src/rag_agent/agent/pipeline.py | 76 ++++++++- src/rag_agent/cli.py | 34 +++- src/rag_agent/config.py | 2 +- src/rag_agent/index/postgres.py | 26 +-- src/rag_agent/telegram_bot.py | 156 ++++++++++++++++++ src/rag_agent/webhook.py | 35 ++++ 11 files changed, 350 insertions(+), 21 deletions(-) create mode 100644 src/rag_agent/telegram_bot.py diff --git a/README.md b/README.md index 65b216e..19a794a 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,21 @@ Scripts: `scripts/create_db.py` (Python, uses `ensure_schema` and `RAG_*` env), If `GIGACHAT_CREDENTIALS` is set (e.g. in `.env` for local runs), embeddings use GigaChat API; otherwise the stub client is used. Optional env: `GIGACHAT_EMBEDDINGS_MODEL` (default `Embeddings`), `GIGACHAT_VERIFY_SSL` (`true`/`false`). Ensure `RAG_EMBEDDINGS_DIM` matches the model output (see GigaChat docs). +## Agent (GigaChat) + +Ответы на вопросы формирует агент на базе GigaChat: поиск по базе знаний (RAG) + генерация текста. Если задана переменная `GIGACHAT_CREDENTIALS`, используется `GigaChatLLMClient` в `src/rag_agent/agent/pipeline.py`; иначе — заглушка. Модель чата задаётся через `RAG_LLM_MODEL` (по умолчанию `GigaChat`). + +## Telegram-бот + +Общение с пользователем через бота в Telegram: бот отвечает на текстовые сообщения, используя знания из базы (RAG + GigaChat). + +1. Создайте бота через [@BotFather](https://t.me/BotFather) и получите токен. +2. Добавьте в `.env`: `TELEGRAM_BOT_TOKEN=<токен>`. +3. Запуск: `rag-agent bot` (или `python -m rag_agent.telegram_bot`). +4. Через Docker: `docker compose up -d` поднимает БД, вебхук-сервер и бота в отдельных контейнерах; в `.env` должен быть задан `TELEGRAM_BOT_TOKEN`. + +Требуются: `RAG_DB_DSN`, `RAG_REPO_PATH`, `GIGACHAT_CREDENTIALS`, `TELEGRAM_BOT_TOKEN`. Расширенное логирование (входящие сообщения, число эмбеддингов, число чанков из БД, ответ LLM): `RAG_BOT_VERBOSE_LOGGING=true|false` (по умолчанию `true` для отладки). + ## Notes -- LLM client is still a stub; replace it in `src/rag_agent/agent/pipeline.py` for real answers. - This project requires Postgres with the `pgvector` extension. diff --git a/docker-compose.yml b/docker-compose.yml index 91ab669..d8e2336 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -58,6 +58,31 @@ services: networks: - rag_net + bot: + build: + context: . + dockerfile: Dockerfile + image: rag-agent:latest + container_name: rag-bot + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + environment: + RAG_DB_DSN: "postgresql://${POSTGRES_USER:-rag}:${POSTGRES_PASSWORD:-rag_secret}@postgres:5432/${POSTGRES_DB:-rag}" + RAG_REPO_PATH: "/data" + RAG_EMBEDDINGS_DIM: ${RAG_EMBEDDINGS_DIM:-1024} + GIGACHAT_CREDENTIALS: ${GIGACHAT_CREDENTIALS:-} + GIGACHAT_EMBEDDINGS_MODEL: ${GIGACHAT_EMBEDDINGS_MODEL:-Embeddings} + TELEGRAM_BOT_TOKEN: ${TELEGRAM_BOT_TOKEN:-} + RAG_BOT_VERBOSE_LOGGING: ${RAG_BOT_VERBOSE_LOGGING:-true} + volumes: + - ${RAG_REPO_HOST:-${RAG_REPO_PATH:-./data}}:/data + entrypoint: ["rag-agent"] + command: ["bot"] + networks: + - rag_net + networks: rag_net: driver: bridge diff --git a/pyproject.toml b/pyproject.toml index 9e466d7..21aa367 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ dependencies = [ "gigachat>=0.2.0", "fastapi>=0.115.0", "uvicorn[standard]>=0.32.0", + "python-telegram-bot>=21.0", ] [project.scripts] diff --git a/src/rag_agent/__pycache__/cli.cpython-312.pyc b/src/rag_agent/__pycache__/cli.cpython-312.pyc index be3e16dcc2b119ffe8eb9b1ffe68755dc8eecff5..8e0f885301e1a71cd90dfabc76f2a2cc0d690946 100644 GIT binary patch literal 8404 zcmcgxU2GdycAg=J&m&~?wlv?$$8`6oGMqjZk!uH5eU?p%MoAKD$F^ZQ&}5QCx%$~{oJp&WqH1La;Qy-*H{ouUuQ zePWmBhjPDkKn&a%Vi>}k7-|2*=1Q>4ylLJP3H=J&5mt33^ZA03R5FFUd==WH{EdVn-BJ=$^NDOGFUipD%_tJldZ#m4=(4C@s9>jbDPqJ2n=tk7mLyb3!Z=Xxy!7#p zL76M;EVpxb|XKuR85LqADN{`M{j9m#Wd^Q-h(aC^sp(bUe0=-rP5`)#zf3l zzhY07EhS|-dII)~lR?{5J}D=P1vzs|b;_{yN+u^I^0I1!hMXx&s%<7I&tx-Gs#Tsz z9(m!oYL)V-f+(r>8R?doNlUU4F{@k=l$G?r>s4DZIgctU<3b67@YT_gZ^Qn`Bgw3E zYh*&2OB6|A7W70;%pDn#lOr-ojgVwIkxYZGjDR}Diu0;lQ@S~L5Do;{2fX^>C;tK5 z7MO<9d*|fslOMjgFwyXL-rsd^*Pr#T`t~gQmyfOOJzCp)bamj^qwz=LPqIJGK6-oA zGqG@S-5Izuc6)5;z+c7wBK9Cu?>Ss|9$9mqt~pQFonx!q*d}mhw&@1$3Du3?z>eN$ z6hkxgHLbFWp$+Z#UEdklVH@(tt6~CLUaN1tKh#Ylqd4_`pk-!VdaG6Hv{kcW?D=!J zHl?m|z{%I*px0(Q^jfPl=4XLtf~|hN_B)@k!ZTeuWve8`ieGf6#+;_KHdaYu8*xZR*tn6a(9JisWT9iG;6YaTMTAZpjWv@YPGj@_!P#Nh} zY4zIk6KST(D+i3Rw)}N2vk{%HRkqIa1arrcGNqZ6>8(zNsX8i-*+VTly{506%zhNC zIPRG~3UN%;4%GYv^A@<>s#A&Tw4g_3WyEMXbvg7^W0{Y7I7T^aP}pJhkLaydsp}oQ z;inAqCTMKcS#>Ed=oA$fJa=4gwMytc(Q4^6{q0qE#SODKC^A*gbF}A_@}mCKbM(Sl zu^9Zk74N%ywn*juKS|}OaI8oA! zE93g;^L9izXHc|usp(Ow0p+|gYN`Y(?ux(SSusCEF9sLv;=ggI`|Zx3SGlNBQ43_Q zpr$wZ0u+J26d*E?)^Ss>`ED{1?yAZqOG-gy4)Ky(Cjq`5A;6*kNnp5!I3{fCnc|q_+baC?wFu zs2)+8PL{HOeZcvrW(ayqf`*!)MI|^P)k4zQ!jx*u$kd4tw2-P9-bBy_tAS(|d5Zvy z6aXfOw+K3B)u%x#bbA`E(!my>C6Y}RwN8TOSas4lQotpv9HKD-N0p(Dil9j)Xg4Dr zuzDmdt5yxasT^V?l{avX&seG^I3Y27b`)-^dpk6<>9CDA03BN-14w=v$*mexA_uWy zrz?zNsTSI=I*d2Ts#8a10JUTk7TomCiwsgeshp_6Epi#!X!+zxIQ0w6x;y;gTMHM! zt=>I;|IEEJb?2UiiFM9@cW}wN7LL}!(a*Tpx_4mt*n{BT_WY=)?mf2f@_Ojdg9~fJ z1oqHVonVEIhF6mT*x(s|_$k9$jxKWRd}xVV znprVFn0RRa8GmHM4Bha`(EjC_N7l8`$=c{-z3b9brrYAU?O5a%#dZH+!{wur?yb4@ z(#rmtYyYN=3pf_7|KnhMT@6>T(bco)L=<%T2dCG=Ll3&vA}4E+laG)6W4Io^v^d%D zhF86V>$?v(g1Z{Qp0(hiTJR99#A?A9FzXN8b#6GAuKjDBhijdO8~YCZyK8vq{NlIo z#6OIG>KcaS>2fSOsH`B+-J!ocx6FM!`APNB_4Yq+_P9Em| z*~`KGU+f)|#~s^n7~&%E?(8PIbO;$)2ooV>tg`JDU4 z&pFrom&kr-Zj%(v2$l40s8C6nECW#k+b(Jl5!<1`+HpZ|n>ax+!l|~>Tqac@dDToxd2$Ve zAlKp6ToQsnd)sQ*aWw$QgDw0Hj7d0-wq}9(1s7;=&JQlXd%3~)H{AX^Z`^)EF9P8W zJ7e+vnz2}{&#sq&us;Hn`u~Zr0cn_FopC>RTx3)+Vl9fk+tBDJ2zZzm!Yu?2(O+Ah zA2UBRVzu{J1^pqtKrh9LbTr!Y5J^-x@Z@bhKJ)?#+<8^ZE2f9d zSfpy*g0`*t8=B{+?7xPAxd2HCVY*Pti>CxQO%|n4Oxz$Z0o#bD1pvq)=%C)83}X#} zC&8nwIM*nhp!R54a$3IOX@s>mB{e+_LyLvwuAc zPGOM;2ej(!T@UoF_K!Xa{-o!}J&#&QXFS`wzkk_U_a9iewC?V1@U8~$ zY4G6&AAtHD_icB>6Sy;bd$!^4{=&<9tP2;u2rw4M;{Nwv(T)koY+Qy}ISa;R>oJ){ zbG73VS=eA*Sr801!Qoai(JXQ+7DJ8EKs8lj85Ngv)6vRC%11)y|oZHRFr_+Cp8SNDZ zTe3p3fNMLWed2!cz&2Ui(+zErH6k9|CM(xgRx}p$`Wmtx5{I|QYJtQ;yV)Cwi)|bk zNQ6bNmhIp*qcr1$PysShY0O{da>=}S9n2gjf@M;9B7L`%A&@Gy(3E4jdlEB`0?m~Q zV5|zlYv5hegr;D*87W&V+oDm5?#q`o^C@RbX<vOOBrgCqy?kP-?hN#$8hwn9ET-xRP(1%d)A zsrf#dGZls~K8QlV6^mi$g*6ZZ0wVzzA*0CnkVZNsWefSVCQp+IERZ#sqL!OVLLr4mS^LrCF)%~reFBC9nD@U(n zU>5`;L3k|9C9~VrLC}<{P{b`rW+SL{;FqH}eucKa9KK#C3YP^f9>gb`Y$Sw(%6w5G zv$`@t0FH>ReD?KHUTDHcK?8ZGghE~@mgE_YL$07mz^GUlhECW9I4Gbacp5Y{lgn0E zUpNhA+h~mbmHW;Dfq;F2zNx2#=3oTL5tL?Lg1m0Io04LC3+ai`(b0%%1_WNdjPhM? z!gm3BPJsZJ;+Ktel!cNE-y?+A$1e&)&4Vio*Uw)$fARJ4R}yDmy`H%K>gDrSBM~Pg zy-B*5B=CX5GfvXw`G$${Tr>R+f`7(|LXW3(x=`up86z2 zJ0i`MYVh8M`&*Do2o7vA=Ad=q3WWKsHSslk-3|AlwqiGU-?|$JK!hQTgN({R?e?BEUl}{zO2%6(}ME#;b~e5UF-t#yCR3 zHh{4ef6(ZFy&5r7#Zw(rNAXn1BnwQzQ_Vv`G5pr>k@^LC`zZ{|18_rK?f&p>P*JCU z&DmFT_SK!c;e!sxe=zy(Ve1eANiA03c(p?-ptXjdSl}Mg^0~#=ZVSG8D zAM4mv@Ue0cJ|L3sAu)Hgke5I-Y?MO=)%dov-yA33M~*|PBaxUc!7HRhg5Zh~f;UV- z;3qBxzlmw-Mqeuc*Rp&xEKCl_pBH)MU?A&S0Qpt<0a6e%&S+62SLA3Sww3m!?a#P!0oC8pX9T znl#I~b#4?x#fq7tgrA~mV5fz6`|#n*c*FFGW)a`P8X7FB3AnT9G82g(F?6e`dHae# zohyi?tTaaM0wqQ=GO85h&RF(yX5g2M_ZN)kb0+*>%+Tk|3HblgWMxgiX7J`=*wbH{ z?5yca25y^p`!dL|XW31wnf>Ob>jZm|-JD}htasC4W38LPARE|>jk9on*<@yea4b33 UV);{Z_s=>0eau&JJyf*+1*Jlcod5s; delta 2294 zcmZ`)TWlN06`ffw$>lCd@hOs`_>fdurffZIS*at}YM97^o!W+NHEl)5*ihV+M43w} zvrF3&s3-`URxMn#>U>l{3$$r}TqqwZ^wIV!0)he#iXv8&M$T6114Z-gL_mY2KfSXw z=@w`g*gG?4?%c=hxp&{sJsj}<-s2GwJPZG-mxkKbyix8Ww03lQ1tSG1LfNPJ^L}gY zDhIS+KB$HAAuXH_Ywh`VEs~FD(R@^kgyF^}Nc<1fSs9;2u$OA>F>6=tLHdl}d2+t12|pPX+%!3kzAqHHSybe1 zgb`X6>cV^veIwC9t4{k1!Wwrs!lPv;;Er*0Bg!<4Ow&%CtJ{p;)=FsF+^{t;jP7>u z$jG)doRDT81lExRW3M9gBF79ayNv#piVIW*C>Fwa$jV?_5G1iK>iBM7ou40OceElx z(9+|eyWm;DB3V{TW=~cC9RZ&Wn zSzT628fjyoT{o&^kw|RrEY1}wv#LU%j?&0<5p|&|FBFV9QFj3C>}7I0ncQ~IZf4?x z;MDq8?|RUfT^%u8RBZ z8T!DJy!ZoS{rbHln|*&2&VK9%+n;?Rp->)w_DKK@Phy>2+1ruuL?8clKM(SEso=z* zSi}G~9~VrZBJkmNw@&#|nA;Ahdrvc7-pD=M$gRV(B9wn)y-b~ZE@G*a+S%w(Z(Im=t(Vw?(IWB`5 zxk9f;;w#Y$j56tJrM#Ga%=h$6sjTYIX&r^4mF5*tUlRNK8SDFj>r0OF=W^q(oLF){ z3rKplWTXLw15QPqDb&gaX`W!!!A_VwJF(H2Sr>hy zW75Z1Y}(*ztvkx{)En>Z)mh;)_BTlHp01V33KS)QzXVZW?;5;%oZg8mqw6;}k3Tpk zKj0JZ^DjN*eS3Ujhfl1RH_va6S>Uz*1ShtW7I+0fdbP9eko>P+B0Y3IvHUbMY>Ji_ zxj<%h@&W)^j=The_1Sf3st{HqC|)a+Dr6Rr3QZ-CUG0IHDabN|WjV_?3Y#TjI*4l2 zNQDf-0yY?fmIC0xay4+dpsD&op{SA)Q|2#MD=K&`(H|#MJq#lA>|Bp2%JNJN45_kA z*nfabF%hQ+$sf^IQVA^4pQMs0)&hx#3C0ww-&tI@8V?E3Kc|u_EtWT068i?$^;>sq z<-{_nKwz@`OGH>|R??pfU(%{dt*nlbTY#}r)Kw6VY#8GM)ctqV_9x^$K+%7oqX%g4 z0G<8|y1s+1AD}azasuW)MIb&#XV^4=a1I|jZ1~HE?hZV5I2yu%!+{ux(>5G>_c97ed3V~@f}ti{|1*~G5r7l diff --git a/src/rag_agent/__pycache__/config.cpython-312.pyc b/src/rag_agent/__pycache__/config.cpython-312.pyc index 4b5a21b63185d379b55195f4469ce7e13fa20aa0..188d5a647032074e057352e4f7023146a396949e 100644 GIT binary patch delta 1409 zcma)6O=ufO6rNeFR;yi&B+L4-9Lcd|%l3kCp&@NBuBnq6$4PLE-S(i+E*sCp-gNcT z*_A;m)P?5IzeB`c+G`2DIHv|mk2w}vOoR+}mzW;fLVGDL4g_)tomp9RX)j&aZ@>56 zzW4V1%sxr~F&O7yn00*s5SuV-ta4GD|lBJZ@k_v$U zh(sbgkb44f`;7Zs7~H;{>x%!x|Env-SV)z`aHy#Q&?W#OMN(w5qlE5Yu?F&I-`YibFLf=D!v{ zhBt;8dj_YpARS?h7C4MD@I=eB9WQ9tEYs0MTI7at4imoBbV%72869R9^}~n#@Z62Q zzzh@aFTsYIU@QOtJl=)wu2674k)CFFLH>m!81<&+sP4m1%Iyewl})SfMD4bPvm#F5 ze)oq^{^jHbz^N}`&!x8l_tQK6#saVgw9K-HR_7P>)rFPo`pW$4m+7!OEZ5E_x$@HG zH^r>7sp~pF@&EibI-1hJhyktH*SVoBSE`fp|fuUbBebJWV)_Z}YwG zHMzTy>+x#kmbpP+WtM4n+q>=;;d&}N@~Cj(+d?P%%H!PFqvG5{*vVaVkutZPLJ{c* zprGXMh9-6A8DXy7q4NFmBubn*txzuf=anYvosb)Bz;Drc_EIyisD>a<#QPCM$ik~fLv&+(A{!QjjKOr9-Qaor+w z^ePkiZ_92lI0`@r4?*E)5IzJWhhX3*Q2Z4P{V9QH;_l?9Z||egHX7}q;y#*cqp5v# zu8q#^qnS3E`9S#*s2^SVJl&3*-2=nSMNsdK?ZM1bL79N#M?it&#{nP=90{BahTs)= PoD|_Ad=gE=?qL1`yj?~Z delta 937 zcmZWo%}*0S6rb7MZug_x@=-oSL<*v8K#ocb8Zm0s0}?PX0d-TDfo!rZb#`kkG2z0+ zgAp=!4_rJH6Ac&t0}oz3ZGs-WYe`8ooSZk!#zZISZ{Bz3{odOj1INAD&yHguap>-H<@8b-BLz{7r1td&h8QIDA|mN+M2v>gwlvxoS~5t6nA_TtNgQH{ z+ag(Fi`%X%ZjN8UqdiJ&+>IzJtpqeS7U|O(t*lc18W->r{sG@8V5$;hM${u9nNqTZ zWP?=CxDFpzZgQ@C%9+Bz01WX9r?{`4<~wSieoER8-l+=44u7v*$`1%jYRfAGsPUfMWz;RC@i0mP1X5!{RVm z-<0(sjZFy05dW&ztqvin2;tV=K`A|e`CDU1%@BvbF)lpJl3c=H@3U{y@gxz_r3Yq2Y1*Yzin3ePqPQ>JY`K=n?@MbRxSoX znHR0eErZ!2lVt$10Puedp2<1^x&XQbxW+$}uwigX(t*G@71gQ%mA$&k->M<3M8b(3 zY?O$%x=~s4*V$#^&>=Pa$Q8TBU)fi;I_-?!M0QGNFf~_i6AioAW*c~kbw;Bat1wXh z3)ys%by^n8KW5V+Xw!r-NL@S)Eb`rMCbm3pttN_5&tqaPTSzt;+oTkgR)bO)(vZPe zV?C-@{mME^L9(7ykgFqGmff&^aL9dw6*w_w${VB>&}$5q6~316x`3vNF+M~i-;sHU u3g6J!k&YC5=iC=O(f3O+@cAYZ&{C1o-vqVQo5uHWE3e@qKF;QGo18yXSHmp; diff --git a/src/rag_agent/agent/pipeline.py b/src/rag_agent/agent/pipeline.py index 8a890c7..8a15469 100644 --- a/src/rag_agent/agent/pipeline.py +++ b/src/rag_agent/agent/pipeline.py @@ -1,14 +1,21 @@ from __future__ import annotations +import os from dataclasses import dataclass +from pathlib import Path from typing import Protocol import psycopg +from dotenv import load_dotenv + from rag_agent.config import AppConfig from rag_agent.index.embeddings import EmbeddingClient from rag_agent.retrieval.search import search_similar +_repo_root = Path(__file__).resolve().parent.parent.parent +load_dotenv(_repo_root / ".env") + class LLMClient(Protocol): def generate(self, prompt: str, model: str) -> str: @@ -20,10 +27,49 @@ class StubLLMClient: def generate(self, prompt: str, model: str) -> str: return ( "LLM client is not configured. " - "Replace StubLLMClient with a real implementation." + "Set GIGACHAT_CREDENTIALS in .env for GigaChat answers." ) +class GigaChatLLMClient: + """LLM generation via GigaChat API. Credentials from env GIGACHAT_CREDENTIALS.""" + + def __init__( + self, + credentials: str, + model: str = "GigaChat", + verify_ssl_certs: bool = False, + ) -> None: + self._credentials = credentials.strip() + self._model = model + self._verify_ssl_certs = verify_ssl_certs + + def generate(self, prompt: str, model: str) -> str: + from gigachat import GigaChat + + use_model = model or self._model + with GigaChat( + credentials=self._credentials, + model=use_model, + verify_ssl_certs=self._verify_ssl_certs, + ) as giga: + response = giga.chat(prompt) + return (response.choices[0].message.content or "").strip() + + +def get_llm_client(config: AppConfig) -> LLMClient: + """Return GigaChat LLM client if credentials set, else stub.""" + credentials = os.getenv("GIGACHAT_CREDENTIALS", "").strip() + if credentials: + return GigaChatLLMClient( + credentials=credentials, + model=config.llm_model, + verify_ssl_certs=os.getenv("GIGACHAT_VERIFY_SSL", "false").lower() + in ("1", "true", "yes"), + ) + return StubLLMClient() + + def build_prompt(question: str, contexts: list[str]) -> str: joined = "\n\n".join(contexts) return ( @@ -42,10 +88,32 @@ def answer_query( top_k: int = 5, story_id: int | None = None, ) -> str: - query_embedding = embedding_client.embed_texts([question])[0] + answer, _ = answer_query_with_stats( + conn, config, embedding_client, llm_client, question, top_k, story_id + ) + return answer + + +def answer_query_with_stats( + conn: psycopg.Connection, + config: AppConfig, + embedding_client: EmbeddingClient, + llm_client: LLMClient, + question: str, + top_k: int = 5, + story_id: int | None = None, +) -> tuple[str, dict]: + """Like answer_query but returns (answer, stats) for logging. stats: query_embeddings, chunks_found, answer.""" + query_embeddings = embedding_client.embed_texts([question]) results = search_similar( - conn, query_embedding, top_k=top_k, story_id=story_id + conn, query_embeddings[0], top_k=top_k, story_id=story_id ) contexts = [f"Source: {item.path}\n{item.content}" for item in results] prompt = build_prompt(question, contexts) - return llm_client.generate(prompt, model=config.llm_model) + answer = llm_client.generate(prompt, model=config.llm_model) + stats = { + "query_embeddings": len(query_embeddings), + "chunks_found": len(results), + "answer": answer, + } + return answer, stats diff --git a/src/rag_agent/cli.py b/src/rag_agent/cli.py index 3c8af6b..0a3f718 100644 --- a/src/rag_agent/cli.py +++ b/src/rag_agent/cli.py @@ -25,7 +25,7 @@ from rag_agent.index.postgres import ( update_story_indexed_range, upsert_document, ) -from rag_agent.agent.pipeline import StubLLMClient, answer_query +from rag_agent.agent.pipeline import answer_query, get_llm_client def _file_version(path: Path) -> str: @@ -55,13 +55,24 @@ def cmd_index(args: argparse.Namespace) -> None: existing = filter_existing(changed_files) else: removed = [] - existing = [ - p for p in Path(config.repo_path).rglob("*") if p.is_file() - ] + repo_path = Path(config.repo_path) + if not repo_path.exists(): + raise SystemExit(f"RAG_REPO_PATH does not exist: {config.repo_path}") + existing = [p for p in repo_path.rglob("*") if p.is_file()] + + allowed = list(iter_text_files(existing, config.allowed_extensions)) + print( + f"repo={config.repo_path} all_files={len(existing)} " + f"allowed={len(allowed)} ext={config.allowed_extensions}" + ) + if not allowed: + print("No files to index (check path and extensions .md, .txt, .rst)") + return for path in removed: delete_document(conn, story_id, str(path)) + indexed = 0 for path, text in iter_text_files(existing, config.allowed_extensions): chunks = chunk_text_by_lines( text, @@ -88,11 +99,18 @@ def cmd_index(args: argparse.Namespace) -> None: replace_chunks( conn, document_id, chunks, embeddings, base_chunks=base_chunks ) + indexed += 1 + print(f"Indexed {indexed} documents for story={args.story}") if args.changed: update_story_indexed_range(conn, story_id, base_ref, head_ref) +def cmd_bot(args: argparse.Namespace) -> None: + from rag_agent.telegram_bot import main as bot_main + bot_main() + + def cmd_serve(args: argparse.Namespace) -> None: import uvicorn uvicorn.run( @@ -113,7 +131,7 @@ def cmd_ask(args: argparse.Namespace) -> None: if story_id is None: raise SystemExit(f"Story not found: {args.story}") embedding_client = get_embedding_client(config.embeddings_dim) - llm_client = StubLLMClient() + llm_client = get_llm_client(config) answer = answer_query( conn, config, @@ -185,6 +203,12 @@ def build_parser() -> argparse.ArgumentParser: ) serve_parser.set_defaults(func=cmd_serve) + bot_parser = sub.add_parser( + "bot", + help="Run Telegram bot: answers questions using RAG (requires TELEGRAM_BOT_TOKEN)", + ) + bot_parser.set_defaults(func=cmd_bot) + return parser diff --git a/src/rag_agent/config.py b/src/rag_agent/config.py index e55f7e9..7034efa 100644 --- a/src/rag_agent/config.py +++ b/src/rag_agent/config.py @@ -61,7 +61,7 @@ def load_config() -> AppConfig: chunk_overlap_lines=_env_int("RAG_CHUNK_OVERLAP_LINES", 8), embeddings_dim=_env_int("RAG_EMBEDDINGS_DIM", 1024), # GigaChat Embeddings = 1024; OpenAI = 1536 embeddings_model=os.getenv("RAG_EMBEDDINGS_MODEL", "stub-embeddings"), - llm_model=os.getenv("RAG_LLM_MODEL", "stub-llm"), + llm_model=os.getenv("RAG_LLM_MODEL", "GigaChat"), allowed_extensions=tuple( _env_list("RAG_ALLOWED_EXTENSIONS", [".md", ".txt", ".rst"]) ), diff --git a/src/rag_agent/index/postgres.py b/src/rag_agent/index/postgres.py index 596b9f3..b52d901 100644 --- a/src/rag_agent/index/postgres.py +++ b/src/rag_agent/index/postgres.py @@ -5,6 +5,7 @@ from datetime import datetime, timezone from typing import Iterable, Sequence import psycopg +from pgvector import Vector from pgvector.psycopg import register_vector from rag_agent.ingest.chunker import TextChunk @@ -113,16 +114,20 @@ def ensure_schema(conn: psycopg.Connection, embeddings_dim: int) -> None: except psycopg.ProgrammingError: conn.rollback() pass - try: - cur.execute( - """ + cur.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conrelid = 'chunks'::regclass AND conname = 'chunks_change_type_check' + ) THEN ALTER TABLE chunks ADD CONSTRAINT chunks_change_type_check CHECK (change_type IN ('added', 'modified', 'unchanged')); - """ - ) - except psycopg.ProgrammingError: - conn.rollback() - pass # constraint may already exist + END IF; + END $$; + """ + ) cur.execute( """ CREATE INDEX IF NOT EXISTS idx_documents_story_id @@ -343,6 +348,7 @@ def fetch_similar( top_k: int, story_id: int | None = None, ) -> list[tuple[str, str, float]]: + vec = Vector(query_embedding) with conn.cursor() as cur: if story_id is not None: cur.execute( @@ -354,7 +360,7 @@ def fetch_similar( ORDER BY c.embedding <=> %s LIMIT %s; """, - (query_embedding, story_id, query_embedding, top_k), + (vec, story_id, vec, top_k), ) else: cur.execute( @@ -365,7 +371,7 @@ def fetch_similar( ORDER BY c.embedding <=> %s LIMIT %s; """, - (query_embedding, query_embedding, top_k), + (vec, vec, top_k), ) rows = cur.fetchall() return [(row[0], row[1], row[2]) for row in rows] diff --git a/src/rag_agent/telegram_bot.py b/src/rag_agent/telegram_bot.py new file mode 100644 index 0000000..6bc1ff2 --- /dev/null +++ b/src/rag_agent/telegram_bot.py @@ -0,0 +1,156 @@ +"""Telegram bot: answers user questions using RAG (retrieval + GigaChat).""" + +from __future__ import annotations + +import asyncio +import logging +import os +from pathlib import Path + +from dotenv import load_dotenv + +_repo_root = Path(__file__).resolve().parent.parent +load_dotenv(_repo_root / ".env") + +logger = logging.getLogger(__name__) + +# Расширенное логирование: входящие сообщения, число эмбеддингов, число чанков из БД, ответ LLM. +# Включить/выключить: RAG_BOT_VERBOSE_LOGGING=true|false (по умолчанию true для отладки). +VERBOSE_LOGGING_MAX_ANSWER_CHARS = 500 + + +def _verbose_logging_enabled() -> bool: + return os.getenv("RAG_BOT_VERBOSE_LOGGING", "true").lower() in ("1", "true", "yes") + + +def _run_rag( + question: str, + top_k: int = 5, + story_id: int | None = None, + with_stats: bool = False, +) -> str | tuple[str, dict]: + """Synchronous RAG call: retrieval + LLM. Used from thread. + If with_stats=True, returns (answer, stats); else returns answer only. + """ + from rag_agent.config import load_config + from rag_agent.index.embeddings import get_embedding_client + from rag_agent.index.postgres import connect, ensure_schema + from rag_agent.agent.pipeline import answer_query, answer_query_with_stats, get_llm_client + + config = load_config() + conn = connect(config.db_dsn) + try: + ensure_schema(conn, config.embeddings_dim) + embedding_client = get_embedding_client(config.embeddings_dim) + llm_client = get_llm_client(config) + if with_stats: + return answer_query_with_stats( + conn, + config, + embedding_client, + llm_client, + question, + top_k=top_k, + story_id=story_id, + ) + return answer_query( + conn, + config, + embedding_client, + llm_client, + question, + top_k=top_k, + story_id=story_id, + ) + finally: + conn.close() + + +def run_bot() -> None: + token = os.getenv("TELEGRAM_BOT_TOKEN", "").strip() + if not token: + logger.error( + "TELEGRAM_BOT_TOKEN is required. Set it in .env or environment. " + "Container will stay up; restart after setting the token." + ) + import time + while True: + time.sleep(3600) + + from telegram import Update + from telegram.ext import Application, ContextTypes, MessageHandler, filters + + verbose = _verbose_logging_enabled() + + async def handle_message(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: + if not update.message or not update.message.text: + return + question = update.message.text.strip() + if not question: + await update.message.reply_text("Напишите вопрос текстом.") + return + user_id = update.effective_user.id if update.effective_user else None + chat_id = update.effective_chat.id if update.effective_chat else None + if verbose: + logger.info( + "received message user_id=%s chat_id=%s text=%s", + user_id, + chat_id, + repr(question[:200] + ("…" if len(question) > 200 else "")), + ) + try: + loop = asyncio.get_event_loop() + result = await loop.run_in_executor( + None, + lambda: _run_rag( + question, + top_k=5, + story_id=None, + with_stats=verbose, + ), + ) + if verbose: + answer, stats = result + logger.info( + "query_embeddings=%s chunks_found=%s", + stats["query_embeddings"], + stats["chunks_found"], + ) + answer_preview = stats["answer"] + if len(answer_preview) > VERBOSE_LOGGING_MAX_ANSWER_CHARS: + answer_preview = ( + answer_preview[:VERBOSE_LOGGING_MAX_ANSWER_CHARS] + "…" + ) + logger.info("llm_response=%s", repr(answer_preview)) + else: + answer = result + if len(answer) > 4096: + answer = answer[:4090] + "\n…" + await update.message.reply_text(answer) + except Exception as e: + logger.exception("RAG error") + await update.message.reply_text( + f"Не удалось получить ответ: {e!s}. " + "Проверьте RAG_DB_DSN и GIGACHAT_CREDENTIALS." + ) + + app = Application.builder().token(token).build() + app.add_handler(MessageHandler(filters.TEXT & ~filters.COMMAND, handle_message)) + logger.info("Telegram bot started (polling)") + app.run_polling(drop_pending_updates=True) + + +def main() -> None: + logging.basicConfig( + format="%(asctime)s %(levelname)s %(name)s %(message)s", + level=logging.INFO, + ) + # Убрать из лога пустые HTTP-ответы polling (без сообщений от пользователя) + for name in ("telegram", "httpx", "httpcore"): + logging.getLogger(name).setLevel(logging.WARNING) + try: + run_bot() + except KeyboardInterrupt: + pass + except ValueError as e: + raise SystemExit(e) from e diff --git a/src/rag_agent/webhook.py b/src/rag_agent/webhook.py index 6a35fc1..8166be9 100644 --- a/src/rag_agent/webhook.py +++ b/src/rag_agent/webhook.py @@ -34,6 +34,14 @@ def _branch_from_ref(ref: str) -> str | None: return ref.removeprefix("refs/heads/") +# GitHub/GitLab send null SHA as "before" when a branch is first created. +_NULL_SHA = "0000000000000000000000000000000000000000" + + +def _is_null_sha(sha: str | None) -> bool: + return sha is not None and sha == _NULL_SHA + + def _verify_github_signature(body: bytes, secret: str, signature_header: str | None) -> bool: if not secret or not signature_header or not signature_header.startswith("sha256="): return not secret @@ -118,9 +126,36 @@ def _pull_and_index( logger.warning("git checkout %s failed: %s", branch, _decode_stderr(e.stderr)) return + # Branch deletion: after is null SHA → nothing to index + if _is_null_sha(payload_after): + logger.info("webhook: branch deletion detected (after is null SHA) for branch=%s; skipping index", branch) + return + # Prefer commit range from webhook payload (GitHub/GitLab before/after) so we index every push # even when the clone is the same dir as the one that was pushed from (HEAD already at new commit). if payload_before and payload_after and payload_before != payload_after: + # Update working tree to new commits (fetch only fetches refs; index reads files from disk) + origin_ref = f"origin/{branch}" + merge_proc = subprocess.run( + ["git", "-C", repo_path, "merge", "--ff-only", origin_ref], + capture_output=True, + text=True, + timeout=60, + ) + if merge_proc.returncode != 0: + logger.warning( + "webhook: git merge --ff-only failed (branch=%s); stderr=%s", + branch, _decode_stderr(merge_proc.stderr), + ) + return + # New branch: before is null SHA → use auto (merge-base with default branch) + if _is_null_sha(payload_before): + logger.info( + "webhook: new branch detected (before is null SHA), using --base-ref auto story=%s head=%s", + branch, payload_after, + ) + _run_index(repo_path, story=branch, base_ref="auto", head_ref=payload_after) + return logger.info( "webhook: running index from payload story=%s %s..%s", branch, payload_before, payload_after,