h!ddlmZddlZddlZddlmZddlmZmZm Z ddl Z ejjddZeejjddZeejjd d Zeejjd d Zd ZdedededeefdZdZdeedeeefdZeGddZdedeeeeffdZdeefdededededededeefd Zd!Zd"eedefd#ZdPdedede ededef d$Zefd%eded&edefd'Z dQd(edededefd)Z!d*Z"d%ed+edefd,Z#e$d-k(rddl%Z%e%jLd./Z'e'jQd0ed1d23e'jQd4ed56e'jQd7ed86e'jQd9ed:6e'jSZ*e*jVr>e,d;e*jVee*jVe*jZ<Z.e,d=e.e*j^r>e,d>e*j^e!e*j^e*jZ<Z.e,d?e.e*j`re,d@e*jZdAe*j`e e*j`e*jZeBZ1e,dCe1jdDgD]Z2e,dEe2dFdGdHe2dIdJe2dKe,dLe,e1jdMdNddOyyy)R)PathN) dataclass)ListTupleOptionalOPENAI_EMBED_MODELztext-embedding-3-smallRAG_CHUNK_SIZE500RAG_CHUNK_OVERLAP80 RAG_TOP_K3c,ddl}|jdS)Nr cl100k_base)tiktoken get_encoding)rs /var/www/html/dynbot/rag_db.py_get_tokenizerrs   //text max_tokensoverlapreturnct}|j|}g}d}t|}||krNt||z|}|||} |j |j | ||k(r |S||z }|dkrd}||krN|S)Nr)rencodelenminappenddecode) rrrenctokenschunksstartnend chunk_tokenss r_split_by_tokensr's  C ZZ F F E F A !)%*$a(eC(  cjj./ !8  Mg  19E !) MrcTtj|tj}tj|tj}|tjj |dddzz }|tjj |dddzz }||j zS)N)dtypeT)axiskeepdimsg-q=)npasarrayfloat32linalgnormT)aba_normb_norms r_cosine_sim_matrixr7-s} 1BJJ'A 1BJJ'A "))..T.:UB CF "))..T.:UB CF FHH rtextscddlm}|}|jjt|}|j Dcgc]}|j }}|Scc}w)Nr)OpenAI)modelinput)openair: embeddingscreate EMBED_MODELdata embedding)r8r:clientrespdvectorss r _embed_textsrG5sL XF    # #+U # CD$(II.qq{{.G. N/sAcZeZdZUeed<eed<eed<dZeeed<dZee e ed<y)RAGChunk tenant_idtitlecontentNtagsrB) __name__ __module__ __qualname__int__annotations__strrMrrBrfloatrrrIrI=s2N J LD(3-'+IxU $+rrIpdf_pathcddl}g}|j|5}t|D]/\}}|jd}|s|j |dz|f1 ddd|S#1swY|SxYw)Nrrr*)fitzopen enumerateget_textr)rVrXpagesdocipagers rextract_pdf_textr`Fsr E 8 , ~ ,GAt==(D a!eT]+ ,, L , Ls%A!A!!A+pdf,ragrJ base_titlerMchunk_size_tokensoverlap_tokensc g}t|}|D]R\}} t| ||} t| D]2\} } |d|dd| dzd} |jt || | |4T|S)Nz :: p02dz :: cr*)rJrKrLrM)r`r'rZrrI)rVrJrbrMrcrdr" page_textspage_norpiecesidxpiecerKs rmake_chunks_from_pdfrlQs  F!(+J#` !$(9>J#F+ `JC!l%}E#a%EE MM(YeUY]^ _ `` Mrc ddlm} ddlm}|| t d||fS#t$rd}Y'wxYw#t$rd}Y1wxYw)Nr) SessionLocal)Documentz\Could not import SessionLocal or Document. Adjust `_import_models()` to your project layout.)apprn Exceptionmodelsro ImportError)rnros r_import_modelsrtasg$#x/ j    !!   s"3 00 AAr"c >t\}}|} d}d}tdt||D]}||||z}|Dcgc]}|j} }t | } t || D]e\}} t j| } ||jd|j|j| |jxsd} |j| g|j|t|z }||jScc}w#|jwxYw)Nrra)rJcase_idrKrLrBrM)rtrangerrLrGzipjsondumpsrJrKrMaddcommitclose)r"rnrosessionBATCH rows_writtenr^batchcr8rFvecemb_jsonr]s rupsert_chunks_to_dbrqs+-L(nG q#f+u- 'A1QuW%E(-.1QYY.E."5)GeW- !3::c?kk ''II&,9  C  ! NN  CJ &L# '$ %/$  s)D DB"D D Dc~t|}|t|j}t||||}t |}|S)NrJrbrM)rSrstemrlr)rVrJrbrMr"writtens rpreprocess_pdf_to_dbrs@8}H(^(( !(iJ]a bF!&)G Nr user_querytop_kc vt\}}|} |j|j|j|k(|jj dj }|sdg|dd|jSggggf\}}} } |D]} tj| j} |j| |j| j| j| j| j| j|sdg|dd|jSt|gd} t!| g|d}t#j$| d|}g}t'|dD]:\}}|j|t)| || |t+||||d<g}|D]3}|jd |d d |d d |dj-5dj/|}|||d|jS#t$rYwxYw#|jwxYw)Nz&No embedded documents for this tenant.)contextmatchesrwarningzNo valid embeddings parsed.rr*)r#)rank document_idrK similarityrL[rz] rK rLz --- )rrr)rtqueryfilterrJrBisnotallr~rzloadsrrLrKidrqrGr7r-argsortrZrQrTstripjoin)rrJrrnrordocsr>r8titlesidsrEr query_vecsimstop_idxrrrj context_partsmrs rquery_rag_context_from_dbrsA+-L(nG-}}X&--   ) +    $ $T *  #% !b5MuvL  I*,RR& E63 A jj-!!#& QYY' agg& 144  !b5Mjk0  -!*.q1 !9+z:1=**dU#FU+"7!4 ID# NN"3s8}#DI. :     XA  1QvYKr!G*R) @R@R@T?U!V W X$$]3"wG 9  8  s>AH&H&BH H&1CH& H#H&"H##H&&H8folderct|}|jstd|d}t|j dD]'}|t t |||j|z })|S)NzFolder not found: rz*.pdfr)rexistsFileNotFoundErrorsortedglobrrSr)rrJrMptotalpdfs rpreprocess_folder_pdfs_to_dbrsq V A 88:"4VH =>> EaffWo&e %c#h)PSPXPX_cdde LrubDu bist ein präziser Assistent. Nutze den folgenden Kontext aus den House of PM Best-Practices, falls relevant, um die Nutzerfrage zu beantworten. Wenn der Kontext nicht passt, antworte ohne ihn zu zitieren. Zitiere kurz in Klammern die Quelle [Titel] bei Verwendung. KONTEXT: {context} NUTZERFRAGE: {question} ANTWORT (prägnant, fachlich, deutsch): rag_contextc8tj|xsd|S)Nz"(kein relevanter Kontext gefunden))rquestion)RAG_PROMPT_TEMPLATEformat)rrs rbuild_rag_augmented_promptrs  % %k.a=alv % wwr__main__z1Preprocess PDFs into DB and/or query RAG context.) descriptionz--tenantr*z Tenant ID)typedefaulthelpz--pdfz"Path to a single PDF to preprocess)rrz--folderz(Path to a folder with PDFs to preprocessz--queryzQuery to test retrievalzPreprocessing PDF: )rJzStored chunks: zPreprocessing folder: zStored chunks (all PDFs): zQuerying RAG for tenant z: )rJrz Top matches:rz- (rz.3fz) rKz id=rz --- Context preview: rri)Nra)ra)3pathlibrrzos dataclassesrtypingrrrnumpyr-environgetr@rQCHUNK_SIZE_TOKENSCHUNK_OVERLAP_TOKENS DEFAULT_TOP_KrrSr'r7rTrGrIr`rlrtrrdictrrrrrNargparseArgumentParserparser add_argument parse_argsargsrprinttenantcntrrresrrUrrrs !((jjnn13KL  '7?@2::>>*=tDEBJJNN;45 0 3C#$s)$S d4;&7 ,, ,stE#s(O'<&/2C/C 3 $' %(  # -0  *-  IMX  " X3:33HSM`ctwMZ0#0#0c0^b0fCX[ +x3xSxSx z $X $ $1d eF  akJ c0TU  3]^  2KL    D xx #DHH:./"488t{{C u%& {{ &t{{m45*4;;$++N *3%01 zz ( R |DE' dkkQ^_ nB' TA C,,Bqzl%-@P?QR S T )* cggi$Ud+,'r