o
    0eIj                     @   sL   d Z ddlZddlZddlmZ ddlmZ edZdd Z	dd	d
Z
dS )zNLocal keyword retrieval over data/regulations + data/certs (offline fallback).    N)Path)configz	[a-z0-9]+c            
   
   C   s   g } t tjd dD ]-}| }ttd|D ]\}}| }t	|dkr8| 
|j d| ||jd qqt tjd dD ]7}t| }| D ])\}}t|trz|D ]}	| 
|j d	|	d
pm|	d t|	|jd q\qQqD| S )Nregulationsz*.mdz\n## (   #)idtextsourcecertsz*.json:certNotcNo)sortedr   DATAglob	read_text	enumerateresplitstriplenappendstemnamejsonloadsitems
isinstancelistgetdumps)
chunksmdr   ipartcjobjkeyarrrec r*   //opt/greenlight/greenlight/sources/local_rag.pybuild_corpus   s,    

r,      c                    s   |pt  }tt|  }g }|D ]}tt|d   t fdd|D }|||f q|jdd d |rNdd |d | D pM|d	 d
 gS g S )Nr   c                 3   s    | ]	}| v rd V  qdS )   Nr*   ).0wtoksr*   r+   	<genexpr>&   s    zsearch.<locals>.<genexpr>c                 S   s
   | d  S )Nr   r*   )xr*   r*   r+   <lambda>(   s   
 zsearch.<locals>.<lambda>)r'   c                 S   s   g | ]
\}}|d kr|qS )r   r*   )r/   _chr*   r*   r+   
<listcomp>)   s    zsearch.<locals>.<listcomp>r   r.   )r,   set_WORDfindalllowersumr   sort)querycorpuskqscoredr7   overlapr*   r1   r+   search    s   
,rE   )Nr-   )__doc__r   r   pathlibr   
greenlightr   compiler:   r,   rE   r*   r*   r*   r+   <module>   s    
