o
    /f                     @   s~  d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZm	Z	 d dl
mZ d dlZd dlmZ d dlZd dlZd dlZd dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d
efddZed edZe	dZ edZ!e	dZ"edZ#e $dZ%e Z&e'd e'd e(ej)* rdndZ(dd Z+dd Z,dd Z-dd Z.dd Z/d d! Z0d"d# Z1d'd%d&Z2dS )(    N)T5ForConditionalGenerationT5Tokenizer)wrap)	stopwords)KeywordProcessor)	Sense2Vec)cosine_similarity)SentenceTransformer)NormalizedLevenshteinseedc                 C   s0   t |  tj |  t|  tj|  d S )N)randomr   nptorchmanual_seedcudamanual_seed_all)r    r   B/var/www/chatrigo.edurigo1.com/get_recommendations/testing/main.pyset_seed   s   

r   *   zt5-basezramsrigouthamg/t5_squad_v1zmsmarco-distilbert-base-v3s2v_oldpunktr   r   cpuc                 C   s,   d}t | D ]}| }|d | }q|S )N  )nltksent_tokenize
capitalize)contentfinalsentr   r   r   postprocesstext*   s
   r!   c              
      s   |   dd} d|  } d} j| |dddd|}|d	 |d
 }}|j||ddddddd} fdd|D }	|	d }
t|
}
|
  }
|
S )N
r   zsummarize: i   FTpt
max_lengthpad_to_max_length
truncationreturn_tensors	input_idsattention_mask         K   i,  )r)   r*   early_stopping	num_beamsnum_return_sequencesno_repeat_ngram_size
min_lengthr%   c                       g | ]	} j |d dqS T)skip_special_tokensdecode.0ids	tokenizerr   r   
<listcomp>8       zsummarizer.<locals>.<listcomp>r   )stripreplaceencode_plustogenerater!   )textmodelr=   devicemax_lenencodingr)   r*   outsdecsummaryr   r<   r   
summarizer1   s   rM   c           
         s   t j }|j| dd ddh}ttj}|g d7 }|td7 }|j	|d |j
dd	d
d |jdd}dd |D }t }|D ]}|| qB||  fdd|D }	|	d d S )Nen)inputlanguagePROPNNOUN)z-lrb-z-rrb-z-lcb-z-rcb-z-lsb-z-rsb-english)posg?g      ?average)alpha	thresholdmethod   nc                 S   s   g | ]}|d  qS r   r   r:   valr   r   r   r>   H       z get_keywords.<locals>.<listcomp>c                    s   g | ]}| v r|qS r   r   )r:   keywordkeywords_foundr   r   r>   M   s       )pkeunsupervisedMultipartiteRankload_documentliststringpunctuationr   wordscandidate_selectioncandidate_weighting
get_n_bestr   add_keywordextract_keywords)
originaltextsummarytext	extractorrT   stoplist
keyphraseskeywordskeyword_processorr`   important_keywordsr   ra   r   get_keywords>   s    


ry   c              	      sz   d | |} j|dddddt}|d |d }}|j||dd	d
ddd} fdd|D }	|	d dd }
|
S )Nzcontext: {} answer: {}i  FTr#   r$   r)   r*      r,   r-   H   )r)   r*   r/   r0   r1   r2   r%   c                    r4   r5   r7   r9   r<   r   r   r>   U   r?   z get_question.<locals>.<listcomp>r   z	question:r   )formatrB   rC   rG   rD   rA   r@   )contextanswerrF   r=   rE   rI   r)   r*   rJ   rK   questionr   r<   r   get_questionP   s   r   c           
         s   g }z|j | g dd |j |d} fdd|D }W n   g }Y d}| g}| }|D ]}	t|d  |	 |k rL|	|vrL|	|vrL||	 q/|dd  S )	N)rR   PERSONPRODUCTLOCORGEVENTNORPzWORK OF ARTFACGPENUMFACILITY)sensesrZ   c                    sN   g | ]#}|d   dd   dd kr|d   dd  dd  qS )r   |r,   _r   )splitrA   titler@   r]   senser   r   r>   ^   s   N z'sense2vec_get_words.<locals>.<listcomp>g333333?r   r,   )get_best_sensemost_similarr   normalized_levenshtein
similaritylowerappend)
words2vtopnr   outputr   rW   r   	checklistxr   r   r   sense2vec_get_wordsY   s   ,
r   c                    s   t || }t |}t|g  fddttD }t|d D ]8}||d d f }	tj|| d d  f dd}
||	 d| |
dd  }|t| } | || q"fdd D S )Nc                    s   g | ]
}| d  kr|qS r\   r   )r:   i)keywords_idxr   r   r>   m   s    zmmr.<locals>.<listcomp>r,   )axisc                    s   g | ]} | qS r   r   )r:   idx)rk   r   r   r>   u   r_   )	r   r   argmaxrangelenmaxreshaper   remove)doc_embeddingword_embeddingsrk   top_nlambda_paramword_doc_similarityword_similaritycandidates_idxr   candidate_similaritiestarget_similaritiesmmrmmr_idxr   )r   rk   r   r   i   s   

r   c                 C   s   t | |||}t|dkr|S |  g}|| |d |   }||g}	||}
tt|d}t|	|
|||}|  g}|D ]}| |  krS||  qB|dd  }|S )Nr   r   rz   r,   )	r   r   r   extendencodeminr   r   r   )r   origsentencesense2vecmodelsentencemodelr   	lambdavaldistractorsdistractors_newembedding_sentencekeyword_embeddingdistractor_embeddingsmax_keywordsfiltered_keywordsr   wrdr   r   r   get_distractorsw   s"   



r   r+   c              	   C   s   t | ttt}t| |}g }|D ]6}t|D ]/}t||tt}t	||t
tdd}|d d }	| g|	 }
t|
 ||| |
d qq|S )N(   g?r+   )r   correct_answeroptions)rM   summary_modelsummary_tokenizerrG   ry   r   r   question_modelquestion_tokenizerr   r   sentence_transformer_modelr   r   shuffler   )r}   num_questions_per_keywordsummary_textrv   r   r~   r   r   r   selected_distractorsr   r   r   r   generate_question   s   

r   )r+   )3ossysjsonr   r   numpyr   transformersr   r   	textwrap3r   r   nltk.corpusr   ri   rd   	traceback	flashtextr   	sense2vecr   sklearn.metrics.pairwiser   sentence_transformersr	   !similarity.normalized_levenshteinr
   intr   from_pretrainedr   r   r   r   r   	from_diskr   r   downloadrG   r   is_availabler!   rM   ry   r   r   r   r   r   r   r   r   r   <module>   sL    





	