o
    /f                     @   s   d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZ d dl	m
Z
 d dlmZ d dlmZ d dlZd dlZd dlZd dlZd dlZd dlmZ d dlZed	 ed
 e dZe Zdd Zdd Zdd Zdd Zdd Zdd Z dd Z!dS )    N)	stopwordssent_tokenize)KeywordProcessor)max_similarity)adapted_lesk)wordnet)WordNetLemmatizerpunktr   en_core_web_smc                 C   s^   t j }|j| dd dh}|j|d |  g }|jdd}|D ]	}||d  q#|S )Nen)inputlanguagePROPNpos   )nr   )pkeunsupervisedTextRankload_documentcandidate_selectioncandidate_weighting
get_n_bestappend)article	extractorr   resultexeach r!   N/var/www/chatrigo.edurigo1.com/get_recommendations/testing/main_version_2_2.pyget_important_words   s   
r#   c                 C   s   t | }dd |D S )Nc                 S   s    g | ]}t |d kr| qS )   )lenstrip).0sentr!   r!   r"   
<listcomp>&        z+split_text_to_sentences.<locals>.<listcomp>r   )r   	sentencesr!   r!   r"   split_text_to_sentences$   s   r,   c                 C   sf   t  }dd | D }| D ]}|| q|D ]}||}|D ]	}|| | qqdd | D S )Nc                 S   s   i | ]}|g qS r!   r!   )r'   wordr!   r!   r"   
<dictcomp>*   s    z!map_sentences.<locals>.<dictcomp>c                 S   s    i | ]\}}|t |td dqS )T)keyreverse)sortedr%   )r'   r/   valuer!   r!   r"   r.   1   r*   )r   add_keywordextract_keywordsr   items)important_wordsr+   	processorkey_sentencesr-   r(   foundr!   r!   r"   map_sentences(   s   
r:   c                 C   s   g }t j|  t jd}|D ],}| |  }|D ]}| d dd }|| kr9||v r9||vr9|	| qq|d d S )Nr   r   _    )
wnsynsetslowerNOUNhyponyms	hypernymslemma_namesreplace
capitalizer   )keyword	imp_wordsdistractorsr?   synset
hyper_hypoconceptnamer!   r!   r"   generate_distractors3   s   
rN   c                    sX  g }|   D ]\}|sqt|td}tjdtd|tjd}t| z&t	
}|jd d }t|}dd |jD }	  fdd|	D 7  W n   Y fd	d D  d
d  D   g  }
tt|
d d }
t|
dk rt|}| |
vr|
|  t|
dk szt|
 |
|
  }||||
d q|S )N)r/   z\b{}\b________)flagsi  c                 S   s&   g | ]}|j d ks|j dkr|jqS )PERSONORG)label_textr'   entr!   r!   r"   r)   P   s   & z&generate_questions.<locals>.<listcomp>c                    s    g | ]}| vr|kr|qS r!   r!   rU   rI   rG   r!   r"   r)   Q   r*   c                    s    g | ]}|     kr|qS r!   )r@   r'   
distractor)rG   r!   r"   r)   T   r*   c                 S   s   g | ]}t |qS r!   )
lemmatizer	lemmatizerX   r!   r!   r"   r)   U   s       )questioncorrect_answeroptions)r5   maxr%   resubformatescape
IGNORECASErN   	wikipediapagecontentnlpentsrF   listsetrandomchoicer   shuffleindex)mapped_sentsr6   	questionsr+   r(   r]   	wiki_pagesummarydocentitiesr_   new_distractorr^   r!   rW   r"   generate_questions>   s@    




rx   c                 C   s(   t dd| } t dd| } |  } | S )Nz[^\w\s] z\s+r<   )ra   rb   r&   )r]   r!   r!   r"   preprocess_questiong   s   rz   c           	      C   s   t | }t| }t||}t||}g }t }|D ]}|d }t|}||vr1|| || qt|dkr>t	
|d}|S )Nr]   
   )r#   r,   r:   rx   rl   rz   addr   r%   rm   sample)	article_textrH   sentsrq   rr   unique_questionsseen_questionsr]   question_strr!   r!   r"   process_articlep   s    



r   )"spacyr   stringnltk.corpusr   nltk.tokenizer   	flashtextr   pywsd.similarityr   
pywsd.leskr   r   r>   requestsjsonra   rm   rf   	nltk.stemr	   nltkdownloadloadri   rZ   r#   r,   r:   rN   rx   rz   r   r!   r!   r!   r"   <module>   s6    


)	