o oh@sddlZddlZddlZddlZddlZddlZddlmZddlmZddl m Z ddl m Z ddl mZddlmZdZd d ZGd d d ZGd ddZddZddZedkrdeedSdS)N)tqdm)OllamaEmbeddings)FAISS)ChatGroq)PromptTemplate) ChatOllama8gsk_Q6G2mqkFL74aSdlDkU3OWGdyb3FYBdtLlFxhe78b1g17n1Ew181wcCstdd}d|d|}tj|r:ddt|D}|r0tj||dd}td ||Std |d dStd |d |d dS)Nznomic-embed-text)modelzmy_embeddings_video//cSsg|] }|dr|qS))z.faissz.pkl)endswith).0frJ/var/www/eduai.edurigo.com/question_generate/testing/generate_mcq_video.py sz'load_all_embeddings..T)allow_dangerous_deserializationzEmbeddings loaded from zNo embedding files found in .z"No embeddings found for client_id z and reference_id )rospathisdirlistdirr load_localprint) client_id reference_id embeddings reference_dirembedding_filesvectorsrrrload_all_embeddingss  rc@s<eZdZddZddZddZddZd d Zd d Zd S)QuestionHistorycCs"d|d|d|_||_dS)Nzquestion_history_video/r z.json) file_pathload questions)selfrrrrr__init__ szQuestionHistory.__init__cCstj|jrHz*t|jd}|}Wdn1swY|r.tt |WStWStj yGt d|jdtYSwt dtS)NrzWarning: Invalid JSON in z!. Starting with an empty history.z6No history file found. Starting with an empty history.) rrexistsr!openreadstripsetjsonloadsJSONDecodeErrorr)r$r contentrrrr"$s zQuestionHistory.loadcCs\tjtj|jddt|jd}tt|j |WddS1s'wYdS)NT)exist_okw) rmakedirsrdirnamer!r(r,dumplistr#)r$r rrrsave1s"zQuestionHistory.savecCs|j|dSN)r#addr$questionrrrr86szQuestionHistory.addcCs ||jvSr7)r#r9rrr __contains__9 zQuestionHistory.__contains__cCs t|jSr7)lenr#)r$rrr__len__<r<zQuestionHistory.__len__N) __name__ __module__ __qualname__r%r"r6r8r;r>rrrrr s  r c@seZdZddZddZdS) RateLimitercCs||_||_g|_dSr7) max_requestsperiodrequests)r$rCrDrrrr%@s zRateLimiter.__init__csrtfddjD_tjjkr/jjd}|dkr/t|IdHjtdS)Ncsg|] }|jkr|qSr)rD)r reqnowr$rrrGsz$RateLimiter.wait..r)timerEr=rCrDasynciosleepappend)r$ sleep_timerrGrwaitEszRateLimiter.waitN)r?r@rAr%rNrrrrrB?s rBc sZt||}|stdgddfSt|||jddid}|dkr)tdtdntdd d td d gd dt dtj g}d}d}t dddfdd} t |dd} t d} d} || IdH} dd| D}t||krt | kr|t|}t|d}t|}zH| |||IdH}|D]5}t||kr|dvr|||d|d7}|t|dtdd|d D7}q| dWnty}z td!|WYd}~qrd}~wwt||krt | ks~t||krtd"t|d#|d$|t|fWdS1s&wYdS)%Nz=No embeddings found for the given client ID and reference ID.rk ) search_kwargszllama3-70b-8192) model_name groq_api_keyzhttp://127.0.0.1:11434z llama3:8b)base_urlr context num_questionsa: {context} Generate {num_questions} multiple-choice questions based on the information provided above. Each question should have one correct option and three incorrect options. Use the following format: Q: A) B) C) D) Correct: Ensure questions are concise and directly related to the content. Try to cover different aspects of the provided information. Do not include phrases like "in the document", "according to the text", or any other references to the source material in the questions. )input_variablestemplatezFQ: (.+?)\nA\) (.+?)\nB\) (.+?)\nC\) (.+?)\nD\) (.+?)\nCorrect: ([A-D])<)rCrDc sIdHj||d}|gIdH}|j}g}|D]G}|\}} } } } } |}tjdd|tj d}tjdd|tj d}|vri| |t ||| | | | d| dq"|S)N)rUrVze\b(in|from|according to|mentioned in|stated in|as per) (the|this)? ?(document|text|passage|content)\b)flagszQ^(The|This) (document|text|passage|content) (states|mentions|says|indicates) that)ABCD)numberr:optionscorrect_answer) rNformatainvoker/finditergroupsr*resub IGNORECASErLr=)rU start_number batch_sizepromptresponse response_strr#matchr:option_aoption_boption_coption_dcorrect_letterllm mcq_regex mcq_templatequestion_history rate_limiterrrprocess_contextts0  z+generate_mcq_video..process_contextzGenerating MCQs)totaldesci,z6Provide a comprehensive overview of the entire contentcSsg|]}|jqSr) page_content)r resultrrrrsz&generate_mcq_video..r:css|] }t|VqdSr7)r=split)r optionrrr sz%generate_mcq_video..rczError processing context: zWarning: Only generated z unique questions out of z requested.)rrr as_retrieverr GROQ_API_KEYrrricompileDOTALLrBrrIrfr=minrandomchoicerLr8rsumvaluesupdate Exceptionr6)rrVrGPUr retrievermcqsquestion_numbertotal_token_countr}pbartimeoutqueryresultscontextsremaining_questionsrmrUnew_mcqsmcqerrwrgenerate_mcq_videoNsj         .  &rc sd}d}d}t|||IdH\}}}|stddStd|td||D].}td|dd|d |d D] \}}t|d |qAtd |d dq+dS)Nrrz'No multiple-choice questions generated.zTotal questions in history: zTotal token count: z Question rbz: r:rcz) zCorrect Answer: rd )rritems) rrVrr history_count token_countrrtextrrrmains r__main__)rrr,rJrirIrlangchain_community.embeddingsr langchain_community.vectorstoresrlangchain_groqrlangchain.promptsrlangchain_ollamarrrr rBrrr?runrrrrs*       c