from http.client import HTTPException from tempfile import NamedTemporaryFile import tempfile import traceback from typing import List from fastapi import FastAPI, UploadFile, File, Form, Body from fastapi.responses import JSONResponse from langchain.docstore.document import Document from langchain.text_splitter import RecursiveCharacterTextSplitter from pydantic import BaseModel from langchain_community.embeddings import OllamaEmbeddings import uvicorn #from moviepy import VideoFileClip from pydub import AudioSegment from fastapi.datastructures import UploadFile from fastapi import HTTPException from io import BytesIO import subprocess import os import asyncio import requests app = FastAPI() from training import load_pdf, split_text_with_semantic_chunker, count_tokens_in_documents, save_documents_to_txt, create_and_save_embeddings, merge_all_faiss @app.get('/helloEdurigoAiChatAndGenQuestionsTesting') def hello(): return {'message': 'Hello from edurigo_ai testing env'} """ TRAINING ( CREATING EMBBEDDINGS ) FOR LLAMA-3 MODEL """ # from training import load_and_split_document, create_embeddings, count_tokens class TrainResponse(BaseModel): message: str token_count: int class ErrorResponse(BaseModel): error: str def fetch_document_with_curl(url): try: # Use subprocess to call curl and fetch the document content result = subprocess.run(["curl", "-k", "-s", url], capture_output=True, check=True) return result.stdout except subprocess.CalledProcessError as e: raise HTTPException(status_code=400, detail=f"Error fetching document: {e}") @app.post("/train_with_document_edurigo_ai", response_model=TrainResponse, responses={400: {"model": ErrorResponse}, 500: {"model": ErrorResponse}}) def process_document(client_id: str = Form(...), document_url: str = Form(...), reference_id: str = Form(...)): # Check if document is in supported format file_extension = os.path.splitext(document_url)[1].lower() if file_extension not in ['.pdf', '.txt', '.docx', '.doc', '.pptx', '.ppt']: raise HTTPException(status_code=400, detail="Unsupported file type. Only .pdf, .txt, .docx, .doc, .pptx and .ppt are supported.") try: # Fetch document content from the provided URL document_content = fetch_document_with_curl(document_url) # Save document content to a temporary file with tempfile.NamedTemporaryFile(delete=False, suffix=file_extension) as temp_file: temp_file.write(document_content) temp_file_path = temp_file.name print(f"Temporary file saved at: {temp_file_path}") # Load document pages from the PDF or other format docs = load_pdf(temp_file_path) print(f"{len(docs)} pages loaded from the document.") # Initialize embeddings and split documents with semantic chunker embeddings = OllamaEmbeddings(model='nomic-embed-text') split_documents = split_text_with_semantic_chunker(docs, embeddings) print("Text successfully split into semantic chunks.") print(f"Documents after splitting: {len(split_documents)} chunks created.") # Save split documents to a text file in the specified output directory output_dir = f'temp/{client_id}_{reference_id}' save_documents_to_txt(split_documents, output_dir) # Create and save embeddings for each split document create_and_save_embeddings(split_documents, client_id, reference_id) merged_embeddings = merge_all_faiss(client_id, reference_id) token_count = count_tokens_in_documents(docs) return {"message": "Training and embedding creation completed successfully.","token_count": token_count} except Exception as e: raise HTTPException(status_code=500, detail=f"Processing failed: {e}") finally: if os.path.exists(temp_file_path): os.remove(temp_file_path) """ FOR UNTRAIN FROM LOCAL """ import shutil @app.post("/untrain_document_edurigo_ai") async def delete_document(reference_id: str = Form(...), client_id: str = Form(...)): try: local_path = f"my_embeddings/{client_id}" file_path = os.path.join(local_path, reference_id) if os.path.exists(file_path): if os.path.isfile(file_path): os.remove(file_path) else: shutil.rmtree(file_path) return {"message": "Document untrained successfully"} else: raise HTTPException(status_code=404, detail="Document not found") except Exception as e: raise HTTPException(status_code=500, detail=str(e)) """ GENERATE MCQS LOCALY """ from generate_mcq import generate_mcq class GenerateMCQRequest(BaseModel): client_id: int num_questions: int reference_id: int isGPU:int @app.post("/generate_mcq") async def generate_mcq_endpoint(client_id: int = Form(...), num_questions: int = Form(...), reference_id: int = Form(...), isGPU:int=Form()): mcqs, history_count = await generate_mcq(client_id, num_questions, reference_id, isGPU) return {"history_count": history_count, "mcqs": mcqs} """ CHAT WITH EDURIGO AI """ # from generate_mcq import generate_mcq # class GenerateMCQRequest(BaseModel): # client_id: int # num_questions: int # reference_id: int # @app.post("/generate_mcq") # async def generate_mcq_endpoint(client_id: int = Form(...), num_questions: int = Form(...), reference_id: int = Form(...)): # mcqs, history_count = await generate_mcq(client_id, num_questions, reference_id) # return {"history_count": history_count, "mcqs": mcqs} """ CHAT WITH EDURIGO AI """ from chatting import answer_question class ChatResponse(BaseModel): answer: str processing_time: float total_tokens: int source_documents: List[str] class ErrorResponse(BaseModel): error: str @app.post("/chat_with_edurigo_ai", response_model=ChatResponse, responses={400: {"model": ErrorResponse}, 500: {"model": ErrorResponse}}) async def chat_with_document(client_id: str = Form(...), question: str = Form(...)): try: answer, source_reference_ids, total_tokens, processing_time = answer_question(int(client_id), question) return ChatResponse( answer=answer, processing_time=processing_time, total_tokens=total_tokens, source_documents=source_reference_ids # This is now a list of reference_ids ) except FileNotFoundError as e: raise HTTPException(status_code=400, detail=str(e)) except Exception as e: raise HTTPException(status_code=500, detail=f"An error occurred: {str(e)}") """ Video """ from training_video import convert_video_to_audio,audio_to_text, load_and_split_document1, save_faiss_index,count_tokens_video class TrainResponse1(BaseModel): message: str token_count: int class ErrorResponse1(BaseModel): error: str def fetch_document_with_curl1(url): try: result = subprocess.run(["curl", "-k", "-s", url], capture_output=True, check=True) return result.stdout except subprocess.CalledProcessError as e: raise HTTPException(status_code=400, detail=f"Error fetching document: {e}") """ def load_and_split_document1(file_path): try: with open(file_path, 'r', encoding='utf-8') as file: text = file.read() text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200) chunks = text_splitter.split_text(text) documents = [Document(page_content=chunk, metadata={"source": file_path}) for chunk in chunks] return documents except Exception as e: raise HTTPException(status_code=500, detail=f"Error processing document: {e}") """ """ def count_tokens(documents): return sum(len(doc.page_content.split()) for doc in documents) """ @app.post("/train_with_video_edurigo_ai", response_model=TrainResponse1, responses={400: {"model": ErrorResponse1}, 500: {"model": ErrorResponse1}}) async def process_document(client_id: str = Form(...), document_url: str = Form(...), reference_id: str = Form(...)): file_extension = os.path.splitext(document_url)[1].lower() if file_extension not in ['.mp4']: raise HTTPException(status_code=400, detail="Unsupported file type. Only .mp4 supported.") temp_file_path = f"temp_document{file_extension}" try: mp3_output_path = f"mp3-files/{reference_id}.mp3" output_file = f"{reference_id}.txt" faiss_save_path = f"my_embeddings_video/{client_id}/{reference_id}" # Fetch and save document document_content = fetch_document_with_curl1(document_url) with open(temp_file_path, "wb") as temp_file: temp_file.write(document_content) # Convert and process audio # clip = VideoFileClip(temp_file_path) # video_duration = clip.duration / 60 # Convert to minutes video_duration =5 if video_duration > 15: raise HTTPException(status_code=400, detail="Video file too large.") convert_video_to_audio(temp_file_path, mp3_output_path) text = audio_to_text(mp3_output_path, output_file) # Load and split document final_documents = load_and_split_document1(output_file) if not final_documents: raise HTTPException(status_code=400, detail="No valid documents to process.") token_count = count_tokens_video(final_documents) # Save FAISS index folder_path = f"my_embeddings_video/{client_id}/{reference_id}" os.makedirs(folder_path, exist_ok=True) save_faiss_index(final_documents, faiss_save_path) return {"message": "Success", "token_count": token_count} except Exception as e: raise HTTPException(status_code=500, detail=f"Processing failed: {str(e)}") finally: if os.path.exists(temp_file_path): os.remove(temp_file_path) if os.path.exists(mp3_output_path): os.remove(mp3_output_path) """ GENERATE Video MCQS LOCALY """ from generate_mcq_video import generate_mcq_video class GenerateMCQRequest(BaseModel): client_id: int num_questions: int reference_id: int isGPU:int @app.post("/generate_mcq_video") async def generate_mcq_endpoint(client_id: int = Form(...), num_questions: int = Form(...), reference_id: int = Form(...), isGPU:int=Form()): mcqs, history_count, token_count = await generate_mcq_video(client_id, num_questions, reference_id, isGPU) return {"history_count": history_count, "mcqs": mcqs, "token_count": token_count} """ CHAT WITH EDURIGO AI WITH ENHANCED USER EXPIRIENCE """ """ import json from fastapi.responses import StreamingResponse from chatting import answer_question_stream class ChatResponse(BaseModel): answer: str token_count: int async def generate_stream(client_id: int, question: str): async for chunk, token_count in answer_question_stream(client_id, question): yield f"data: {json.dumps({'chunk': chunk, 'token_count': token_count})}\n\n" @app.post("/chat_with_edurigo_ai") async def chat(client_id: int = Form(...), question: str = Form(...)): async def generate(): async for chunk, token_count in answer_question_stream(client_id, question): yield f"data: {json.dumps({'chunk': chunk, 'token_count': token_count})}\n\n" return StreamingResponse(generate(), media_type="text/event-stream") """