o :iL@sdZddlZddlZddlZddlZddlZddlZddlZddl m Z m Z m Z m Z mZddlmZmZddlmZmZddlmZddlmZGdd d ZeZdS) z High-Performance Professional Image Scraper v2. Optimized for SPEED, QUALITY, and RELEVANCE. Fetches single-person, front-facing professional portraits with AI validation. N)OptionalDictAnyListTuple)Image ImageEnhance)ThreadPoolExecutor as_completed) GroqService) OllamaServicec@s0eZdZdZdZdZdZgdZdddd d Zd d Z d e e e ffddZ de e e fd e fddZde d ee fddZde d ee fddZded eee ffddZded eee ffddZde de d efdd Zd*d"e de d ee fd#d$Zde d ee fd%d&Zde e e fd ee fd'd(Zd)S)+ImageScraperServicezx Enhanced image scraper with AI validation and face detection. >2x faster with 3x better quality filtering. ii8)zoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36zuMozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36zeMozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36zPhttps://images.unsplash.com/photo-1560250097-0b93528c311a?w=1920&h=1080&fit=cropzShttps://images.unsplash.com/photo-1573496359142-b8d87734a5a2?w=1920&h=1080&fit=cropzShttps://images.unsplash.com/photo-1519085360753-af0119f7cbe7?w=1920&h=1080&fit=cropzShttps://images.unsplash.com/photo-1507003211169-0a1dd7228f2d?w=1920&h=1080&fit=crop)manager executive consultant professionalcCst|_|jjddiz t|_t|_d|_ Wn d|_d|_d|_ Yz ddl }||_ d|_ WdSd|_ d|_ YdS)N Connectionz keep-aliveTFr) requestsSessionsessionheadersupdater groq_servicer ollama_servicehas_aicv2has_cv2)selfrrM/var/www/eduai.edurigo.com/roleplay/testing/services/image_scraper_service.py__init__-s"    zImageScraperService.__init__returncCst|jdddddS)NzJtext/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8zen-US,en;q=0.9https://duckduckgo.com/1)z User-AgentAcceptzAccept-LanguageRefererDNT)randomchoice USER_AGENTS)rrrr _get_headersDs z ImageScraperService._get_headers roleplay_datacCs|dd}|dd}t|dkr$dt|kr#dkr$|S|jr]|jr]z.d|d|d }|jjd |d gd d }|rWt|dkrW|ddddWSWnY|dd}d|S)zH Extracts job title with AI fallback for complex roles. z Learner rolecategoryzXExtract a 2-4 word professional job title from this description: Role: z Category: z{ Return ONLY the job title, nothing else. Examples: "Marketing Manager", "Software Engineer", "Sales Director" Job Title:user)rolecontent) max_tokens2"'N ) getlensplitstriprr_call_groq_apireplacejoin)rr,r3catpromptrespwordsrrr _get_job_title_fastMs.  &  z'ImageScraperService._get_job_title_fast job_titlecCsdd|dd|dd|dd|dg}|dd|dd |d d |d d g}|S)zU Builds ULTRA-STRICT queries for professional studio headshots only. r8z0" professional headshot studio portrait businessz0" corporate headshot linkedin professional photoz/" business portrait studio background executivez1" professional portrait corporate office headshotrz site:unsplash.comz site:pexels.comr0z! "studio lighting" "professional"r:z! "business attire" "front facing"r)rrH strict_termsqueriesrrr build_fast_queriesqs       z&ImageScraperService.build_fast_queriesqueryc CstdD]}ze|jjdd|i|dd}td|j}|s%tdWqdd || d d d d d d}|jjd||dd}| }|dg}g}|ddD]} | dd} d| vrf| rf| | qQ|WSt y} z|dkr|tdWYd} ~ qd} ~ wwgS)z@ Fast DuckDuckGo image search with retry logic. r0r#qr1)paramsrtimeoutzvqd="?([^"&]+)"??zus-enjsonrIz,,,layout:tall,size:Large,,r$images)lorNvqdfpiaxiazhttps://duckduckgo.com/i.jsresultsNr5imager-thumbr)rangerr<r+researchtexttimesleepgrouprRlowerappend Exception) rrMattemptr vqd_matchrOdatar\urlsresimg_urlerrr _scrape_ddg_fastsV         z$ImageScraperService._scrape_ddg_fastimgc Cs&|j\}}|dks |dkrdS||}|dks|dkrdSzo|d}t|}|}|dks4|dkr7Wd S|}|d krBWd St|} | } | d krRWd S| dt|dddddf} | jdd} | ddkrx| ddkrxWdS| ddkr| ddkrWdSWdSWdSYdS)zc STRICT professional image quality validation. Returns (is_valid, reason). i )F too_small333333?g?)Fnot_portrait_ratioL<)Fpoor_studio_lighting#)Flow_contrast_unprofessionalr/)Fflat_unprofessionalNg?)rrI)axisr0r)Foutdoor_sky_backgroundrIx)Foutdoor_nature_background)Tpass)sizeconvertnparraymeanstdint) rrrwhratio grayscalearrmean_brightnessstd_devrgb_arrcolor_variance top_section avg_colorrrr _assess_image_qualitys@    " z)ImageScraperService._assess_image_qualityc Csj|jsdSz|jt||jj}|j||jj}|j|jjj d}|j |dddd}t |dkr8WdSt |d krAWd S|d\}}}} |j \} } || d } | | d kr]Wd S||d } t | | d | }|dkrtWdS|| }| | }||}|dkrWdS|dkrWdSd|krdksWdSWdSWdSty}zdWYd}~Sd}~ww)zz STRICT face detection - must be frontal, centered, and professional. Returns (has_face, reason). )Tno_cv2z#haarcascade_frontalface_default.xml?r[)rr) scaleFactor minNeighborsminSizer)Fno_face_detectedrI)Fmultiple_peopler0rt)Fface_not_centered_verticallyg?)Fface_not_centered_horizontallygQ?)Fface_too_small_not_headshotg?)Fextreme_closeup_unprofessionalg333333?rQ)Fimproper_headshot_framing)Tprofessional_headshot_detectedTdetection_errorN)Tr)rrcvtColorrr COLOR_RGB2BGRCOLOR_BGR2GRAYCascadeClassifierrl haarcascadesdetectMultiScaler=rabsrh)rrrcv_imggray face_cascadefacesxyrrimg_wimg_h face_center_y face_center_xhorizontal_deviation face_areaimg_area face_ratiorprrr _detect_frontal_facesV       z(ImageScraperService._detect_frontal_face base64_imgcCsX|jr|jsdSzd|d}t|jdr#|j||}d|vWSWdSYdS)z Uses AI vision model to validate image relevance (optional). Only used if available and as final validation. Ta7Analyze this image strictly. Answer ONLY 'YES' or 'NO'. Requirements (ALL must be met): - Exactly ONE person visible - Person facing camera directly (not side profile) - Professional/business attire or neutral clothing - Studio, office, or plain background (not outdoor/casual) - Appears relevant to job role: z Answer (YES or NO): analyze_imageYES)rrhasattrrupper)rrrHrDresponserrr _validate_with_ai9s  z%ImageScraperService._validate_with_air-urlc szgd}tfdd|DsWdS|jj|dd}|jdks*t|jdkr-WdStt |j d }| |\}}|sEWdS| |\}} |sQWdS|j\} } d } | | | krtt| | } | | krj| } |d d | | f}nt| | }| |d }||d ||| f}||j|jftj}t|d }t|d}t|d}t }|j|ddddt|}d|}|WSty}zWYd}~dSd}~ww)z Downloads, validates, and processes a single image URL. Returns Base64 string if all validations pass. )z.jpgz.jpegz.pngzimages.unsplashz images.pexelsz pixabay.comc3s|] }|vVqdSN)rf).0extrrr _sz=ImageScraperService._fetch_and_process_url..Nr)rrPi NRGBgqq?rr0gffffff?rg{Gz?JPEG\T)formatqualityoptimizezdata:image/jpeg;base64,) anyrr<r+ status_coder=r4ropenioBytesIOrrrrrcropresize TARGET_WIDTH TARGET_HEIGHTLANCZOSr SharpnessenhanceContrast Brightnesssavebase64 b64encodegetvaluedecoderh)rrrHvalid_extensionsrjrr quality_passquality_reason face_pass face_reasonrr target_rationew_hnew_wleftbufb64 base64_datarprrr _fetch_and_process_urlWsJ      z*ImageScraperService._fetch_and_process_urlcs|tfdddDr|jd}n*tfdddDr(|jd}ntfddd Dr:|jd }n|jd }z|||WSYd S) zI Returns a curated fallback image based on job category. c3|]}|vVqdSrrrterm job_lowerrr rz:ImageScraperService._get_fallback_image..)rdirectorleadheadrc3rrrrrrr rr)rceo presidentvprc3rrrrrrr rr)radvisoranalystrrN)rfrFALLBACK_IMAGESr<r)rrH fallback_urlrrr _get_fallback_images z'ImageScraperService._get_fallback_imagecst}td|tdd}tdt|dg}tdd"fdd |D}t|D] }|}||q:Wd n1sPwYt t |}td t|d t|d d|sytd St dt|} t| d>fdd|d dD} t| D]"}|} | rt|} td| d d| Wd SqWd n1swYtd } | rtdt|d d| Stdd S)zZ Main entry point: fetches best professional image for roleplay scenario. z'INFO: Starting Enhanced Image Search...zINFO: Target Role: 'r9z INFO: Built z search queriesr[) max_workerscsi|] }j||qSr)submitrq)rrN)executorrrr z:ImageScraperService.get_roleplay_image..Nz INFO: Found z unique candidates in z.2fsz)WARN: No candidates found, using fallback csg|] }j|qSr)rr)rrrrHrrr rz:ImageScraperService.get_roleplay_image..z%SUCCESS: High-quality image found in z5WARN: No valid images found in search, using fallbackz"SUCCESS: Fallback image loaded in zERROR: All methods failed)rcprintrGrLr=r r resultextendlistdictfromkeysrmin)rr, start_timerKall_candidate_urlsfuture_to_queryfuturermcandidate_urlsmax_concurrentfuturesr elapsedfallbackrrr get_roleplay_imagesV       $      z&ImageScraperService.get_roleplay_imageN)r-)__name__ __module__ __qualname____doc__rrTIMEOUTr*rr!rstrr+rrGrrLrqrrboolrrrrrrrrrrr r s, $:6BE"r )rr`rrr(rcrnumpyrtypingrrrrrPILrrconcurrent.futuresr r services.groq_servicer services.ollama_servicer r image_scraper_servicerrrr s$   _