import os import time import random import re import json import requests from langchain_community.embeddings import OllamaEmbeddings from langchain_community.vectorstores import FAISS from langchain_core.prompts import ChatPromptTemplate from crawl4ai import AsyncWebCrawler import asyncio from langchain_groq import ChatGroq from langchain_core.prompts import PromptTemplate from langchain_core.output_parsers import StrOutputParser, PydanticOutputParser from langchain_core.runnables import RunnablePassthrough from langchain_community.document_loaders import PyPDFLoader, TextLoader, Docx2txtLoader, UnstructuredPowerPointLoader from PyPDF2 import PdfReader from langchain_experimental.text_splitter import SemanticChunker from pydantic import BaseModel, Field from typing import List, Optional, Dict from urllib.parse import urlparse from langchain_ollama import ChatOllama import shutil GROQ_API_KEY = "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" OLLAMA_MODEL = "nomic-embed-text" def map_skills_to_jobProfile(skills_db, job_profiles_data): # Initialize the chat model chat = ChatOllama( base_url='http://127.0.0.1:11434', model="llama3:8b" ) #chat = ChatGroq(temperature=0, groq_api_key=GROQ_API_KEY, model_name="llama-3.3-70b-versatile") # Convert mapped_skills dictionary to a JSON string #formatted_mapped_skills = json.dumps(mapped_skills, ensure_ascii=False) # Format job profiles with descriptions for better context formatted_job_profiles = [] for profile in job_profiles_data: name = profile.get('name', '') description = profile.get('description', '') if description: formatted_job_profiles.append(f"{name}: {description}") else: formatted_job_profiles.append(name) job_profiles_text = "\n".join(formatted_job_profiles) # Define the updated system prompt system_prompt = PromptTemplate( input_variables=["skills_db", "job_profiles"], template=''' You are an expert in skill-to-job-profile mapping. Your task is to validate,refine and map skills to job profiles to ensure accuracy, completeness, and clarity. ### Input: ### Available Skills: {skills_db} ### Job Profiles (with descriptions where available): {job_profiles} ### Output Format: {{ "Job Profile 1": ["Skill 1", "Skill 2", "Skill 3"], "Job Profile 2": ["Skill 4", "Skill 5"], "Job Profile 3": ["Skill 6", "Skill 7", "Skill 8"] }} ### Enhanced Mapping Guidelines: 1. **Leverage Job Descriptions:** - Use the provided job descriptions to better understand the role requirements. - Map skills that align with the specific responsibilities and expertise mentioned in descriptions. 2. **Ensure Relevance:** - Verify that each skill is appropriately mapped to the correct job profile(s) based on both the role name and description. - Remove any incorrectly assigned skills. 3. **Check for Completeness:** - Identify and add missing skills relevant to each job profile based on the description context. - For profiles without descriptions, use industry-standard expectations for that role. 4. **Refine Overlapping Skills:** - If a skill belongs to multiple job profiles, ensure it is correctly included in each where relevant. - Consider the description context to determine skill relevance. 5. **Improve Consistency & Clarity:** - Standardize skill naming to avoid duplicates with different names. - Use precise and industry-recognized terminology. 6. **Handle Profiles with/without Descriptions:** - For profiles with descriptions: prioritize skills that match the described responsibilities. - For profiles without descriptions: use standard industry expectations for that role title. 7. **Include Soft Skills Where Relevant:** - Evaluate the job description for behavioral, interpersonal, or communication requirements. - Map relevant soft skills such as Communication, Problem Solving, Teamwork, Leadership, or Adaptability when implied or directly stated - Do not exclude soft skills if they play a crucial role in client-facing or collaborative roles. 7. **Response Format:** - **ONLY return the JSON object without any explanations, comments, or additional text.** - **DO NOT wrap the JSON in code blocks (```json ... ```).** - Use the exact job profile names as provided in the input. ''' ) # Format the prompt safely formatted_prompt = system_prompt.format( skills_db=skills_db.replace("{", "{{").replace("}", "}}"), job_profiles=job_profiles_text.replace("{", "{{").replace("}", "}}") ) try: output = chat.invoke(formatted_prompt) out = output.content print("Raw API response:", out) skill_to_JobProfile_mapping = extract_json_from_response(out) if skill_to_JobProfile_mapping is None: return None, 0 token_consumed = output.usage_metadata["output_tokens"] #token_consumed = output.response_metadata['token_usage']['total_tokens'] except Exception as e: print(f"An error occurred during chat invocation: {e}") return None, 0 return skill_to_JobProfile_mapping, token_consumed def map_skills_to_jobProfile_without_description(skills_db, JobProfile, model_name="mixtral-8x7b-32768", temperature=0): # Initialize the chat model #chat = ChatGroq(temperature=temperature, groq_api_key=groq_api_key, model_name=model_name) chat = ChatOllama( base_url = 'http://127.0.0.1:11434', model = "llama3:8b" #model = "deepseek-r1:8b" ) #print(chat) # Define the system prompt system_prompt = PromptTemplate( input_variables=["skills_db", "JobProfile"], template=''' You are an expert in skill-to-Job Profile mapping. Your task is to analyze a given set of skills and map them to the most appropriate JobProfile. ### Input Skills: {skills_db} ### Designations: {JobProfile} ### Output Format: {{ "JobProfile 1": ["Skill 1", "Skill 2", "Skill 3"], "JobProfile 2": ["Skill 4", "Skill 5"], "JobProfile 3": ["Skill 6", "Skill 7", "Skill 8"] }} ### Notes: - Ensure that each skill is mapped only to the most relevant JobProfile(s). - If a skill fits multiple JobProfile, include it in both. - Use clear and concise mappings. - Respond only with the JSON object in the specified format, without any additional text. - If you found no skills for some JobProfile check does JobProfile at last one more time ### Instructions: - **Respond only with the JSON object in the specified format.** - **Do not include any additional text, explanations, or code fences (e.g., ```json).** - **Ensure that the JSON is properly formatted and valid.** ''' ) # Format the prompt with the input skills and designations formatted_prompt = system_prompt.format(skills_db=skills_db, JobProfile=JobProfile) #print(formatted_prompt) # Send the formatted prompt to ChatGroq try: output = chat.invoke(formatted_prompt) total_tokens = output.usage_metadata["output_tokens"] #token_consumed = output.response_metadata['token_usage']['total_tokens'] skill_to_jobProfile_mapping = output.content except Exception as e: print(f"An error occurred during chat invocation: {e}") return None, 0 return skill_to_jobProfile_mapping, total_tokens import json import re def extract_json_from_response(text): try: # Match from the first { to the last } (very basic but works if only one JSON object is present) start = text.find('{') end = text.rfind('}') if start != -1 and end != -1: json_str = text[start:end + 1] return json.loads(json_str) else: print("Could not find JSON object in text.") return None except Exception as e: print(f"Error extracting or parsing JSON: {e}") return None def map_skills_to_jobProfile_non_empty(skills_db, job_profiles_data, mapped_skills): # Initialize the chat model chat = ChatOllama( base_url='http://127.0.0.1:11434', model="llama3:8b" ) #chat = ChatGroq(temperature=0, groq_api_key=GROQ_API_KEY, model_name="llama-3.3-70b-versatile") # Convert mapped_skills dictionary to a JSON string formatted_mapped_skills = json.dumps(mapped_skills, ensure_ascii=False) # Format job profiles with descriptions for better context formatted_job_profiles = [] for profile in job_profiles_data: name = profile.get('name', '') description = profile.get('description', '') if description: formatted_job_profiles.append(f"{name}: {description}") else: formatted_job_profiles.append(name) job_profiles_text = "\n".join(formatted_job_profiles) # Define the updated system prompt system_prompt = PromptTemplate( input_variables=["skills_db", "job_profiles", "mapped_skills"], template=''' You are an expert in skill-to-job-profile mapping. Your task is to validate and refine a given mapping of skills to job profiles to ensure accuracy, completeness, and clarity. ### Input: - **Existing Mapped Skills** (JSON format): {mapped_skills} ### Available Skills: {skills_db} ### Job Profiles (with descriptions where available): {job_profiles} ### Output Format: {{ "Job Profile 1": ["Skill 1", "Skill 2", "Skill 3"], "Job Profile 2": ["Skill 4", "Skill 5"], "Job Profile 3": ["Skill 6", "Skill 7", "Skill 8"] }} ### Enhanced Mapping Guidelines: 1. **Leverage Job Descriptions:** - Use the provided job descriptions to better understand the role requirements. - Map skills that align with the specific responsibilities and expertise mentioned in descriptions. 2. **Ensure Relevance:** - Verify that each skill is appropriately mapped to the correct job profile(s) based on both the role name and description. - Remove any incorrectly assigned skills. 3. **Check for Completeness:** - Identify and add missing skills relevant to each job profile based on the description context. - For profiles without descriptions, use industry-standard expectations for that role. 4. **Refine Overlapping Skills:** - If a skill belongs to multiple job profiles, ensure it is correctly included in each where relevant. - Consider the description context to determine skill relevance. 5. **Improve Consistency & Clarity:** - Standardize skill naming to avoid duplicates with different names. - Use precise and industry-recognized terminology. 6. **Handle Profiles with/without Descriptions:** - For profiles with descriptions: prioritize skills that match the described responsibilities. - For profiles without descriptions: use standard industry expectations for that role title. 7. **Response Format:** - **ONLY return the JSON object without any explanations, comments, or additional text.** - **DO NOT wrap the JSON in code blocks (```json ... ```).** - Use the exact job profile names as provided in the input. ''' ) # Format the prompt safely formatted_prompt = system_prompt.format( skills_db=skills_db.replace("{", "{{").replace("}", "}}"), job_profiles=job_profiles_text.replace("{", "{{").replace("}", "}}"), mapped_skills=formatted_mapped_skills ) try: output = chat.invoke(formatted_prompt) out = output.content print("Raw API response:", out) skill_to_JobProfile_mapping = extract_json_from_response(out) if skill_to_JobProfile_mapping is None: return None, 0 token_consumed = output.usage_metadata["output_tokens"] #token_consumed = output.response_metadata['token_usage']['total_tokens'] except Exception as e: print(f"An error occurred during chat invocation: {e}") return None, 0 return skill_to_JobProfile_mapping, token_consumed def map_skills_to_jobProfile_non_empty_without_description(skills_db, JobProfile, mapped_skills): # Initialize the chat model #chat = ChatGroq(temperature=temperature, groq_api_key=groq_api_key, model_name=model_name) chat = ChatOllama( base_url = 'http://127.0.0.1:11434', model = "llama3:8b" #model = "deepseek-r1:8b" ) # Convert mapped_skills dictionary to a JSON string formatted_mapped_skills = json.dumps(mapped_skills, ensure_ascii=False) # Define the system prompt system_prompt = PromptTemplate( input_variables=["skills_db", "JobProfile", "mapped_skills"], template=''' You are an expert in skill-to-JobProfile mapping. Your task is to validate and refine a given mapping of skills to JobProfile to ensure accuracy, completeness, and clarity. ### Input: - **Existing Mapped Skills** (JSON format): {mapped_skills} ### Input Skills: {skills_db} ### JobProfile: {JobProfile} ### Output Format: {{ "JobProfile 1": ["Skill 1", "Skill 2", "Skill 3"], "JobProfile 2": ["Skill 4", "Skill 5"], "JobProfile 3": ["Skill 6", "Skill 7", "Skill 8"] }} ### Validation & Enhancement Guidelines: 1. **Ensure Relevance:** - Verify that each skill is appropriately mapped to the correct JobProfile(s). - Remove any incorrectly assigned skills. 2. **Check for Completeness:** - Identify and add missing skills relevant to each JobProfile. - Ensure no important skills are overlooked. 3. **Refine Overlapping Skills:** - If a skill belongs to multiple JobProfile, ensure it is correctly included in each. - Avoid redundant mappings unless justified. 4. **Improve Consistency & Clarity:** - Standardize skill naming to avoid duplicates with different names. - Use precise and industry-recognized terminology. 5. **Handle Unmapped JobProfile:** - If a JobProfile lacks sufficient mapped skills, reevaluate and attempt to match relevant ones. 6. **Respond in Valid JSON Format:** - **ONLY return the JSON object without any explanations, comments, or additional text.** - **DO NOT wrap the JSON in code blocks (```json ... ```).** ''' ) # Format the prompt safely formatted_prompt = system_prompt.format( skills_db=skills_db.replace("{", "{{").replace("}", "}}"), JobProfile=JobProfile.replace("{", "{{").replace("}", "}}"), mapped_skills=formatted_mapped_skills ) try: output = chat.invoke(formatted_prompt) out = output.content print("Raw API response:", out) skill_to_JobProfile_mapping = extract_json_from_response(out) if skill_to_JobProfile_mapping is None: return None, 0 token_consumed = output.usage_metadata["output_tokens"] except Exception as e: print(f"An error occurred during chat invocation: {e}") return None, 0 return skill_to_JobProfile_mapping, token_consumed # Send the formatted prompt to ChatGroq # try: # output = chat.invoke(formatted_prompt) # # Debugging: Print raw API response # print(f"Raw API response: {output.content}") # out = output.content # print(type(out)) # print("out") # # Extract the JSON part from the response # ##json_data1 = extract_json_from_response(output.content) # #print(type(json_data1)) # #print("JSON Com") # # if not json_data: # # print("Error: No valid JSON found in the response") # # return None, 0 # skill_to_JobProfile_mapping = extract_json_from_response(out) # if skill_to_JobProfile_mapping is None: # return None, 0 # # Ensure response is valid JSON # #skill_to_JobProfile_mapping = json.loads(out) # Convert response to dict # print(type(skill_to_JobProfile_mapping)) # print(skill_to_JobProfile_mapping) # #token_consumed = output.response_metadata['token_usage']['total_tokens'] # token_consumed = output.usage_metadata["output_tokens"] # except json.JSONDecodeError as e: # print(f"Error decoding JSON response: {e}") # return None, 0 # except Exception as e: # print(f"An error occurred during chat invocation: {e}") # return None, 0 # return skill_to_JobProfile_mapping, token_consumed # Example usage if __name__ == "__main__": GROQ_API_KEY = "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" # Replace with your actual Groq API key industry = "Infrastructure" #skill_tree,token_consumed = generate_skill_tree(prompt_text=industry, groq_api_key=GROQ_API_KEY) skillset =''' Budgeting, Cost Estimation, Civil Engineering, Structural Analysis, AutoCAD, Revit, Communication, Leadership, Teamwork, Urban Design, Water Supply and Distribution , SQL, C++ , Management, Housekeeping Management,Front Desk Management,Complaint Handling,Recruitment,Training & Development ''' JobProfile = ''' HR, Developer, Manager, Planner,CEO, Infrastructure Deverloper ''' mapped = { "Manager": [ "Time Management", "Decision Making", "Industry Trends", "Sector Expertise", "Business Acumen", "Stakeholder Communication", "Risk Management", "Agile Methodologies", "Leadership", "Teamwork", "Adaptability" ], "Engineer": [ "Data Encryption", "Incident Response", "Cloud Architecture", "Cloud Migration", "Cloud Monitoring", "Network Security", "Programming Languages", "Py" ], "CEO": [ "Negotiation", "Empathy", "Leadership", "Risk Management", "Business Acumen", "Sector Expertise", "Industry Trends", "Stakeholder Communication", "Creativity" ], "CO-Founder": [ "Creativity", "Sector Expertise", "Empathy", "Leadership", "Stakeholder Communication", "Risk Management", "Business Acumen", "Industry Trends", "Negotiation" ], "Developer": [ "Py", "Programming Languages", "Software Testing", "Version Control" ], "Senior Developer": [ "Network Security", "Py", "Programming Languages", "Data Encryption", "Incident Response" ], "HR Head": [ "Communication", "Stakeholder Communication" ], "Product Engineer": [ "Software Testing", "Version Control", "Programming Languages", "Cloud Migration", "Cloud Monitoring", "Py", "Cloud Architecture" ] } mapped_skills = { "Manager": [ "Time Management", "Decision Making", "Industry Trends", "Sector Expertise", "Business Acumen", "Stakeholder Communication", "Risk Management", "Agile Methodologies", "Leadership", "Teamwork", "Adaptability" ], "Engineer": [ "Data Encryption", "Incident Response", "Cloud Architecture", "Cloud Migration", "Cloud Monitoring", "Network Security", "Programming Languages", "Py" ], "CEO": [ "Negotiation", "Empathy", "Leadership", "Risk Management", "Business Acumen", "Sector Expertise", "Industry Trends", "Stakeholder Communication", "Creativity" ], "CO-Founder": [ "Creativity", "Sector Expertise", "Empathy", "Leadership", "Stakeholder Communication", "Risk Management", "Business Acumen", "Industry Trends", "Negotiation" ], "Developer": [ "Py", "Programming Languages", "Software Testing", "Version Control" ], "Senior Developer": [ "Network Security", "Py", "Programming Languages", "Data Encryption", "Incident Response" ], # "HR Head": [ # "Communication", # "Stakeholder Communication" # ], "Product Engineer": [ "Software Testing", "Version Control", "Programming Languages", "Cloud Migration", "Cloud Monitoring", "Py", "Cloud Architecture" ] } # print(type(mapped_skills)) json_data = json.dumps(mapped_skills) # 'indent' makes it readable print("OPO") print(json_data) print("LKL") # print(type(json_data)) # print(json_data) #print(mapped_skills) skill_tree,token_consumed = map_skills_to_jobProfile_non_empty(skills_db= skillset,JobProfile =JobProfile, mapped_skills= json_data) #skill_tree,token_consumed = map_skills_to_designations(skills_db= skillset,JobProfile =JobProfile) print(token_consumed) print(skill_tree) print(type(skill_tree))