816 lines
No EOL
32 KiB
Python
816 lines
No EOL
32 KiB
Python
"""
|
|
Web crawler agent for gathering and analyzing web content.
|
|
Provides web research and content extraction capabilities.
|
|
"""
|
|
|
|
import logging
|
|
import json
|
|
import re
|
|
import time
|
|
import os
|
|
from typing import Dict, Any, List
|
|
from urllib.parse import quote_plus, unquote
|
|
|
|
from bs4 import BeautifulSoup
|
|
import requests
|
|
import markdown
|
|
|
|
from modules.workflow.agentBase import AgentBase
|
|
from modules.shared.configuration import APP_CONFIG
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
class AgentWebcrawler(AgentBase):
|
|
"""AI-driven agent for web research and information retrieval"""
|
|
|
|
def __init__(self):
|
|
"""Initialize the web crawler agent"""
|
|
super().__init__()
|
|
self.name = "webcrawler"
|
|
self.label = "Web Crawler"
|
|
self.description = "Gathers and analyzes web content using AI with multi-step research"
|
|
self.capabilities = [
|
|
"web_research",
|
|
"content_gathering",
|
|
"data_extraction",
|
|
"information_synthesis",
|
|
"source_verification"
|
|
]
|
|
|
|
# Web crawling configuration
|
|
self.srcApikey = APP_CONFIG.get("Agent_Webcrawler_SERPAPI_APIKEY","")
|
|
self.srcEngine = APP_CONFIG.get("Agent_Webcrawler_SERPAPI_ENGINE","google")
|
|
self.srcCountry = APP_CONFIG.get("Agent_Webcrawler_SERPAPI_COUNTRY","auto")
|
|
self.maxUrl = int(APP_CONFIG.get("Agent_Webcrawler_SERPAPI_MAX_URLS", "5"))
|
|
self.maxSearchTerms = int(APP_CONFIG.get("Agent_Webcrawler_SERPAPI_MAX_SEARCH_KEYWORDS", "3"))
|
|
self.maxResults = int(APP_CONFIG.get("Agent_Webcrawler_SERPAPI_MAX_SEARCH_RESULTS", "5"))
|
|
self.timeout = int(APP_CONFIG.get("Agent_Webcrawler_SERPAPI_TIMEOUT", "30"))
|
|
self.userAgent = APP_CONFIG.get("Agent_Webcrawler_SERPAPI_USER_AGENT", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
|
|
|
|
if not self.srcApikey:
|
|
logger.error("SerpAPI key not configured")
|
|
|
|
def setDependencies(self, serviceBase=None):
|
|
"""Set external dependencies for the agent."""
|
|
self.setService(serviceBase)
|
|
|
|
async def processTask(self, task: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""
|
|
Process a task by focusing on required outputs and using AI to guide the research process.
|
|
|
|
Args:
|
|
task: Task dictionary with prompt, inputDocuments, outputSpecifications
|
|
|
|
Returns:
|
|
Dictionary with feedback and documents
|
|
"""
|
|
try:
|
|
# Extract task information
|
|
prompt = task.get("prompt", "")
|
|
inputDocuments = task.get("inputDocuments", [])
|
|
outputSpecs = task.get("outputSpecifications", [])
|
|
workflow = task.get("context", {}).get("workflow", {})
|
|
|
|
# Check AI service
|
|
if not self.service or not self.service.base:
|
|
return {
|
|
"feedback": "The Web Crawler agent requires an AI service to function.",
|
|
"documents": []
|
|
}
|
|
|
|
# Create research plan
|
|
if workflow:
|
|
self.workflowManager.logAdd(workflow, "Creating research plan...", level="info", progress=35)
|
|
researchPlan = await self._createResearchPlan(prompt)
|
|
|
|
# Check if this is truly a web research task
|
|
if not researchPlan.get("requiresWebResearch", True):
|
|
return {
|
|
"feedback": "This task doesn't appear to require web research. Please try a different agent.",
|
|
"documents": []
|
|
}
|
|
|
|
# Gather raw material through web research
|
|
if workflow:
|
|
self.workflowManager.logAdd(workflow, "Gathering research material...", level="info", progress=45)
|
|
rawResults = await self._gatherResearchMaterial(researchPlan, workflow)
|
|
|
|
# Format results into requested output documents
|
|
if workflow:
|
|
self.workflowManager.logAdd(workflow, "Creating output documents...", level="info", progress=55)
|
|
documents = await self._createOutputDocuments(
|
|
prompt,
|
|
rawResults,
|
|
outputSpecs,
|
|
researchPlan
|
|
)
|
|
|
|
# Generate feedback
|
|
feedback = researchPlan.get("feedback", f"I conducted web research on '{prompt[:50]}...' and gathered information from {len(rawResults)} relevant sources.")
|
|
|
|
return {
|
|
"feedback": feedback,
|
|
"documents": documents
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error during web research: {str(e)}", exc_info=True)
|
|
return {
|
|
"feedback": f"Error during web research: {str(e)}",
|
|
"documents": []
|
|
}
|
|
|
|
async def _createResearchPlan(self, prompt: str) -> Dict[str, Any]:
|
|
"""
|
|
Use AI to create a detailed research plan.
|
|
|
|
Args:
|
|
prompt: The research query
|
|
|
|
Returns:
|
|
Research plan dictionary
|
|
"""
|
|
researchPrompt = f"""
|
|
Create a detailed web research plan for this task: "{prompt}"
|
|
|
|
Analyze the request carefully and create a structured plan in JSON format with the following elements:
|
|
{{
|
|
"requiresWebResearch": true/false, # Whether this genuinely requires web research
|
|
"researchQuestions": ["question1", "question2", ...], # 2-4 specific questions to answer
|
|
"searchTerms": ["term1", "term2", ...], # Up to {self.maxSearchTerms} effective search terms
|
|
"directUrls": ["url1", "url2", ...], # Any URLs directly mentioned in the request (up to {self.maxUrl})
|
|
"expectedSources": ["type1", "type2", ...], # Types of sources that would be most valuable
|
|
"contentFocus": "what specific content to extract or focus on",
|
|
"feedback": "explanation of how the research will be conducted"
|
|
}}
|
|
|
|
Respond with ONLY the JSON object, no additional text or explanations.
|
|
"""
|
|
|
|
try:
|
|
# Get research plan from AI
|
|
response = await self.service.base.callAi([
|
|
{"role": "system", "content": "You are a research expert. Respond with valid JSON only."},
|
|
{"role": "user", "content": researchPrompt}
|
|
])
|
|
|
|
# Extract JSON
|
|
jsonStart = response.find('{')
|
|
jsonEnd = response.rfind('}') + 1
|
|
|
|
if jsonStart >= 0 and jsonEnd > jsonStart:
|
|
plan = json.loads(response[jsonStart:jsonEnd])
|
|
|
|
# Ensure we have the expected fields with defaults if missing
|
|
if "searchTerms" not in plan:
|
|
plan["searchTerms"] = [prompt]
|
|
if "directUrls" not in plan:
|
|
plan["directUrls"] = []
|
|
if "researchQuestions" not in plan:
|
|
plan["researchQuestions"] = ["What information can be found about this topic?"]
|
|
|
|
return plan
|
|
else:
|
|
# Fallback plan
|
|
logger.warning(f"Not able creating research plan, generating fallback plan")
|
|
return {
|
|
"requiresWebResearch": True,
|
|
"researchQuestions": ["What information can be found about this topic?"],
|
|
"searchTerms": [prompt],
|
|
"directUrls": [],
|
|
"expectedSources": ["Web pages", "Articles"],
|
|
"contentFocus": "Relevant information about the topic",
|
|
"feedback": f"I'll conduct web research on '{prompt}' and gather relevant information."
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Error creating research plan: {str(e)}")
|
|
# Simple fallback plan
|
|
return {
|
|
"requiresWebResearch": True,
|
|
"researchQuestions": ["What information can be found about this topic?"],
|
|
"searchTerms": [prompt],
|
|
"directUrls": [],
|
|
"expectedSources": ["Web pages", "Articles"],
|
|
"contentFocus": "Relevant information about the topic",
|
|
"feedback": f"I'll conduct web research on '{prompt}' and gather relevant information."
|
|
}
|
|
|
|
async def _gatherResearchMaterial(self, researchPlan: Dict[str, Any], workflow: Dict[str, Any]) -> List[Dict[str, Any]]:
|
|
"""
|
|
Gather research material based on the research plan.
|
|
|
|
Args:
|
|
researchPlan: Research plan dictionary
|
|
workflow: Current workflow object
|
|
|
|
Returns:
|
|
List of research results
|
|
"""
|
|
allResults = []
|
|
|
|
# Process direct URLs
|
|
directUrls = researchPlan.get("directUrls", [])[:self.maxUrl]
|
|
for i, url in enumerate(directUrls):
|
|
progress = 45 + int((i / len(directUrls)) * 5) # Progress from 45% to 50%
|
|
if hasattr(self, 'workflowManager') and self.workflowManager:
|
|
self.workflowManager.logAdd(workflow, f"Processing direct URL {i+1}/{len(directUrls)}...", level="info", progress=progress)
|
|
logger.info(f"Processing direct URL: {url}")
|
|
try:
|
|
# Fetch and extract content
|
|
soup = self._readUrl(url)
|
|
|
|
if soup:
|
|
# Extract title and content
|
|
title = self._extractTitle(soup, url)
|
|
content = self._extractMainContent(soup)
|
|
|
|
# Add to results
|
|
allResults.append({
|
|
"title": title,
|
|
"url": url,
|
|
"sourceType": "directUrl",
|
|
"content": content,
|
|
"summary": "" # Will be filled later
|
|
})
|
|
except Exception as e:
|
|
logger.warning(f"Error processing URL {url}: {str(e)}")
|
|
|
|
# Process search terms
|
|
searchTerms = researchPlan.get("searchTerms", [])[:self.maxSearchTerms]
|
|
for i, term in enumerate(searchTerms):
|
|
progress = 50 + int((i / len(searchTerms)) * 5) # Progress from 50% to 55%
|
|
if hasattr(self, 'workflowManager') and self.workflowManager:
|
|
self.workflowManager.logAdd(workflow, f"Searching term {i+1}/{len(searchTerms)}...", level="info", progress=progress)
|
|
logger.info(f"Searching for: {term}")
|
|
try:
|
|
# Perform search
|
|
searchResults = self._searchWeb(term)
|
|
|
|
# Process each search result
|
|
for result in searchResults:
|
|
# Check if URL is already in results
|
|
if not any(r["url"] == result["url"] for r in allResults):
|
|
allResults.append({
|
|
"title": result["title"],
|
|
"url": result["url"],
|
|
"sourceType": "searchResult",
|
|
"content": result["data"],
|
|
"snippet": result["snippet"],
|
|
"summary": "" # Will be filled later
|
|
})
|
|
|
|
# Stop if we've reached the maximum results
|
|
if len(allResults) >= self.maxResults:
|
|
break
|
|
except Exception as e:
|
|
logger.warning(f"Error searching for {term}: {str(e)}")
|
|
|
|
# Stop if we've reached the maximum results
|
|
if len(allResults) >= self.maxResults:
|
|
break
|
|
|
|
# Create summaries for all results
|
|
allResults = await self._summarizeAllResults(allResults, researchPlan)
|
|
|
|
return allResults
|
|
|
|
async def _summarizeAllResults(self, results: List[Dict[str, Any]], researchPlan: Dict[str, Any]) -> List[Dict[str, Any]]:
|
|
"""
|
|
Create summaries for all research results.
|
|
|
|
Args:
|
|
results: List of research results
|
|
researchPlan: Research plan with questions and focus
|
|
|
|
Returns:
|
|
Results with added summaries
|
|
"""
|
|
for i, result in enumerate(results):
|
|
logger.info(f"Summarizing result {i+1}/{len(results)}: {result['title'][:30]}...")
|
|
|
|
try:
|
|
# Limit content length to avoid token issues
|
|
content = self._limitText(result.get("content", ""), maxChars=8000)
|
|
researchQuestions = researchPlan.get("researchQuestions", ["What relevant information does this page contain?"])
|
|
contentFocus = researchPlan.get("contentFocus", "Relevant information")
|
|
|
|
# Create summary using AI
|
|
summaryPrompt = f"""
|
|
Summarize this web page content based on these research questions:
|
|
{', '.join(researchQuestions)}
|
|
|
|
Focus on: {contentFocus}
|
|
|
|
Web page: {result['url']}
|
|
Title: {result['title']}
|
|
|
|
Content:
|
|
{content}
|
|
|
|
Create a concise summary that:
|
|
1. Directly answers the research questions if possible
|
|
2. Extracts the most relevant information from the page
|
|
3. Includes specific facts, figures, or quotes if available
|
|
4. Is around 2000 characters long
|
|
|
|
Only include information actually found in the content. No fabrications or assumptions.
|
|
"""
|
|
|
|
# Get summary from AI
|
|
summary = await self.service.base.callAi([
|
|
{"role": "system", "content": "You are a research expert. Respond with valid JSON only."},
|
|
{"role": "user", "content": summaryPrompt}
|
|
])
|
|
|
|
# Add summary to result
|
|
result["summary"] = summary.strip()
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Error summarizing result {i+1}: {str(e)}")
|
|
result["summary"] = f"Error creating summary: {str(e)}"
|
|
|
|
return results
|
|
|
|
async def _createOutputDocuments(self, prompt: str, results: List[Dict[str, Any]],
|
|
outputSpecs: List[Dict[str, Any]], researchPlan: Dict[str, Any]) -> List[Dict[str, Any]]:
|
|
"""
|
|
Create output documents based on research results and specifications.
|
|
|
|
Args:
|
|
prompt: Original research prompt
|
|
results: List of research results
|
|
outputSpecs: Output specifications
|
|
researchPlan: Research plan
|
|
|
|
Returns:
|
|
List of output documents
|
|
"""
|
|
# If no output specs provided, create default output
|
|
if not outputSpecs:
|
|
outputSpecs = [{
|
|
"label": "webResearchResults.md",
|
|
"description": "Comprehensive web research results"
|
|
}]
|
|
|
|
# Generate documents
|
|
documents = []
|
|
|
|
# Process each output specification
|
|
for spec in outputSpecs:
|
|
outputLabel = spec.get("label", "")
|
|
outputDescription = spec.get("description", "")
|
|
|
|
# Determine format based on file extension
|
|
formatType = self._determineFormatType(outputLabel)
|
|
|
|
# Create appropriate document based on format
|
|
if formatType == "json":
|
|
# JSON output - structured data
|
|
document = await self._createJsonDocument(prompt, results, researchPlan, outputLabel)
|
|
elif formatType == "csv":
|
|
# CSV output - tabular data
|
|
document = await self._createCsvDocument(results, outputLabel)
|
|
else:
|
|
# Text-based output (markdown, html, text) - narrative report
|
|
document = await self._createNarrativeDocument(
|
|
prompt, results, researchPlan, formatType, outputLabel, outputDescription
|
|
)
|
|
|
|
documents.append(document)
|
|
|
|
return documents
|
|
|
|
async def _createNarrativeDocument(self, prompt: str, results: List[Dict[str, Any]],
|
|
researchPlan: Dict[str, Any], formatType: str,
|
|
outputLabel: str, outputDescription: str) -> Dict[str, Any]:
|
|
"""
|
|
Create a narrative document (markdown, html, text) from research results.
|
|
|
|
Args:
|
|
prompt: Original research prompt
|
|
results: Research results
|
|
researchPlan: Research plan
|
|
formatType: Output format (markdown, html, text)
|
|
outputLabel: Output filename
|
|
outputDescription: Output description
|
|
|
|
Returns:
|
|
Document object
|
|
"""
|
|
# Create content based on format
|
|
if formatType == "markdown":
|
|
contentType = "text/markdown"
|
|
templateFormat = "markdown"
|
|
elif formatType == "html":
|
|
contentType = "text/html"
|
|
templateFormat = "html"
|
|
else:
|
|
contentType = "text/plain"
|
|
templateFormat = "text"
|
|
|
|
# Prepare research context
|
|
researchQuestions = researchPlan.get("researchQuestions", [])
|
|
searchTerms = researchPlan.get("searchTerms", [])
|
|
|
|
# Create document structure based on results
|
|
sourcesSummary = []
|
|
for result in results:
|
|
sourcesSummary.append({
|
|
"title": result.get("title", "Untitled"),
|
|
"url": result.get("url", ""),
|
|
"summary": result.get("summary", ""),
|
|
"snippet": result.get("snippet", "")
|
|
})
|
|
|
|
# Truncate content for prompt
|
|
sourcesJson = json.dumps(sourcesSummary, indent=2)
|
|
if len(sourcesJson) > 10000:
|
|
# Logic to truncate each summary while preserving structure
|
|
for i in range(len(sourcesSummary)):
|
|
if len(sourcesJson) <= 10000:
|
|
break
|
|
# Gradually truncate summaries
|
|
sourcesSummary[i]["summary"] = sourcesSummary[i]["summary"][:500] + "..."
|
|
sourcesJson = json.dumps(sourcesSummary, indent=2)
|
|
|
|
# Create report prompt
|
|
reportPrompt = f"""
|
|
Create a comprehensive {formatType} research report based on the following web research:
|
|
|
|
TASK: {prompt}
|
|
|
|
RESEARCH QUESTIONS:
|
|
{', '.join(researchQuestions)}
|
|
|
|
SEARCH TERMS USED:
|
|
{', '.join(searchTerms)}
|
|
|
|
SOURCES AND FINDINGS:
|
|
{sourcesJson}
|
|
|
|
REPORT DETAILS:
|
|
- Format: {templateFormat}
|
|
- Filename: {outputLabel}
|
|
- Description: {outputDescription}
|
|
|
|
Create a well-structured report that:
|
|
1. Includes an executive summary of key findings
|
|
2. Addresses each research question directly
|
|
3. Integrates information from all relevant sources
|
|
4. Cites sources appropriately for each piece of information
|
|
5. Provides a comprehensive synthesis of the research
|
|
6. Is formatted professionally and appropriately for {templateFormat}
|
|
|
|
The report should be scholarly, accurate, and focused on the original research task.
|
|
"""
|
|
|
|
try:
|
|
# Generate report with AI
|
|
reportContent = await self.service.base.callAi([
|
|
{"role": "system", "content": "You are a research expert. Respond with valid JSON only."},
|
|
{"role": "user", "content": reportPrompt}
|
|
])
|
|
|
|
# Convert to HTML if needed
|
|
if formatType == "html" and not reportContent.lower().startswith("<html"):
|
|
# Check if it's markdown that needs conversion
|
|
if reportContent.startswith("#"):
|
|
reportContent = markdown.markdown(reportContent)
|
|
# Wrap in basic HTML structure if needed
|
|
if not reportContent.lower().startswith("<html"):
|
|
reportContent = f"<html><head><title>Web Research Results</title></head><body>{reportContent}</body></html>"
|
|
|
|
return self.formatAgentDocumentOutput(outputLabel, reportContent, contentType)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error creating narrative document: {str(e)}")
|
|
# Create error document
|
|
if formatType == "markdown":
|
|
content = f"# Web Research Error\n\nAn error occurred: {str(e)}"
|
|
elif formatType == "html":
|
|
content = f"<html><body><h1>Web Research Error</h1><p>An error occurred: {str(e)}</p></body></html>"
|
|
else:
|
|
content = f"WEB RESEARCH ERROR\n\nAn error occurred: {str(e)}"
|
|
|
|
return self.formatAgentDocumentOutput(outputLabel, content, contentType)
|
|
|
|
async def _createJsonDocument(self, prompt: str, results: List[Dict[str, Any]],
|
|
researchPlan: Dict[str, Any], outputLabel: str) -> Dict[str, Any]:
|
|
"""
|
|
Create a JSON document from research results.
|
|
|
|
Args:
|
|
prompt: Original research prompt
|
|
results: Research results
|
|
researchPlan: Research plan
|
|
outputLabel: Output filename
|
|
|
|
Returns:
|
|
Document object
|
|
"""
|
|
try:
|
|
# Create structured data
|
|
sourcesData = []
|
|
for result in results:
|
|
sourcesData.append({
|
|
"title": result.get("title", "Untitled"),
|
|
"url": result.get("url", ""),
|
|
"summary": result.get("summary", ""),
|
|
"snippet": result.get("snippet", ""),
|
|
"sourceType": result.get("sourceType", "")
|
|
})
|
|
|
|
# Create metadata
|
|
metadata = {
|
|
"query": prompt,
|
|
"timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
|
|
"researchQuestions": researchPlan.get("researchQuestions", []),
|
|
"searchTerms": researchPlan.get("searchTerms", [])
|
|
}
|
|
|
|
# Compile complete report object
|
|
jsonContent = {
|
|
"metadata": metadata,
|
|
"summary": researchPlan.get("feedback", "Web research results"),
|
|
"sources": sourcesData
|
|
}
|
|
|
|
# Convert to JSON string
|
|
content = json.dumps(jsonContent, indent=2)
|
|
|
|
return self.formatAgentDocumentOutput(outputLabel, content, "application/json")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error creating JSON document: {str(e)}")
|
|
return self.formatAgentDocumentOutput(outputLabel, json.dumps({"error": str(e)}), "application/json")
|
|
|
|
async def _createCsvDocument(self, results: List[Dict[str, Any]], outputLabel: str) -> Dict[str, Any]:
|
|
"""
|
|
Create a CSV document from research results.
|
|
|
|
Args:
|
|
results: Research results
|
|
outputLabel: Output filename
|
|
|
|
Returns:
|
|
Document object
|
|
"""
|
|
try:
|
|
# Create CSV header
|
|
csvLines = ["Title,URL,Source Type,Snippet"]
|
|
|
|
# Add results
|
|
for result in results:
|
|
# Escape CSV fields
|
|
title = result.get("title", "").replace('"', '""')
|
|
url = result.get("url", "").replace('"', '""')
|
|
sourceType = result.get("sourceType", "").replace('"', '""')
|
|
snippet = result.get("snippet", "").replace('"', '""')
|
|
|
|
csvLines.append(f'"{title}","{url}","{sourceType}","{snippet}"')
|
|
|
|
# Combine into CSV content
|
|
content = "\n".join(csvLines)
|
|
|
|
return self.formatAgentDocumentOutput(outputLabel, content, "text/csv")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error creating CSV document: {str(e)}")
|
|
return self.formatAgentDocumentOutput(outputLabel, "Error,Error\nFailed to create CSV,{0}".format(str(e)), "text/csv")
|
|
|
|
def _determineFormatType(self, outputLabel: str) -> str:
|
|
"""
|
|
Determine the format type based on the filename.
|
|
|
|
Args:
|
|
outputLabel: Output filename
|
|
|
|
Returns:
|
|
Format type (markdown, html, text, json, csv)
|
|
"""
|
|
outputLabelLower = outputLabel.lower()
|
|
|
|
if outputLabelLower.endswith(".md"):
|
|
return "markdown"
|
|
elif outputLabelLower.endswith(".html"):
|
|
return "html"
|
|
elif outputLabelLower.endswith(".txt"):
|
|
return "text"
|
|
elif outputLabelLower.endswith(".json"):
|
|
return "json"
|
|
elif outputLabelLower.endswith(".csv"):
|
|
return "csv"
|
|
else:
|
|
# Default to markdown
|
|
return "markdown"
|
|
|
|
def _searchWeb(self, query: str) -> List[Dict[str, str]]:
|
|
"""
|
|
Conduct a web search using SerpAPI and return the results.
|
|
|
|
Args:
|
|
query: The search query
|
|
|
|
Returns:
|
|
List of search results
|
|
"""
|
|
if not self.srcApikey:
|
|
return []
|
|
|
|
# Get user language from serviceBase if available
|
|
userLanguage = "en" # Default language
|
|
if self.service.base.userLanguage:
|
|
userLanguage = self.service.base.userLanguage
|
|
|
|
try:
|
|
# Format the search request for SerpAPI
|
|
params = {
|
|
"engine": self.srcEngine,
|
|
"q": query,
|
|
"api_key": self.srcApikey,
|
|
"num": self.maxResults, # Number of results to return
|
|
"hl": userLanguage # Identified user language
|
|
}
|
|
|
|
# Make the API request
|
|
response = requests.get("https://serpapi.com/search", params=params, timeout=self.timeout)
|
|
response.raise_for_status()
|
|
|
|
# Parse JSON response
|
|
search_results = response.json()
|
|
|
|
# Extract organic results
|
|
results = []
|
|
|
|
if "organic_results" in search_results:
|
|
for result in search_results["organic_results"][:self.maxResults]:
|
|
# Extract title
|
|
title = result.get("title", "No title")
|
|
|
|
# Extract URL
|
|
url = result.get("link", "No URL")
|
|
|
|
# Extract snippet
|
|
snippet = result.get("snippet", "No description")
|
|
|
|
# Get actual page content
|
|
try:
|
|
targetPageSoup = self._readUrl(url)
|
|
content = self._extractMainContent(targetPageSoup)
|
|
except Exception as e:
|
|
logger.warning(f"Error extracting content from {url}: {str(e)}")
|
|
content = f"Error extracting content: {str(e)}"
|
|
|
|
results.append({
|
|
'title': title,
|
|
'url': url,
|
|
'snippet': snippet,
|
|
'data': content
|
|
})
|
|
|
|
# Limit number of results
|
|
if len(results) >= self.maxResults:
|
|
break
|
|
else:
|
|
logger.warning(f"No organic results found in SerpAPI response for: {query}")
|
|
|
|
return results
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error searching with SerpAPI for {query}: {str(e)}")
|
|
return []
|
|
|
|
def _readUrl(self, url: str) -> BeautifulSoup:
|
|
"""
|
|
Read a URL and return a BeautifulSoup parser for the content.
|
|
|
|
Args:
|
|
url: The URL to read
|
|
|
|
Returns:
|
|
BeautifulSoup object with the content or None on errors
|
|
"""
|
|
if not url or not url.startswith(('http://', 'https://')):
|
|
return None
|
|
|
|
headers = {
|
|
'User-Agent': self.userAgent,
|
|
'Accept': 'text/html,application/xhtml+xml,application/xml',
|
|
'Accept-Language': 'en-US,en;q=0.9',
|
|
}
|
|
|
|
try:
|
|
# Initial request
|
|
response = requests.get(url, headers=headers, timeout=self.timeout)
|
|
|
|
# Handling for status 202
|
|
if response.status_code == 202:
|
|
# Retry with backoff
|
|
backoffTimes = [0.5, 1.0, 2.0, 5.0]
|
|
|
|
for waitTime in backoffTimes:
|
|
time.sleep(waitTime)
|
|
response = requests.get(url, headers=headers, timeout=self.timeout)
|
|
|
|
if response.status_code != 202:
|
|
break
|
|
|
|
# Raise for error status codes
|
|
response.raise_for_status()
|
|
|
|
# Parse HTML
|
|
return BeautifulSoup(response.text, 'html.parser')
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error reading URL {url}: {str(e)}")
|
|
return None
|
|
|
|
def _extractTitle(self, soup: BeautifulSoup, url: str) -> str:
|
|
"""
|
|
Extract the title from a webpage.
|
|
|
|
Args:
|
|
soup: BeautifulSoup object of the webpage
|
|
url: URL of the webpage
|
|
|
|
Returns:
|
|
Extracted title
|
|
"""
|
|
if not soup:
|
|
return f"Error with {url}"
|
|
|
|
# Extract title from title tag
|
|
titleTag = soup.find('title')
|
|
title = titleTag.text.strip() if titleTag else "No title"
|
|
|
|
# Alternative: Also look for h1 tags if title tag is missing
|
|
if title == "No title":
|
|
h1Tag = soup.find('h1')
|
|
if h1Tag:
|
|
title = h1Tag.text.strip()
|
|
|
|
return title
|
|
|
|
def _extractMainContent(self, soup: BeautifulSoup, maxChars: int = 10000) -> str:
|
|
"""
|
|
Extract the main content from an HTML page.
|
|
|
|
Args:
|
|
soup: BeautifulSoup object of the webpage
|
|
maxChars: Maximum number of characters
|
|
|
|
Returns:
|
|
Extracted main content as a string
|
|
"""
|
|
if not soup:
|
|
return ""
|
|
|
|
# Try to find main content elements in priority order
|
|
mainContent = None
|
|
for selector in ['main', 'article', '#content', '.content', '#main', '.main']:
|
|
content = soup.select_one(selector)
|
|
if content:
|
|
mainContent = content
|
|
break
|
|
|
|
# If no main content found, use the body
|
|
if not mainContent:
|
|
mainContent = soup.find('body') or soup
|
|
|
|
# Remove script, style, nav, footer elements that don't contribute to main content
|
|
for element in mainContent.select('script, style, nav, footer, header, aside, .sidebar, #sidebar, .comments, #comments, .advertisement, .ads, iframe'):
|
|
element.extract()
|
|
|
|
# Extract text content
|
|
textContent = mainContent.get_text(separator=' ', strip=True)
|
|
|
|
# Limit to maxChars
|
|
return textContent[:maxChars]
|
|
|
|
def _limitText(self, text: str, maxChars: int = 10000) -> str:
|
|
"""
|
|
Limit text to a maximum number of characters.
|
|
|
|
Args:
|
|
text: Input text
|
|
maxChars: Maximum number of characters
|
|
|
|
Returns:
|
|
Limited text
|
|
"""
|
|
if not text:
|
|
return ""
|
|
|
|
# If text is already under the limit, return unchanged
|
|
if len(text) <= maxChars:
|
|
return text
|
|
|
|
# Otherwise limit text to maxChars
|
|
return text[:maxChars] + "... [Content truncated due to length]"
|
|
|
|
|
|
# Factory function for the Webcrawler agent
|
|
def getAgentWebcrawler():
|
|
"""Returns an instance of the Webcrawler agent."""
|
|
return AgentWebcrawler() |