# Copyright (c) 2025 Patrick Motsch
# All rights reserved.
"""
Web crawl service for handling web research operations.
Manages the two-step process: WEB_SEARCH_DATA then WEB_CRAWL.
"""

import json
import logging
import time
import asyncio
from urllib.parse import urlparse
from typing import Dict, Any, List, Optional
from modules.datamodels.datamodelAi import AiCallOptions, OperationTypeEnum, AiCallPromptWebSearch, AiCallPromptWebCrawl

logger = logging.getLogger(__name__)


class WebService:
    """Service for web search and crawling operations."""
    
    def __init__(self, services):
        """Initialize webcrawl service with service center access."""
        self.services = services
    
    async def performWebResearch(
        self,
        prompt: str,
        urls: List[str],
        country: Optional[str],
        language: Optional[str],
        researchDepth: str = "general",
        operationId: str = None
        ) -> Dict[str, Any]:
        """
        Perform web research in two steps:
        1. Use AI to analyze prompt and extract parameters + URLs
        2. Call WEB_SEARCH_DATA to get URLs (if needed)
        3. Combine URLs and filter to maxNumberPages
        4. Call WEB_CRAWL for each URL
        5. Return consolidated result
        
        Args:
            prompt: Natural language research prompt
            urls: Optional list of URLs provided by user
            country: Optional country code
            language: Optional language code
            operationId: Operation ID for progress tracking
            
        Returns:
            Consolidated research results as dictionary
        """
        # Start progress tracking if operationId provided
        if operationId:
            self.services.chat.progressLogStart(
                operationId,
                "Web Research",
                "Research",
                f"Depth: {researchDepth}"
            )
        
        try:
            # Step 1: AI intention analysis - extract URLs and parameters from prompt
            if operationId:
                self.services.chat.progressLogUpdate(operationId, 0.1, "Analyzing research intent")
            
            analysisResult = await self._analyzeResearchIntent(prompt, urls, country, language, researchDepth)
            
            # Extract parameters from AI analysis
            instruction = analysisResult.get("instruction", prompt)
            extractedUrls = analysisResult.get("urls", [])
            needsSearch = analysisResult.get("needsSearch", True)  # Default to True
            maxNumberPages = analysisResult.get("maxNumberPages", 10)
            countryCode = analysisResult.get("country", country)
            languageCode = analysisResult.get("language", language)
            finalResearchDepth = analysisResult.get("researchDepth", researchDepth)
            suggestedFilename = analysisResult.get("filename", None)
            
            logger.info(f"AI Analysis: instruction='{instruction[:100]}...', urls={len(extractedUrls)}, needsSearch={needsSearch}, maxNumberPages={maxNumberPages}, researchDepth={finalResearchDepth}, filename={suggestedFilename}")
            
            # Combine URLs (from user + from prompt extraction)
            allUrls = []
            if urls:
                allUrls.extend(urls)
            if extractedUrls:
                allUrls.extend(extractedUrls)
            
            # Step 2: Search for URLs and content if needed (based on needsSearch flag)
            searchUrls = []
            searchResultsWithContent = []
            if needsSearch and (not allUrls or len(allUrls) < maxNumberPages):
                self.services.chat.progressLogUpdate(operationId, 0.3, "Searching for URLs and content")
                
                try:
                    searchUrls, searchResultsWithContent = await self._performWebSearch(
                        instruction=instruction,
                        maxNumberPages=maxNumberPages - len(allUrls),
                        country=countryCode,
                        language=languageCode
                    )
                    logger.info(f"Tavily search returned {len(searchUrls)} URLs with {len(searchResultsWithContent)} results containing content")
                except Exception as e:
                    logger.error(f"Error performing Tavily search (continuing with other URLs): {str(e)}", exc_info=True)
                    searchUrls = []
                    searchResultsWithContent = []
                
                # Prioritize Tavily search URLs over AI-extracted URLs (they're more relevant)
                if searchUrls:
                    # Prepend Tavily URLs to the list (they're more relevant)
                    allUrls = searchUrls + allUrls
                    logger.info(f"Using {len(searchUrls)} Tavily URLs + {len(allUrls) - len(searchUrls)} other URLs = {len(allUrls)} total")
                else:
                    # If Tavily search failed, use AI-extracted URLs
                    logger.warning("Tavily search returned no URLs, using AI-extracted URLs only")
                
                self.services.chat.progressLogUpdate(operationId, 0.5, f"Found {len(allUrls)} total URLs")
            
            # If we have search results (even without content), use them directly instead of crawling
            # Tavily search results are more relevant than generic AI-extracted URLs
            # Only crawl if we have NO search results at all
            if searchResultsWithContent and len(searchResultsWithContent) > 0:
                urls_with_actual_content = sum(1 for r in searchResultsWithContent if r.get("content") and len(r.get("content", "")) > 0)
                logger.info(f"Using {len(searchResultsWithContent)} Tavily search results ({urls_with_actual_content} with content) directly (skipping crawl)")
                
                # Convert search results to crawl result format
                crawlResult = []
                for result in searchResultsWithContent:
                    crawlResult.append({
                        "url": result["url"],
                        "title": result.get("title", ""),
                        "content": result.get("content", "")
                    })
                
                # Calculate statistics
                totalResults = len(crawlResult)
                totalContentLength = sum(len(r.get("content", "")) for r in crawlResult)
                urlsWithContent = sum(1 for r in crawlResult if r.get("content") and len(r.get("content", "")) > 0)
                
                # Log content availability
                if urlsWithContent == 0:
                    logger.warning(f"Tavily search returned {len(searchResultsWithContent)} results but none have content - URLs will be used but may need crawling")
                else:
                    logger.info(f"Tavily search provided content for {urlsWithContent}/{len(searchResultsWithContent)} URLs")
                
                # Even if content is empty, use these results - they're more relevant than generic URLs
                # The final answer generation can work with URLs even if content is empty
                
                # Convert to sections format
                sections = []
                for idx, item in enumerate(crawlResult):
                    section = {
                        "id": f"result_{idx}",
                        "content_type": "paragraph",
                        "title": item.get("title") or item.get("url", f"Result {idx + 1}"),
                        "order": idx
                    }
                    content = item.get("content", "")
                    if content:
                        section["textPreview"] = content[:200] + ("..." if len(content) > 200 else "")
                    sections.append(section)
                
                # Return consolidated result
                result = {
                    "metadata": {
                        "title": suggestedFilename or instruction[:100] if instruction else "Web Research Results",
                        "extraction_method": "tavily_search_direct",
                        "research_depth": finalResearchDepth,
                        "country": countryCode,
                        "language": languageCode,
                        "urls_searched": searchUrls[:20],
                        "total_urls": len(searchUrls),
                        "urls_with_content": urlsWithContent,
                        "total_content_length": totalContentLength,
                        "search_date": self.services.utils.timestampGetUtc() if hasattr(self.services, 'utils') else None
                    },
                    "sections": sections,
                    "statistics": {
                        "sectionCount": len(sections),
                        "total_urls": len(searchUrls),
                        "results_count": totalResults,
                        "urls_with_content": urlsWithContent,
                        "total_content_length": totalContentLength
                    },
                    "instruction": instruction,
                    "urls_crawled": searchUrls,
                    "total_urls": len(searchUrls),
                    "results": crawlResult,
                    "total_results": totalResults
                }
                
                if suggestedFilename:
                    result["suggested_filename"] = suggestedFilename
                    result["metadata"]["suggested_filename"] = suggestedFilename
                
                if operationId:
                    self.services.chat.progressLogUpdate(operationId, 0.9, "Completed")
                    self.services.chat.progressLogFinish(operationId, True)
                
                return result
            
            # Step 3: Validate and filter URLs before crawling
            validatedUrls = self._validateUrls(allUrls)
            if not validatedUrls:
                logger.warning(f"All {len(allUrls)} URLs failed validation")
                return {"error": "No valid URLs found to crawl"}
            
            # Filter to maxNumberPages (simple cut, no intelligent filtering)
            if len(validatedUrls) > maxNumberPages:
                validatedUrls = validatedUrls[:maxNumberPages]
                logger.info(f"Limited URLs to {maxNumberPages}")
            
            if not validatedUrls:
                return {"error": "No URLs found to crawl"}
            
            # Step 4: Translate researchDepth to maxDepth and maxWidth
            depthMap = {"fast": 1, "general": 2, "deep": 3}
            maxDepth = depthMap.get(finalResearchDepth.lower(), 2)
            # Scale maxWidth based on research depth: fast=5, general=10, deep=20 pages per level
            widthMap = {"fast": 5, "general": 10, "deep": 20}
            maxWidth = widthMap.get(finalResearchDepth.lower(), 10)
            
            logger.info(f"Research depth settings: depth={finalResearchDepth}, maxDepth={maxDepth}, maxWidth={maxWidth}")
            
            # Step 5: Crawl all URLs with hierarchical logging
            if operationId:
                self.services.chat.progressLogUpdate(operationId, 0.4, "Initiating")
                self.services.chat.progressLogUpdate(operationId, 0.6, f"Crawling {len(validatedUrls)} URLs")
            
            # Use parent operation ID directly (parentId should be operationId, not log entry ID)
            parentOperationId = operationId  # Use the parent's operationId directly
            
            crawlResult = await self._performWebCrawl(
                instruction=instruction,
                urls=validatedUrls,
                maxDepth=maxDepth,
                maxWidth=maxWidth,  # Pass maxWidth to crawl function
                parentOperationId=parentOperationId
            )
            
            if operationId:
                self.services.chat.progressLogUpdate(operationId, 0.9, "Consolidating results")
                self.services.chat.progressLogUpdate(operationId, 0.95, "Completed")
                self.services.chat.progressLogFinish(operationId, True)
            
            # Calculate statistics about crawl results
            totalResults = len(crawlResult) if isinstance(crawlResult, list) else 1
            totalContentLength = 0
            urlsWithContent = 0
            
            # Analyze crawl results to gather statistics
            if isinstance(crawlResult, list):
                for item in crawlResult:
                    if isinstance(item, dict):
                        if item.get("url"):
                            urlsWithContent += 1
                        content = item.get("content", "")
                        if isinstance(content, str):
                            totalContentLength += len(content)
                        elif isinstance(content, dict):
                            # Estimate size from dict
                            totalContentLength += len(str(content))
            elif isinstance(crawlResult, dict):
                if crawlResult.get("url"):
                    urlsWithContent = 1
                content = crawlResult.get("content", "")
                if isinstance(content, str):
                    totalContentLength = len(content)
                elif isinstance(content, dict):
                    totalContentLength = len(str(content))
            
            # Convert crawl results into sections format for generic validator
            sections = []
            if isinstance(crawlResult, list):
                for idx, item in enumerate(crawlResult):
                    if isinstance(item, dict):
                        section = {
                            "id": f"result_{idx}",
                            "content_type": "paragraph",
                            "title": item.get("url", f"Result {idx + 1}"),
                            "order": idx
                        }
                        # Add content preview
                        content = item.get("content", "")
                        if isinstance(content, str) and content:
                            section["textPreview"] = content[:200] + ("..." if len(content) > 200 else "")
                        sections.append(section)
            elif isinstance(crawlResult, dict):
                section = {
                    "id": "result_0",
                    "content_type": "paragraph",
                    "title": crawlResult.get("url", "Research Result"),
                    "order": 0
                }
                content = crawlResult.get("content", "")
                if isinstance(content, str) and content:
                    section["textPreview"] = content[:200] + ("..." if len(content) > 200 else "")
                sections.append(section)
            
            # Return consolidated result with metadata in format that generic validator understands
            result = {
                "metadata": {
                    "title": suggestedFilename or instruction[:100] if instruction else "Web Research Results",
                    "extraction_method": "web_crawl",
                    "research_depth": finalResearchDepth,
                    "max_depth": maxDepth,
                    "country": countryCode,
                    "language": languageCode,
                    "urls_crawled": validatedUrls[:20],  # First 20 URLs for reference
                    "total_urls": len(validatedUrls),
                    "urls_with_content": urlsWithContent,
                    "total_content_length": totalContentLength,
                    "crawl_date": self.services.utils.timestampGetUtc() if hasattr(self.services, 'utils') else None
                },
                "sections": sections,
                "statistics": {
                "sectionCount": len(sections),
                "total_urls": len(validatedUrls),
                    "results_count": totalResults,
                    "urls_with_content": urlsWithContent,
                    "total_content_length": totalContentLength
                },
                # Keep original structure for backward compatibility
                "instruction": instruction,
                "urls_crawled": validatedUrls,
                "total_urls": len(validatedUrls),
                "results": crawlResult,
                "total_results": totalResults
            }
            
            # Add suggested filename if available
            if suggestedFilename:
                result["suggested_filename"] = suggestedFilename
                result["metadata"]["suggested_filename"] = suggestedFilename
            
            return result
            
        except Exception as e:
            logger.error(f"Error in web research: {str(e)}")
            if operationId:
                self.services.chat.progressLogFinish(operationId, False)
            raise
    
    async def _analyzeResearchIntent(
        self,
        prompt: str,
        urls: List[str],
        country: Optional[str],
        language: Optional[str],
        researchDepth: str = "general"
        ) -> Dict[str, Any]:
        """
        Use AI to analyze prompt and extract:
        - URLs from the prompt text
        - Research instruction
        - maxNumberPages, timeRange, country, language from context
        """
        # Build analysis prompt for AI
        analysisPrompt = f"""Analyze this web research request and extract structured information.

RESEARCH REQUEST:
{prompt}

USER PROVIDED:
- URLs: {json.dumps(urls) if urls else "None"}
- Country: {country or "Not specified"}
- Language: {language or "Not specified"}

Extract and provide a JSON response with:
1. instruction: Formulate directly, WHAT you want to find on the web. Do not include URLs in the instruction. Good example: "What is the company Xyz doing?". Bad example: "Conduct web research on the company Xyz"
2. urls: Put list of URLs found in the prompt text, and URL's you know, that are relevant to the research
3. needsSearch: true if web search is needed to identify url's to crawl, false if only crawling of provided URLs is wanted
4. maxNumberPages: Recommended number of URLs to crawl (based on research scope, typical: 2-20)
5. country: Country code if identified in the prompt (2-digit lowercase, e.g., ch, us, de)
6. language: Language identified from the prompt (lowercase, e.g., de, en, fr)
7. researchDepth: Research depth based on instruction complexity - "fast" (quick overview, maxDepth=1), "general" (standard research, maxDepth=2), or "deep" (comprehensive research, maxDepth=3)
8. filename: Generate a concise, descriptive filename (without extension) for the research results. Should be short (max 50 characters), descriptive of the research topic, use underscores instead of spaces, and only contain alphanumeric characters and underscores. Example: "WebResearch_Topic_Context"

Return ONLY valid JSON, no additional text:
{{
    "instruction": "research instruction",
    "urls": ["url1", "url2"],
    "needsSearch": true,
    "maxNumberPages": 10,
    "country": "ch",
    "language": "en",
    "researchDepth": "general",
    "filename": "descriptive_filename_without_extension"
}}"""

        try:
            # Call AI planning to analyze intent
            analysisJson = await self.services.ai.callAiPlanning(
                analysisPrompt,
                debugType="webresearchintent"
            )
            
            # Extract JSON from response (handles markdown code blocks)
            extractedJson = self.services.utils.jsonExtractString(analysisJson)
            if not extractedJson:
                raise ValueError("No JSON found in AI response")
            
            # Parse JSON response
            result = json.loads(extractedJson)
            
            logger.info(f"Intent analysis result: {result}")
            return result
            
        except Exception as e:
            logger.warning(f"Error in AI intent analysis: {str(e)}")
            # Fallback to basic extraction
            return {
                "instruction": prompt,
                "urls": [],
                "needsSearch": True,
                "maxNumberPages": 10,
                "country": country,
                "language": language,
                "researchDepth": researchDepth,
                "filename": None
            }
    
    async def _performWebSearch(
        self,
        instruction: str,
        maxNumberPages: int,
        country: Optional[str],
        language: Optional[str]
        ) -> tuple[List[str], List[Dict[str, Any]]]:
        """
        Perform web search to find URLs and content.
        
        Returns:
            Tuple of (urls, search_results_with_content)
            - urls: List of URL strings
            - search_results_with_content: List of dicts with url, title, content from Tavily search
        """
        search_results_with_content = []
        try:
            # Build search prompt model
            searchPromptModel = AiCallPromptWebSearch(
                instruction=instruction,
                country=country,
                maxNumberPages=maxNumberPages,
                language=language
            )
            searchPrompt = searchPromptModel.model_dump_json(exclude_none=True, indent=2)
            
            # Debug: persist search prompt
            self.services.utils.writeDebugFile(searchPrompt, "websearch_prompt")
            
            # Call AI with WEB_SEARCH_DATA operation
            searchOptions = AiCallOptions(
                operationType=OperationTypeEnum.WEB_SEARCH_DATA,
                resultFormat="json"
            )
            
            # Use unified callAiContent method
            searchResponse = await self.services.ai.callAiContent(
                prompt=searchPrompt,
                options=searchOptions,
                outputFormat="json"
            )
            
            # Check if metadata contains results with content (from Tavily)
            if hasattr(searchResponse, 'metadata') and searchResponse.metadata:
                # Check in additionalData first (where we store custom metadata)
                additional_data = None
                if hasattr(searchResponse.metadata, 'additionalData') and searchResponse.metadata.additionalData:
                    additional_data = searchResponse.metadata.additionalData
                elif isinstance(searchResponse.metadata, dict):
                    additional_data = searchResponse.metadata.get("additionalData", {})
                
                if additional_data:
                    results_with_content = additional_data.get("results_with_content", [])
                    if results_with_content:
                        logger.info(f"Found {len(results_with_content)} search results with content in metadata.additionalData")
                        # Extract URLs and content from metadata
                        for result in results_with_content:
                            if result.get("url"):
                                search_results_with_content.append({
                                    "url": result.get("url"),
                                    "title": result.get("title", ""),
                                    "content": result.get("content", ""),
                                    "score": result.get("score", 0)
                                })
                
                # Also check directly in metadata (fallback)
                if not search_results_with_content:
                    results_with_content = None
                    if hasattr(searchResponse.metadata, 'results_with_content'):
                        results_with_content = searchResponse.metadata.results_with_content
                    elif isinstance(searchResponse.metadata, dict):
                        results_with_content = searchResponse.metadata.get("results_with_content", [])
                    
                    if results_with_content:
                        logger.info(f"Found {len(results_with_content)} search results with content in metadata (direct)")
                        for result in results_with_content:
                            if result.get("url"):
                                search_results_with_content.append({
                                    "url": result.get("url"),
                                    "title": result.get("title", ""),
                                    "content": result.get("content", ""),
                                    "score": result.get("score", 0)
                                })
            
            # Extract content from AiResponse
            searchResult = searchResponse.content
            
            logger.debug(f"Search response content type: {type(searchResult)}, length: {len(str(searchResult)) if searchResult else 0}")
            
            # Debug: persist search response
            if isinstance(searchResult, str):
                self.services.utils.writeDebugFile(searchResult, "websearch_response")
                logger.debug(f"Search response (first 500 chars): {searchResult[:500]}")
            else:
                self.services.utils.writeDebugFile(json.dumps(searchResult, indent=2), "websearch_response")
                logger.debug(f"Search response type: {type(searchResult)}, keys: {list(searchResult.keys()) if isinstance(searchResult, dict) else 'N/A'}")
            
            # Parse and extract URLs and content
            if isinstance(searchResult, str):
                # Extract JSON from response (handles markdown code blocks)
                extractedJson = self.services.utils.jsonExtractString(searchResult)
                if extractedJson:
                    try:
                        searchData = json.loads(extractedJson)
                        logger.debug(f"Parsed JSON from extracted string, type: {type(searchData)}")
                    except json.JSONDecodeError as e:
                        logger.warning(f"Failed to parse extracted JSON: {e}, trying direct parse")
                        searchData = json.loads(searchResult)
                else:
                    try:
                        searchData = json.loads(searchResult)
                        logger.debug(f"Parsed JSON directly from string, type: {type(searchData)}")
                    except json.JSONDecodeError as e:
                        logger.error(f"Failed to parse search result as JSON: {e}")
                        searchData = {}
            else:
                searchData = searchResult
                logger.debug(f"Using searchResult directly as searchData, type: {type(searchData)}")
            
            logger.debug(f"Final searchData type: {type(searchData)}, is dict: {isinstance(searchData, dict)}, keys: {list(searchData.keys()) if isinstance(searchData, dict) else 'N/A'}")
            
            # Extract URLs and content from response
            urls = []
            if isinstance(searchData, dict):
                # Check for new format: {"urls": [...], "results": [...]}
                if "urls" in searchData and "results" in searchData:
                    urls = searchData["urls"]
                    # Extract results with content
                    for r in searchData["results"]:
                        if r.get("url"):
                            # Only add if not already added from metadata
                            if not any(sr["url"] == r.get("url") for sr in search_results_with_content):
                                search_results_with_content.append({
                                    "url": r.get("url"),
                                    "title": r.get("title", ""),
                                    "content": r.get("content", ""),
                                    "score": r.get("score", 0)
                                })
                    logger.debug(f"Extracted {len(urls)} URLs and {len(search_results_with_content)} results with content from new format")
                elif "urls" in searchData:
                    urls = searchData["urls"]
                    logger.debug(f"Extracted {len(urls)} URLs from 'urls' field")
                elif "results" in searchData:
                    # Extract URLs from results (content already in search_results_with_content if from metadata)
                    for r in searchData["results"]:
                        if r.get("url"):
                            urls.append(r.get("url"))
                            # Only add to search_results_with_content if not already added from metadata
                            if not any(sr["url"] == r.get("url") for sr in search_results_with_content):
                                search_results_with_content.append({
                                    "url": r.get("url"),
                                    "title": r.get("title", ""),
                                    "content": r.get("raw_content") or r.get("content", ""),
                                    "score": r.get("score", 0)
                                })
                    logger.debug(f"Extracted {len(urls)} URLs with content from 'results' field")
                else:
                    logger.warning(f"Unexpected search data structure (dict): {list(searchData.keys())}")
            elif isinstance(searchData, list):
                # Handle both cases: list of URL strings or list of dicts with "url" key
                for item in searchData:
                    if isinstance(item, str):
                        # Item is already a URL string
                        urls.append(item)
                    elif isinstance(item, dict):
                        if item.get("url"):
                            urls.append(item.get("url"))
                            # Only add to search_results_with_content if not already added from metadata
                            if not any(sr["url"] == item.get("url") for sr in search_results_with_content):
                                search_results_with_content.append({
                                    "url": item.get("url"),
                                    "title": item.get("title", ""),
                                    "content": item.get("raw_content") or item.get("content", ""),
                                    "score": item.get("score", 0)
                                })
                logger.debug(f"Extracted {len(urls)} URLs from list")
            else:
                logger.warning(f"Unexpected search data type: {type(searchData)}")
            
            # If we got URLs but no content from metadata, extract URLs from search_results_with_content
            if urls and not search_results_with_content:
                # URLs were extracted but no content - create entries with empty content
                for url in urls:
                    search_results_with_content.append({
                        "url": url,
                        "title": "",
                        "content": "",
                        "score": 0
                    })
            elif search_results_with_content and not urls:
                # We have content but no URLs - extract URLs from content results
                urls = [r["url"] for r in search_results_with_content]
            
            # If we have URLs but no search_results_with_content, create entries from URLs
            # This ensures we can use Tavily URLs even if content extraction failed
            if urls and not search_results_with_content:
                logger.warning("Got URLs from Tavily search but no content extracted - creating entries for direct use")
                for url in urls:
                    search_results_with_content.append({
                        "url": url,
                        "title": "",
                        "content": "",  # Empty content - will need crawling if used
                        "score": 0
                    })
            
            logger.info(f"Web search returned {len(urls)} URLs with {len(search_results_with_content)} results")
            if search_results_with_content:
                content_count = sum(1 for r in search_results_with_content if r.get("content") and len(r.get("content", "")) > 0)
                logger.info(f"  - {content_count} results have content, {len(search_results_with_content) - content_count} without content")
                if content_count > 0:
                    first_with_content = next((r for r in search_results_with_content if r.get("content")), None)
                    if first_with_content:
                        logger.info(f"Content preview from first result with content: {first_with_content.get('content', '')[:200]}")
            else:
                logger.warning("No search results extracted - will need to crawl URLs")
            return urls, search_results_with_content
            
        except Exception as e:
            logger.error(f"Error in web search: {str(e)}", exc_info=True)
            # Even if there's an error, try to extract URLs from the response if available
            recovered_urls = []
            recovered_results = []
            try:
                if 'searchResponse' in locals() and searchResponse:
                    logger.info(f"Attempting to extract URLs from error response: {type(searchResponse)}")
                    # Try to get content from response
                    if hasattr(searchResponse, 'content'):
                        errorContent = searchResponse.content
                        if isinstance(errorContent, str):
                            # Try to parse as JSON
                            try:
                                errorData = json.loads(errorContent)
                                if isinstance(errorData, dict):
                                    if "urls" in errorData:
                                        recovered_urls = errorData["urls"]
                                    elif "results" in errorData:
                                        recovered_urls = [r.get("url") for r in errorData["results"] if r.get("url")]
                                        recovered_results = [{"url": r.get("url"), "title": r.get("title", ""), "content": r.get("content", ""), "score": 0} for r in errorData["results"]]
                                elif isinstance(errorData, list):
                                    recovered_urls = [item if isinstance(item, str) else item.get("url", "") for item in errorData if item]
                                if recovered_urls:
                                    logger.info(f"Recovered {len(recovered_urls)} URLs from error response")
                                    # Create entries for recovered URLs
                                    if not recovered_results:
                                        for url in recovered_urls:
                                            recovered_results.append({"url": url, "title": "", "content": "", "score": 0})
                                    return recovered_urls, recovered_results
                            except Exception as parseError:
                                logger.debug(f"Failed to parse error response: {parseError}")
            except Exception as recoverError:
                logger.debug(f"Failed to recover URLs from error: {recoverError}")
            
            # If we have URLs from earlier extraction, return them
            if 'urls' in locals() and urls:
                logger.info(f"Returning {len(urls)} URLs extracted before error occurred")
                # Create entries from URLs
                results_from_urls = [{"url": url, "title": "", "content": "", "score": 0} for url in urls]
                return urls, results_from_urls
            
            return [], []
    
    def _validateUrls(self, urls: List[str]) -> List[str]:
        """
        Validate URLs before crawling - filters out invalid URLs.
        
        Args:
            urls: List of URLs to validate
            
        Returns:
            List of valid URLs
        """
        validatedUrls = []
        for url in urls:
            if not url or not isinstance(url, str):
                logger.debug(f"Skipping invalid URL (not a string): {url}")
                continue
            
            url = url.strip()
            if not url:
                logger.debug(f"Skipping empty URL")
                continue
            
            # Basic URL validation using urlparse
            try:
                parsed = urlparse(url)
                # Check if URL has at least scheme and netloc
                if not parsed.scheme or not parsed.netloc:
                    logger.debug(f"Skipping invalid URL (missing scheme or netloc): {url}")
                    continue
                
                # Only allow http/https schemes
                if parsed.scheme not in ['http', 'https']:
                    logger.debug(f"Skipping URL with unsupported scheme '{parsed.scheme}': {url}")
                    continue
                
                validatedUrls.append(url)
                logger.debug(f"Validated URL: {url}")
                
            except Exception as e:
                logger.warning(f"Error validating URL '{url}': {str(e)}")
                continue
        
        logger.info(f"Validated {len(validatedUrls)}/{len(urls)} URLs")
        return validatedUrls
    
    async def _performWebCrawl(
        self,
        instruction: str,
        urls: List[str],
        maxDepth: int = 2,
        maxWidth: int = 10,
        parentOperationId: Optional[str] = None
        ) -> List[Dict[str, Any]]:
        """Perform web crawl on list of URLs - crawls URLs in parallel for better performance."""
        # Create tasks for parallel crawling
        crawlTasks = []
        for urlIndex, url in enumerate(urls):
            task = self._crawlSingleUrl(
                url=url,
                urlIndex=urlIndex,
                totalUrls=len(urls),
                instruction=instruction,
                maxDepth=maxDepth,
                maxWidth=maxWidth,  # Pass maxWidth to single URL crawl
                parentOperationId=parentOperationId
            )
            crawlTasks.append(task)
        
        # Execute all crawl tasks in parallel
        logger.info(f"Starting parallel crawl of {len(urls)} URLs")
        crawlResults = await asyncio.gather(*crawlTasks, return_exceptions=True)
        
        # Process results and handle exceptions
        processedResults = []
        for idx, result in enumerate(crawlResults):
            if isinstance(result, Exception):
                logger.error(f"Error crawling URL {urls[idx]}: {str(result)}")
                processedResults.append({"url": urls[idx], "error": str(result)})
            else:
                processedResults.extend(result if isinstance(result, list) else [result])
        
        logger.info(f"Completed parallel crawl: {len(processedResults)} results")
        return processedResults
    
    async def _crawlSingleUrl(
        self,
        url: str,
        urlIndex: int,
        totalUrls: int,
        instruction: str,
        maxDepth: int,
        maxWidth: int = 10,
        parentOperationId: Optional[str] = None
        ) -> List[Dict[str, Any]]:
        """
        Crawl a single URL - called in parallel for multiple URLs.
        
        Args:
            url: URL to crawl
            urlIndex: Index of URL in the list
            totalUrls: Total number of URLs being crawled
            instruction: Research instruction
            maxDepth: Maximum crawl depth
            parentOperationId: Parent operation ID for progress tracking
            
        Returns:
            List of crawl results for this URL
        """
        # Create separate operation for each URL with parent reference
        urlOperationId = None
        if parentOperationId:
            workflowId = self.services.workflow.id if self.services.workflow else f"no-workflow-{int(time.time())}"
            urlOperationId = f"web_crawl_url_{workflowId}_{urlIndex}_{int(time.time())}"
            self.services.chat.progressLogStart(
                urlOperationId,
                "Web Crawl",
                f"URL {urlIndex + 1}/{totalUrls}",
                url[:50] + "..." if len(url) > 50 else url,
                parentOperationId=parentOperationId
            )
        
        try:
            logger.info(f"Crawling URL {urlIndex + 1}/{totalUrls}: {url}")
            
            if urlOperationId:
                displayUrl = url[:50] + "..." if len(url) > 50 else url
                self.services.chat.progressLogUpdate(urlOperationId, 0.2, f"Crawling: {displayUrl}")
                self.services.chat.progressLogUpdate(urlOperationId, 0.3, "Initiating crawl")
            
            # Build crawl prompt model for single URL
            # maxWidth is passed from performWebResearch based on researchDepth
            logger.info(f"Crawling URL {urlIndex + 1}/{totalUrls} with maxDepth={maxDepth}, maxWidth={maxWidth}")
            crawlPromptModel = AiCallPromptWebCrawl(
                instruction=instruction,
                url=url,  # Single URL
                maxDepth=maxDepth,
                maxWidth=maxWidth  # Scaled based on researchDepth: fast=5, general=10, deep=20
            )
            crawlPrompt = crawlPromptModel.model_dump_json(exclude_none=True, indent=2)
            
            # Debug: persist crawl prompt (with URL identifier in content for clarity)
            debugPrompt = f"URL: {url}\n\n{crawlPrompt}"
            self.services.utils.writeDebugFile(debugPrompt, "webcrawl_prompt")
            
            # Call AI with WEB_CRAWL operation
            crawlOptions = AiCallOptions(
                operationType=OperationTypeEnum.WEB_CRAWL,
                resultFormat="json"
            )
            
            if urlOperationId:
                self.services.chat.progressLogUpdate(urlOperationId, 0.4, "Calling crawl connector")
            
            # Use unified callAiContent method with parentOperationId for hierarchical logging
            crawlResponse = await self.services.ai.callAiContent(
                prompt=crawlPrompt,
                options=crawlOptions,
                outputFormat="json",
                parentOperationId=urlOperationId  # Pass URL operation ID as parent for sub-URL logging
            )
            
            if urlOperationId:
                self.services.chat.progressLogUpdate(urlOperationId, 0.7, "Processing crawl results")
            
            # Extract content from AiResponse
            crawlResult = crawlResponse.content
            
            # Debug: persist crawl response
            if isinstance(crawlResult, str):
                self.services.utils.writeDebugFile(crawlResult, "webcrawl_response")
            else:
                self.services.utils.writeDebugFile(json.dumps(crawlResult, indent=2), "webcrawl_response")
            
            # Parse crawl result
            if isinstance(crawlResult, str):
                try:
                    # Extract JSON from response (handles markdown code blocks)
                    extractedJson = self.services.utils.jsonExtractString(crawlResult)
                    crawlData = json.loads(extractedJson) if extractedJson else json.loads(crawlResult)
                except:
                    crawlData = {"url": url, "content": crawlResult}
            else:
                crawlData = crawlResult
            
            # Process crawl results and create hierarchical progress logging for sub-URLs
            if urlOperationId:
                self.services.chat.progressLogUpdate(urlOperationId, 0.8, "Processing crawl results")
            
            # Recursively process crawl results to find nested URLs and create child operations
            processedResults = self._processCrawlResultsWithHierarchy(crawlData, url, urlOperationId, maxDepth, 0)
            
            # Count total URLs crawled (including sub-URLs) for progress message
            totalUrlsCrawled = self._countUrlsInResults(processedResults)
            
            # Ensure it's a list of results
            if isinstance(processedResults, list):
                results = processedResults
            elif isinstance(processedResults, dict):
                results = [processedResults]
            else:
                results = [{"url": url, "content": str(processedResults)}]
            
            if urlOperationId:
                if totalUrlsCrawled > 1:
                    self.services.chat.progressLogUpdate(urlOperationId, 0.9, f"Crawled {totalUrlsCrawled} URLs (including sub-URLs)")
                else:
                    self.services.chat.progressLogUpdate(urlOperationId, 0.9, "Crawl completed")
                self.services.chat.progressLogFinish(urlOperationId, True)
            
            return results
                
        except Exception as e:
            logger.error(f"Error crawling URL {url}: {str(e)}")
            if urlOperationId:
                self.services.chat.progressLogFinish(urlOperationId, False)
            return [{"url": url, "error": str(e)}]
    
    def _processCrawlResultsWithHierarchy(
        self,
        crawlData: Any,
        parentUrl: str,
        parentOperationId: Optional[str],
        maxDepth: int,
        currentDepth: int
    ) -> List[Dict[str, Any]]:
        """
        Recursively process crawl results to create hierarchical progress logging for sub-URLs.
        
        Args:
            crawlData: Crawl result data (dict, list, or other)
            parentUrl: Parent URL being crawled
            parentOperationId: Parent operation ID for hierarchical logging
            maxDepth: Maximum crawl depth
            currentDepth: Current depth in the crawl tree
        
        Returns:
            List of processed crawl results
        """
        import time
        results = []
        
        # Handle list of results
        if isinstance(crawlData, list):
            for idx, item in enumerate(crawlData):
                if isinstance(item, dict):
                    # Check if this item has sub-URLs or nested results
                    itemUrl = item.get("url") or item.get("source") or parentUrl
                    
                    # Create child operation for sub-URL if we're not at max depth
                    if currentDepth < maxDepth and parentOperationId:
                        # Check if this item has nested results or children
                        hasNestedResults = "results" in item or "children" in item or "subUrls" in item
                        
                        if hasNestedResults or (itemUrl != parentUrl and currentDepth > 0):
                            # This is a sub-URL - create child operation
                            workflowId = self.services.workflow.id if self.services.workflow else f"no-workflow-{int(time.time())}"
                            subUrlOperationId = f"{parentOperationId}_sub_{idx}_{int(time.time())}"
                            self.services.chat.progressLogStart(
                                subUrlOperationId,
                                "Crawling Sub-URL",
                                f"Depth {currentDepth + 1}",
                                itemUrl[:50] + "..." if len(itemUrl) > 50 else itemUrl,
                                parentOperationId=parentOperationId
                            )
                            
                            try:
                                # Process nested results recursively
                                if "results" in item:
                                    nestedResults = self._processCrawlResultsWithHierarchy(
                                        item["results"], itemUrl, subUrlOperationId, maxDepth, currentDepth + 1
                                    )
                                    item["results"] = nestedResults
                                elif "children" in item:
                                    nestedResults = self._processCrawlResultsWithHierarchy(
                                        item["children"], itemUrl, subUrlOperationId, maxDepth, currentDepth + 1
                                    )
                                    item["children"] = nestedResults
                                elif "subUrls" in item:
                                    nestedResults = self._processCrawlResultsWithHierarchy(
                                        item["subUrls"], itemUrl, subUrlOperationId, maxDepth, currentDepth + 1
                                    )
                                    item["subUrls"] = nestedResults
                                
                                self.services.chat.progressLogUpdate(subUrlOperationId, 0.9, "Completed")
                                self.services.chat.progressLogFinish(subUrlOperationId, True)
                            except Exception as e:
                                logger.error(f"Error processing sub-URL {itemUrl}: {str(e)}")
                                if subUrlOperationId:
                                    self.services.chat.progressLogFinish(subUrlOperationId, False)
                    
                    results.append(item)
                else:
                    results.append(item)
        
        # Handle dict with results array
        elif isinstance(crawlData, dict):
            if "results" in crawlData:
                # Process nested results
                nestedResults = self._processCrawlResultsWithHierarchy(
                    crawlData["results"], parentUrl, parentOperationId, maxDepth, currentDepth
                )
                crawlData["results"] = nestedResults
                results.append(crawlData)
            elif "children" in crawlData:
                # Process children
                nestedResults = self._processCrawlResultsWithHierarchy(
                    crawlData["children"], parentUrl, parentOperationId, maxDepth, currentDepth
                )
                crawlData["children"] = nestedResults
                results.append(crawlData)
            elif "subUrls" in crawlData:
                # Process sub-URLs
                nestedResults = self._processCrawlResultsWithHierarchy(
                    crawlData["subUrls"], parentUrl, parentOperationId, maxDepth, currentDepth
                )
                crawlData["subUrls"] = nestedResults
                results.append(crawlData)
            else:
                # Single result dict
                results.append(crawlData)
        else:
            # Other types - wrap in dict
            results.append({"url": parentUrl, "content": str(crawlData)})
        
        return results
    
    def _countUrlsInResults(self, results: Any) -> int:
        """
        Recursively count total URLs in crawl results (including nested sub-URLs).
        
        Args:
            results: Crawl results (dict, list, or other)
        
        Returns:
            Total count of URLs found
        """
        count = 0
        
        if isinstance(results, list):
            for item in results:
                count += self._countUrlsInResults(item)
        elif isinstance(results, dict):
            # Count this URL if it has a url field
            if "url" in results or "source" in results:
                count += 1
            # Recursively count nested results
            if "results" in results:
                count += self._countUrlsInResults(results["results"])
            if "children" in results:
                count += self._countUrlsInResults(results["children"])
            if "subUrls" in results:
                count += self._countUrlsInResults(results["subUrls"])
        elif isinstance(results, str):
            # Single URL string
            count = 1
        
        return count

