gateway/modules/workflow/managerChat.py

import asyncio
import logging
import uuid
import json
import time
from typing import Dict, Any, Optional, List, Union
from datetime import datetime, UTC

from modules.interfaces.interfaceAppModel import User
from modules.interfaces.interfaceChatModel import (
    TaskStatus, ChatDocument, TaskItem, TaskAction, TaskResult, ChatStat, ChatLog, ChatMessage, ChatWorkflow
)
from modules.workflow.serviceCenter import ServiceCenter
from modules.interfaces.interfaceChatObjects import ChatObjects

logger = logging.getLogger(__name__)

class ChatManager:
    """Chat manager with improved AI integration and method handling"""
    
    def __init__(self, currentUser: User, chatInterface: ChatObjects):
        self.currentUser = currentUser
        self.chatInterface = chatInterface
        self.service: ServiceCenter = None
        self.workflow: ChatWorkflow = None
        
        # Circuit breaker for AI calls
        self.ai_failure_count = 0
        self.ai_last_failure_time = None
        self.ai_circuit_breaker_threshold = 5
        self.ai_circuit_breaker_timeout = 300  # 5 minutes
        
        # Timeout settings
        self.ai_call_timeout = 120  # 2 minutes
        self.task_execution_timeout = 600  # 10 minutes

    # ===== Initialization and Setup =====
    async def initialize(self, workflow: ChatWorkflow) -> None:
        """Initialize chat manager with workflow"""
        self.workflow = workflow
        self.service = ServiceCenter(self.currentUser, self.workflow)

    # ===== WORKFLOW PHASES =====
    
    # Phase 1: High-Level Task Planning
    async def planHighLevelTasks(self, userInput: str, workflow: ChatWorkflow) -> Dict[str, Any]:
        """Phase 1: Plan high-level tasks from user input"""
        try:
            logger.info(f"Planning high-level tasks for workflow {workflow.id}")
            
            # Create planning prompt
            prompt = self._createTaskPlanningPrompt({
                'user_request': userInput,
                'available_documents': self._getAvailableDocuments(workflow),
                'workflow_id': workflow.id
            })
            
            # Get AI response
            response = await self.service.callAiTextAdvanced(prompt)
            
            # Parse and validate task plan
            task_plan = self._parseTaskPlanResponse(response)
            
            if not self._validateTaskPlan(task_plan):
                logger.warning("Generated task plan failed validation, using fallback")
                task_plan = self._createFallbackTaskPlan({
                    'user_request': userInput,
                    'available_documents': self._getAvailableDocuments(workflow)
                })
            
            logger.info(f"High-level task planning completed: {len(task_plan.get('tasks', []))} tasks")
            return task_plan
            
        except Exception as e:
            logger.error(f"Error in high-level task planning: {str(e)}")
            return self._createFallbackTaskPlan({
                'user_request': userInput,
                'available_documents': self._getAvailableDocuments(workflow)
            })
    
    # Phase 2: Task Definition and Action Generation
    async def defineTaskActions(self, task_step: Dict[str, Any], workflow: ChatWorkflow, previous_results: List[str] = None, 
                               enhanced_context: Dict[str, Any] = None) -> List[TaskAction]:
        """Phase 2: Define specific actions for a task step with enhanced retry context"""
        try:
            logger.info(f"Defining actions for task: {task_step.get('description', 'Unknown')}")
            
            # Use enhanced context if provided (for retries), otherwise create basic context
            if enhanced_context:
                context = enhanced_context
            else:
                context = {
                    'task_step': task_step,
                    'workflow': workflow,
                    'workflow_id': workflow.id,
                    'available_documents': self._getAvailableDocuments(workflow),
                    'previous_results': previous_results or [],
                    'improvements': None,
                    'retry_count': 0,
                    'previous_action_results': [],
                    'previous_review_result': None
                }
            
            # Generate actions using AI
            actions = await self._generateActionsForTaskStep(context)
            
            # Convert to TaskAction objects
            task_actions = []
            for action_dict in actions:
                action_data = {
                    "execMethod": action_dict.get('method', 'unknown'),
                    "execAction": action_dict.get('action', 'unknown'),
                    "execParameters": action_dict.get('parameters', {}),
                    "execResultLabel": action_dict.get('resultLabel', ''),
                    "status": TaskStatus.PENDING
                }
                
                task_action = self.chatInterface.createTaskAction(action_data)
                if task_action:
                    task_actions.append(task_action)
                    logger.info(f"Created task action: {task_action.execMethod}.{task_action.execAction}")
            
            # Update stats for task validation (estimate bytes for action validation)
            if task_actions:
                # Calculate actual action size for stats
                action_size = self.service.calculateObjectSize(task_actions)
                self.service.updateWorkflowStats(eventLabel="action", bytesSent=action_size)
            
            logger.info(f"Task action definition completed: {len(task_actions)} actions")
            return task_actions
            
        except Exception as e:
            logger.error(f"Error defining task actions: {str(e)}")
            return []
    
    # Phase 3: Action Execution
    async def executeTaskActions(self, task_actions: List[TaskAction], workflow: ChatWorkflow) -> List[Dict[str, Any]]:
        """Phase 3: Execute all actions for a task"""
        try:
            logger.info(f"Executing {len(task_actions)} task actions")
            
            results = []
            for i, action in enumerate(task_actions):
                logger.info(f"Executing action {i+1}/{len(task_actions)}: {action.execMethod}.{action.execAction}")
                
                # Execute single action
                result = await self._executeSingleAction(action, workflow)
                results.append(result)
                
                # If action failed, stop execution
                if result.get('status') == 'failed':
                    logger.error(f"Action {i+1} failed, stopping task execution")
                    break
            
            logger.info(f"Task action execution completed: {len(results)} results")
            return results
            
        except Exception as e:
            logger.error(f"Error executing task actions: {str(e)}")
            return []
    
    # Phase 4: Task Review and Quality Assessment
    async def reviewTaskCompletion(self, task_step: Dict[str, Any], task_actions: List[TaskAction], 
                                 action_results: List[Dict[str, Any]], workflow: ChatWorkflow) -> Dict[str, Any]:
        """Phase 4: Review task completion and decide next steps"""
        try:
            logger.info(f"Reviewing task completion: {task_step.get('description', 'Unknown')}")
            
            # Create step result summary from action results
            step_result = {
                'task_step': task_step,
                'action_results': action_results,
                'successful_actions': sum(1 for result in action_results if result.get('status') == 'completed'),
                'total_actions': len(action_results),
                'results': [result.get('result', '') for result in action_results if result.get('status') == 'completed'],
                'errors': [result.get('error', '') for result in action_results if result.get('status') == 'failed']
            }
            
            # Prepare review context
            review_context = {
                'task_step': task_step,
                'task_actions': task_actions,
                'action_results': action_results,
                'step_result': step_result,  # Add the missing step_result
                'workflow_id': workflow.id,
                'previous_results': self._getPreviousResultsFromActions(task_actions)
            }
            
            # Use AI to review the results
            review = await self._performTaskReview(review_context)
            
            # Add quality metrics
            review['quality_metrics'] = self._calculateTaskQualityMetrics(task_step, action_results)
            
            logger.info(f"Task review completed: {review.get('status', 'unknown')}")
            return review
            
        except Exception as e:
            logger.error(f"Error reviewing task completion: {str(e)}")
            return {
                'status': 'failed',
                'reason': f'Review failed: {str(e)}',
                'quality_metrics': {'score': 0, 'confidence': 0}
            }
    
    # Phase 5: Task Handover and State Management
    async def prepareTaskHandover(self, task_step: Dict[str, Any], task_actions: List[TaskAction], 
                                review_result: Dict[str, Any], workflow: ChatWorkflow) -> Dict[str, Any]:
        """Phase 5: Prepare results for next task or workflow completion"""
        try:
            logger.info(f"Preparing task handover: {task_step.get('description', 'Unknown')}")
            
            # Update task actions with results
            for action in task_actions:
                if action.status == TaskStatus.PENDING:
                    action.status = TaskStatus.COMPLETED if review_result.get('status') == 'success' else TaskStatus.FAILED
            
            # Create serializable task actions
            task_actions_serializable = []
            for action in task_actions:
                action_dict = {
                    'id': action.id,
                    'execMethod': action.execMethod,
                    'execAction': action.execAction,
                    'execParameters': action.execParameters,
                    'execResultLabel': action.execResultLabel,
                    'status': action.status.value if hasattr(action.status, 'value') else str(action.status)
                }
                task_actions_serializable.append(action_dict)
            
            # Create handover data
            handover_data = {
                'task_step': task_step,
                'task_actions': task_actions_serializable,
                'review_result': review_result,
                'next_task_ready': review_result.get('status') == 'success',
                'available_results': self._getPreviousResultsFromActions(task_actions)
            }
            
            logger.info(f"Task handover prepared: next_task_ready={handover_data['next_task_ready']}")
            return handover_data
            
        except Exception as e:
            logger.error(f"Error preparing task handover: {str(e)}")
            # Create serializable task actions for exception case
            task_actions_serializable = []
            for action in task_actions:
                action_dict = {
                    'id': action.id,
                    'execMethod': action.execMethod,
                    'execAction': action.execAction,
                    'execParameters': action.execParameters,
                    'execResultLabel': action.execResultLabel,
                    'status': action.status.value if hasattr(action.status, 'value') else str(action.status)
                }
                task_actions_serializable.append(action_dict)
            
            return {
                'task_step': task_step,
                'task_actions': task_actions_serializable,
                'review_result': review_result,
                'next_task_ready': False,
                'available_results': []
            }
    




    # ===== Utility Methods =====

    async def processFileIds(self, fileIds: List[str]) -> List[ChatDocument]:
        """Process file IDs and return ChatDocument objects"""
        documents = []
        
        for fileId in fileIds:
            try:
                # Ensure service is initialized
                if not hasattr(self, 'service') or not self.service:
                    logger.error(f"Service not initialized for file ID {fileId}")
                    continue
                
                # Get file info from service
                fileInfo = self.service.getFileInfo(fileId)
                if fileInfo:
                    # Create document using interface
                    documentData = {
                        "fileId": fileId,
                        "filename": fileInfo.get("filename", "unknown"),
                        "fileSize": fileInfo.get("size", 0),
                        "mimeType": fileInfo.get("mimeType", "application/octet-stream")
                    }
                    document = self.chatInterface.createChatDocument(documentData)
                    if document:
                        documents.append(document)
                        logger.info(f"Processed file ID {fileId} -> {document.filename}")
                else:
                    logger.warning(f"No file info found for file ID {fileId}")
            except Exception as e:
                logger.error(f"Error processing file ID {fileId}: {str(e)}")
        
        
        return documents

    def setUserLanguage(self, language: str) -> None:
        """Set user language for the chat manager"""
        if hasattr(self, 'service') and self.service:
            self.service.user.language = language 

    # ===== Enhanced Task Planning Methods =====
    
    async def _callAIWithCircuitBreaker(self, prompt: str, context: str) -> str:
        """Call AI with circuit breaker pattern for fault tolerance"""
        try:
            # Check circuit breaker
            if self._isCircuitBreakerOpen():
                raise Exception("AI circuit breaker is open - too many recent failures")
            
            # Call AI with timeout
            logger.debug(f"ACTION GENERATION PROMPT: {prompt}")
            response = await asyncio.wait_for(
                self._callAI(prompt, context),
                timeout=self.ai_call_timeout
            )
            
            # Reset failure count on success
            self.ai_failure_count = 0
            return response
            
        except asyncio.TimeoutError:
            self._recordAIFailure("Timeout")
            raise Exception(f"AI call timed out after {self.ai_call_timeout} seconds")
        except Exception as e:
            self._recordAIFailure(str(e))
            raise

    def _isCircuitBreakerOpen(self) -> bool:
        """Check if circuit breaker is open"""
        if self.ai_failure_count >= self.ai_circuit_breaker_threshold:
            if self.ai_last_failure_time:
                time_since_failure = (datetime.now(UTC) - self.ai_last_failure_time).total_seconds()
                if time_since_failure < self.ai_circuit_breaker_timeout:
                    return True
                else:
                    # Reset circuit breaker after timeout
                    self.ai_failure_count = 0
                    self.ai_last_failure_time = None
        return False

    def _recordAIFailure(self, error: str):
        """Record AI failure for circuit breaker"""
        self.ai_failure_count += 1
        self.ai_last_failure_time = datetime.now(UTC)
        logger.warning(f"AI failure recorded ({self.ai_failure_count}/{self.ai_circuit_breaker_threshold}): {error}")

    def _validateTaskPlan(self, task_plan: Dict[str, Any]) -> bool:
        """Validate task plan structure and dependencies"""
        try:
            if not isinstance(task_plan, dict):
                return False
            
            if 'tasks' not in task_plan or not isinstance(task_plan['tasks'], list):
                return False
            
            # Check each task
            task_ids = set()
            for task in task_plan['tasks']:
                if not isinstance(task, dict):
                    return False
                
                required_fields = ['id', 'description', 'expected_outputs', 'success_criteria']
                if not all(field in task for field in required_fields):
                    return False
                
                # Check for duplicate task IDs
                if task['id'] in task_ids:
                    return False
                task_ids.add(task['id'])
                
                # Validate dependencies
                dependencies = task.get('dependencies', [])
                if not isinstance(dependencies, list):
                    return False
                
                # Check that dependencies reference existing tasks
                for dep in dependencies:
                    if dep not in task_ids and dep != 'task_0':  # Allow task_0 as special case
                        return False
                
                # Validate ai_prompt if present (optional field)
                if 'ai_prompt' in task and not isinstance(task['ai_prompt'], str):
                    return False
            
            return True
            
        except Exception as e:
            logger.error(f"Error validating task plan: {str(e)}")
            return False

    def _createFallbackTaskPlan(self, context: Dict[str, Any]) -> Dict[str, Any]:
        """Create a fallback task plan when AI generation fails"""
        logger.warning("Creating fallback task plan due to AI generation failure")
        
        return {
            "overview": "Fallback task plan - comprehensive document analysis and processing",
            "tasks": [
                {
                    "id": "task_1",
                    "description": "Extract and analyze all provided documents comprehensively",
                    "dependencies": [],
                    "expected_outputs": ["comprehensive_document_analysis"],
                    "success_criteria": ["All documents processed and analyzed"],
                    "required_documents": context.get('available_documents', []),
                    "estimated_complexity": "medium",
                    "ai_prompt": "Extract and analyze all content from the provided documents. Identify key information, patterns, and insights that are relevant to the user's request. Provide a comprehensive analysis that can be used for further processing."
                },
                {
                    "id": "task_2",
                    "description": "Generate comprehensive output based on analysis",
                    "dependencies": ["task_1"],
                    "expected_outputs": ["final_output"],
                    "success_criteria": ["Output generated and formatted appropriately"],
                    "required_documents": ["comprehensive_document_analysis"],
                    "estimated_complexity": "low",
                    "ai_prompt": "Based on the comprehensive document analysis, generate the final output that addresses the user's original request. Format the output appropriately and ensure it meets the user's requirements."
                }
            ]
        }

    def _validateActions(self, actions: List[Dict[str, Any]], context: Dict[str, Any]) -> bool:
        """Validate generated actions"""
        try:
            if not isinstance(actions, list):
                logger.error("Actions must be a list")
                return False
            
            if len(actions) == 0:
                logger.warning("No actions generated")
                return False
            
            for i, action in enumerate(actions):
                if not isinstance(action, dict):
                    logger.error(f"Action {i} must be a dictionary")
                    return False
                
                # Check required fields
                required_fields = ['method', 'action', 'parameters', 'resultLabel']
                missing_fields = []
                for field in required_fields:
                    if field not in action or not action[field]:
                        missing_fields.append(field)
                
                if missing_fields:
                    logger.error(f"Action {i} missing required fields: {missing_fields}")
                    return False
                
                # Validate result label format
                result_label = action.get('resultLabel', '')
                if not result_label.startswith('task'):
                    logger.error(f"Action {i} result label must start with 'task': {result_label}")
                    return False
                
                # Validate parameters
                parameters = action.get('parameters', {})
                if not isinstance(parameters, dict):
                    logger.error(f"Action {i} parameters must be a dictionary")
                    return False
            
            logger.info(f"Successfully validated {len(actions)} actions")
            return True
            
        except Exception as e:
            logger.error(f"Error validating actions: {str(e)}")
            return False

    def _createFallbackActions(self, task_step: Dict[str, Any], context: Dict[str, Any]) -> List[Dict[str, Any]]:
        """Create fallback actions when AI generation fails with retry context awareness"""
        logger.warning("Creating fallback actions due to AI generation failure")
        
        # Get available documents
        available_docs = context.get('available_documents', [])
        retry_count = context.get('retry_count', 0)
        previous_action_results = context.get('previous_action_results', [])
        
        if not available_docs:
            logger.warning("No available documents for fallback actions")
            return []
        
        # Create fallback actions for document analysis
        fallback_actions = []
        for i, doc in enumerate(available_docs):
            # Enhanced AI prompt for retry scenarios
            ai_prompt = "Fallback document analysis for " + doc
            if retry_count > 0 and previous_action_results:
                ai_prompt += f". Previous attempt failed - ensure comprehensive extraction with detailed analysis."
            
            fallback_actions.append({
                "method": "document",
                "action": "analyze",
                "parameters": {
                    "documentList": ["task1_previous_results"],
                    "aiPrompt": ai_prompt
                },
                "resultLabel": f"task1_fallback_retry{retry_count}:" + doc + ":analysis",
                "description": f"Fallback document analysis for {doc} (attempt {retry_count + 1})"
            })
        
        logger.info(f"Created {len(fallback_actions)} fallback actions")
        return fallback_actions

    # ===== Prompt Creation Methods =====
    
    def _createTaskPlanningPrompt(self, context: Dict[str, Any]) -> str:
        """Create prompt for task planning"""
        return f"""You are a task planning AI that analyzes user requests and creates structured task plans.

USER REQUEST: {context['user_request']}

AVAILABLE DOCUMENTS: {', '.join(context['available_documents'])}

INSTRUCTIONS:
1. Analyze the user request and available documents
2. Break down the request into 2-4 meaningful high-level task steps
3. Focus on business outcomes, not technical operations
4. For document processing, create ONE task with a comprehensive AI prompt rather than multiple granular tasks
5. Each task should produce meaningful, usable outputs
6. Ensure proper handover between tasks using result labels
7. Return a JSON object with the exact structure shown below

TASK PLANNING PRINCIPLES:
- Combine related operations into single tasks (e.g., "Extract and analyze all candidate profiles" instead of separate "read file" and "analyze content" tasks)
- Use comprehensive AI prompts for document processing rather than multiple small tasks
- Focus on business value and outcomes
- Keep tasks at a meaningful level of abstraction
- Each task should produce results that can be used by subsequent tasks

REQUIRED JSON STRUCTURE:
{{
    "overview": "Brief description of the overall plan",
    "tasks": [
        {{
            "id": "task_1",
            "description": "Clear description of what this task accomplishes (business outcome)",
            "dependencies": ["task_0"],  // IDs of tasks that must complete first
            "expected_outputs": ["output1", "output2"],
            "success_criteria": ["criteria1", "criteria2"],
            "required_documents": ["doc1", "doc2"],
            "estimated_complexity": "low|medium|high",
            "ai_prompt": "Comprehensive AI prompt for document processing tasks (if applicable)"
        }}
    ]
}}

EXAMPLES OF GOOD TASK DESCRIPTIONS:
- "Extract and analyze all candidate profiles to identify key qualifications and experience"
- "Create evaluation matrix and rate candidates against product designer criteria"
- "Generate comprehensive PowerPoint presentation for management decision"
- "Store final presentation in SharePoint for specified account"

EXAMPLES OF BAD TASK DESCRIPTIONS:
- "Open and read the PDF file" (too granular)
- "Identify table structure" (technical detail)
- "Convert data to CSV format" (implementation detail)

NOTE: Respond with ONLY the JSON object. Do not include any explanatory text."""

    async def _createActionDefinitionPrompt(self, context: Dict[str, Any]) -> str:
        """Create prompt for action generation with enhanced document extraction guidance and retry context"""
        task_step = context['task_step']
        workflow = context.get('workflow')
        available_docs = context['available_documents']
        previous_results = context['previous_results']
        improvements = context.get('improvements', '')
        retry_count = context.get('retry_count', 0)
        previous_action_results = context.get('previous_action_results', [])
        previous_review_result = context.get('previous_review_result')
        
        # Get available methods and actions with signatures
        methodList = self.service.getMethodsList()
        method_actions = {}
        for sig in methodList:
            if '.' in sig:
                method, rest = sig.split('.', 1)
                action = rest.split('(')[0]
                method_actions.setdefault(method, []).append((action, sig))
        
        # Get workflow history
        messageSummary = await self.service.summarizeChat(workflow.messages)
        
        # Get available documents and connections
        docRefs = self.service.getDocumentReferenceList()
        connRefs = self.service.getConnectionReferenceList()
        all_doc_refs = docRefs.get('chat', []) + docRefs.get('history', [])
        
        # Build AVAILABLE METHODS section
        available_methods_str = ''
        for method, actions in method_actions.items():
            available_methods_str += f"- {method}:\n"
            for action, sig in actions:
                available_methods_str += f"    - {action}: {sig}\n"
        
        # Get AI prompt from task step if available
        task_ai_prompt = task_step.get('ai_prompt', '')
        
        # Build retry context section
        retry_context = ""
        if retry_count > 0:
            retry_context = f"""
RETRY CONTEXT (Attempt {retry_count}):
Previous action results that failed or were incomplete:
"""
            for i, result in enumerate(previous_action_results):
                retry_context += f"- Action {i+1}: {result.get('actionMethod', 'unknown')}.{result.get('actionName', 'unknown')}\n"
                retry_context += f"  Status: {result.get('status', 'unknown')}\n"
                retry_context += f"  Error: {result.get('error', 'None')}\n"
                retry_context += f"  Result: {result.get('result', '')[:100]}...\n"
            
            if previous_review_result:
                retry_context += f"""
Previous review feedback:
- Status: {previous_review_result.get('status', 'unknown')}
- Reason: {previous_review_result.get('reason', 'No reason provided')}
- Quality Score: {previous_review_result.get('quality_score', 0)}/10
- Missing Outputs: {', '.join(previous_review_result.get('missing_outputs', []))}
- Unmet Criteria: {', '.join(previous_review_result.get('unmet_criteria', []))}
"""
        
        return f"""
You are an action generation AI that creates specific actions to accomplish a task step.

DOCUMENT REFERENCE TYPES:
- docItem: Reference to a single document. Format: "docItem:<id>:<filename>"
- docList: Reference to a group of documents under a label. Format: <label> (e.g., "task1_action2_results" or "docList:msg123:user_uploads").
  - Each docList label maps to a list of docItem references (see AVAILABLE DOCUMENTS).
  - A label like "task1_action2_results" refers to the output of action 2 in task 1.

TASK STEP: {task_step.get('description', 'Unknown')} (ID: {task_step.get('id', 'Unknown')})
EXPECTED OUTPUTS: {', '.join(task_step.get('expected_outputs', []))}
SUCCESS CRITERIA: {', '.join(task_step.get('success_criteria', []))}
TASK AI PROMPT: {task_ai_prompt if task_ai_prompt else 'None provided'}

CONTEXT - Chat History:
{messageSummary}

AVAILABLE METHODS AND ACTIONS (with signatures):
{available_methods_str}

AVAILABLE CONNECTIONS:
{chr(10).join(f"- {conn}" for conn in connRefs)}

AVAILABLE DOCUMENTS:
{chr(10).join(f"- {doc.documentsLabel} contains {', '.join(doc.documents)}" for doc in all_doc_refs)}

(Use the label as a value in documentList to refer to the group)

PREVIOUS RESULTS: {', '.join(previous_results) if previous_results else 'None'}
IMPROVEMENTS NEEDED: {improvements if improvements else 'None'}{retry_context}

ACTION GENERATION PRINCIPLES:
- Create meaningful actions per task step
- Use comprehensive AI prompts for document processing
- Focus on business outcomes, not technical operations
- Combine related operations into single actions when possible
- Use the task's AI prompt if provided, or create a comprehensive one
- Each action should produce meaningful, usable outputs
- For document extraction, ensure prompts are specific and detailed
- Include validation steps in extraction prompts
- If this is a retry, learn from previous failures and improve the approach
- Address specific issues mentioned in previous review feedback

INSTRUCTIONS:
- Generate actions to accomplish this task step using available documents, connections, and previous results
- Use docItem for single documents and docList labels for groups of documents as shown in AVAILABLE DOCUMENTS
- Always pass documentList as a LIST of references (docItem and/or docList)
- For resultLabel, use the format: "task{{task_id}}_action{{action_number}}_{{short_label}}" where:
    - {{task_id}} = the current task's id (e.g., 1)
    - {{action_number}} = the sequence number of the action within the task (e.g., 2)
    - {{short_label}} = a short, descriptive label for the output (e.g., "analysis_results")
  Example: "task1_action2_analysis_results"
- If this is a retry, ensure the new actions address the specific issues from previous attempts
- Follow the JSON structure below. All fields are required.

REQUIRED JSON STRUCTURE:
{{
    "actions": [
        {{
            "method": "method_name",  // Use only the method name (e.g., "document")
            "action": "action_name",  // Use only the action name (e.g., "extract")
            "parameters": {{
                "documentList": ["docItem:doc_abc:file1.txt", "task1_action2_results"],
                "aiPrompt": "Comprehensive AI prompt describing what to accomplish"
            }},
            "resultLabel": "task1_action3_analysis_results",
            "description": "What this action accomplishes (business outcome)"
        }}
    ]
}}

FIELD REQUIREMENTS:
- "method": Must be from AVAILABLE METHODS
- "action": Must be valid for the method
- "parameters": Method-specific, must include documentList as a list if required by the signature
- "resultLabel": Must follow the format above (e.g., "task1_action3_analysis_results")
- "description": Clear summary of the business outcome

EXAMPLES OF GOOD ACTIONS:
1. Comprehensive document analysis:
{{
    "method": "document",
    "action": "extract",
    "parameters": {{
        "documentList": ["docItem:doc_57520394-6b6d-41c2-b641-bab3fc6d7f4b:candidate_1_profile.txt"],
        "aiPrompt": "Extract and analyze the candidate's qualifications, experience, skills, and suitability for the product designer position. Identify key strengths, relevant experience, technical skills, and any areas of concern. Provide a comprehensive assessment that can be used for evaluation."
    }},
    "resultLabel": "task1_action1_candidate_analysis",
    "description": "Comprehensive analysis of candidate profile for evaluation"
}}

2. Multi-document processing:
{{
    "method": "document",
    "action": "extract",
    "parameters": {{
        "documentList": ["task1_action1_candidate_analysis", "task1_action2_candidate_analysis", "task1_action3_candidate_analysis"],
        "aiPrompt": "Compare all three candidate profiles and create an evaluation matrix. Rate each candidate on technical skills, experience level, cultural fit, portfolio quality, and communication skills. Provide clear rankings and recommendations for the product designer position."
    }},
    "resultLabel": "task1_action4_evaluation_matrix",
    "description": "Create comprehensive evaluation matrix comparing all candidates"
}}

NOTE: Respond with ONLY the JSON object. Do not include any explanatory text."""


    def _createResultReviewPrompt(self, review_context: Dict[str, Any]) -> str:
        """Create prompt for result review"""
        task_step = review_context['task_step']
        step_result = review_context['step_result']
        
        # Create serializable version of step_result with only metadata (no document content)
        step_result_serializable = {
            'task_step': {
                'id': task_step.get('id', ''),
                'description': task_step.get('description', ''),
                'expected_outputs': task_step.get('expected_outputs', []),
                'success_criteria': task_step.get('success_criteria', [])
            },
            'action_results': [],
            'successful_actions': step_result.get('successful_actions', 0),
            'total_actions': step_result.get('total_actions', 0),
            'results_count': len(step_result.get('results', [])),
            'errors_count': len(step_result.get('errors', []))
        }
        
        # Convert action_results to serializable format with only metadata (no document content)
        for action_result in step_result.get('action_results', []):
            # Extract only document metadata, not content
            documents_metadata = []
            for doc in action_result.get('documents', []):
                if hasattr(doc, 'filename'):
                    documents_metadata.append({
                        'filename': doc.filename,
                        'fileSize': getattr(doc, 'fileSize', 0),
                        'mimeType': getattr(doc, 'mimeType', 'unknown')
                    })
                elif isinstance(doc, dict):
                    documents_metadata.append({
                        'filename': doc.get('filename', 'unknown'),
                        'fileSize': doc.get('fileSize', 0),
                        'mimeType': doc.get('mimeType', 'unknown')
                    })
            
            serializable_action_result = {
                'status': action_result.get('status', ''),
                'result_summary': action_result.get('result', '')[:200] + '...' if len(action_result.get('result', '')) > 200 else action_result.get('result', ''),
                'error': action_result.get('error', ''),
                'resultLabel': action_result.get('resultLabel', ''),
                'documents_count': len(documents_metadata),
                'documents_metadata': documents_metadata,
                'actionId': action_result.get('actionId', ''),
                'actionMethod': action_result.get('actionMethod', ''),
                'actionName': action_result.get('actionName', ''),
                'success_indicator': 'documents' if len(documents_metadata) > 0 else 'text_result' if action_result.get('result', '').strip() else 'none'
            }
            step_result_serializable['action_results'].append(serializable_action_result)
        
        return f"""You are a result review AI that evaluates task step completion and decides on next actions.

TASK STEP: {task_step.get('description', 'Unknown')}
EXPECTED OUTPUTS: {', '.join(task_step.get('expected_outputs', []))}
SUCCESS CRITERIA: {', '.join(task_step.get('success_criteria', []))}

STEP RESULT: {json.dumps(step_result_serializable, indent=2, ensure_ascii=False)}

INSTRUCTIONS:
1. Evaluate if the task step was completed successfully
2. Check if all expected outputs were produced
3. Verify if success criteria were met
4. Decide on next action: continue, retry, or fail
5. If retry, provide specific improvements needed

IMPORTANT NOTES:
- Actions can produce either text results OR documents (or both)
- Empty result_summary is acceptable if documents were produced (documents_count > 0)
- Focus on whether the action achieved its intended purpose, not just text output
- Document-based actions (like file extractions) often have empty text results but successful document outputs
- Check the 'success_indicator' field: 'documents' means success via document output, 'text_result' means success via text, 'none' means no output

REQUIRED JSON STRUCTURE:
{{
    "status": "success|retry|failed",
    "reason": "Explanation of the decision",
    "improvements": "Specific improvements for retry (if status is retry)",
    "quality_score": 1-10,
    "missing_outputs": ["output1", "output2"],
    "met_criteria": ["criteria1", "criteria2"],
    "unmet_criteria": ["criteria3", "criteria4"]
}}

NOTE: Respond with ONLY the JSON object. Do not include any explanatory text."""

    # ===== HELPER METHODS FOR WORKFLOW PHASES =====
    
    async def _generateActionsForTaskStep(self, context: Dict[str, Any]) -> List[Dict[str, Any]]:
        """Generate actions for a specific task step with enhanced retry context"""
        try:
            # Prepare prompt for action generation
            prompt = await self._createActionDefinitionPrompt(context)
            
            # Call AI with circuit breaker
            response = await self._callAIWithCircuitBreaker(prompt, "action_generation")
            
            # Parse and validate actions
            actions = self._parseActionResponse(response)
            
            # Validate actions
            if not self._validateActions(actions, context):
                logger.warning("Generated actions failed validation, using fallback actions")
                actions = self._createFallbackActions(context['task_step'], context)
            
            logger.info(f"Generated {len(actions)} actions for task step")
            return actions
            
        except Exception as e:
            logger.error(f"Error generating actions for task step: {str(e)}")
            return self._createFallbackActions(context['task_step'], context)
    
    async def _executeSingleAction(self, action: TaskAction, workflow: ChatWorkflow) -> Dict[str, Any]:
        """Execute a single action and return result with enhanced document processing"""
        try:
            # Execute the actual method action using the service center
            result = await self.service.executeAction(
                methodName=action.execMethod,
                actionName=action.execAction,
                parameters=action.execParameters
            )
            
            # Always use the execResultLabel from the action definition
            result_label = action.execResultLabel
            
            # Update action based on result
            if result.success:
                action.setSuccess()
                action.result = result.data.get("result", "")
                action.execResultLabel = result_label
                
                # Create and store message in workflow for successful action
                await self._createActionMessage(action, result, workflow, result_label)
                
            else:
                action.setError(result.error or "Action execution failed")
            
            # Enhanced result processing with better document handling
            documents = result.data.get("documents", [])
            processed_documents = []
            
            # Process documents with better metadata extraction
            for doc in documents:
                if hasattr(doc, 'filename') and doc.filename:
                    # Document object with proper metadata
                    mime_type = getattr(doc, 'mimeType', 'application/octet-stream')
                    
                    # Enhanced MIME type detection for document objects
                    if mime_type == "application/octet-stream":
                        mime_type = self._detectMimeTypeFromDocument(doc, doc.filename)
                    
                    processed_documents.append({
                        'filename': doc.filename,
                        'fileSize': getattr(doc, 'fileSize', 0),
                        'mimeType': mime_type,
                        'content': getattr(doc, 'content', ''),
                        'document': doc
                    })
                elif isinstance(doc, dict):
                    # Dictionary document with metadata
                    filename = doc.get('documentName', doc.get('filename', f"{action.execMethod}_{action.execAction}_{datetime.now(UTC).strftime('%Y%m%d_%H%M%S')}"))
                    fileSize = doc.get('fileSize', len(str(doc.get('documentData', ''))))
                    mimeType = doc.get('mimeType', 'application/octet-stream')
                    
                    # Enhanced MIME type detection for dictionary documents
                    if mimeType == "application/octet-stream":
                        document_data = doc.get('documentData', '')
                        mimeType = self._detectMimeTypeFromContent(document_data, filename)
                    
                    processed_documents.append({
                        'filename': filename,
                        'fileSize': fileSize,
                        'mimeType': mimeType,
                        'content': doc.get('documentData', ''),
                        'document': doc
                    })
                else:
                    # Fallback for unknown document types
                    logger.warning(f"Unknown document type for action {action.execMethod}.{action.execAction}: {type(doc)}")
                    filename = f"{action.execMethod}_{action.execAction}_{datetime.now(UTC).strftime('%Y%m%d_%H%M%S')}"
                    mimeType = 'application/octet-stream'
                    
                    # Try to detect MIME type for unknown document types
                    mimeType = self._detectMimeTypeFromContent(doc, filename)
                    
                    processed_documents.append({
                        'filename': filename,
                        'fileSize': 0,
                        'mimeType': mimeType,
                        'content': str(doc),
                        'document': doc
                    })
            
            return {
                "status": "completed" if result.success else "failed",
                "result": result.data.get("result", ""),
                "error": result.error or "",
                "resultLabel": result_label,
                "documents": processed_documents,
                "actionId": action.id,
                "actionMethod": action.execMethod,
                "actionName": action.execAction
            }
            
        except Exception as e:
            logger.error(f"Error executing single action: {str(e)}")
            action.setError(str(e))
            return {
                "status": "failed",
                "error": str(e),
                "actionId": action.id,
                "actionMethod": action.execMethod,
                "actionName": action.execAction,
                "documents": []
            }
    
    async def _createActionMessage(self, action: TaskAction, result: Any, workflow: ChatWorkflow, result_label: str = None) -> None:
        """Create and store a message for the action result in the workflow with enhanced document processing"""
        try:
            # Get result data
            result_data = result.data if hasattr(result, 'data') else {}
            documents_data = result_data.get("documents", [])
            if result_label is None:
                result_label = action.execResultLabel
            
            # Create message data
            message_data = {
                "workflowId": workflow.id,
                "role": "assistant",
                "message": f"Executed action {action.execMethod}.{action.execAction}",
                "status": "step",
                "sequenceNr": len(workflow.messages) + 1,
                "publishedAt": datetime.now(UTC).isoformat(),
                "actionId": action.id,
                "actionMethod": action.execMethod,
                "actionName": action.execAction,
                "documentsLabel": result_label,  # Use intent label from action definition
                "documents": []
            }
            
            # Process documents if any
            if documents_data:
                processed_documents = []
                for doc_data in documents_data:
                    try:
                        # Handle different document data formats
                        if isinstance(doc_data, dict):
                            # Enhanced document processing for dictionary format
                            document_name = doc_data.get("documentName", doc_data.get("filename", f"{action.execMethod}_{action.execAction}_{datetime.now(UTC).strftime('%Y%m%d_%H%M%S')}"))
                            document_data = doc_data.get("documentData", {})
                            file_size = doc_data.get("fileSize", 0)
                            mime_type = doc_data.get("mimeType", "application/octet-stream")
                        elif hasattr(doc_data, 'filename'):
                            # Document object format
                            document_name = doc_data.filename
                            document_data = getattr(doc_data, 'content', {})
                            file_size = getattr(doc_data, 'fileSize', 0)
                            mime_type = getattr(doc_data, 'mimeType', "application/octet-stream")
                        else:
                            # Fallback for unknown formats
                            document_name = f"{action.execMethod}_{action.execAction}_{datetime.now(UTC).strftime('%Y%m%d_%H%M%S')}"
                            document_data = doc_data
                            file_size = len(str(doc_data))
                            mime_type = "application/octet-stream"
                        
                        # Enhanced MIME type detection using service center
                        if mime_type == "application/octet-stream":
                            mime_type = self._detectMimeTypeFromContent(document_data, document_name)
                        
                        # Convert document data to string content
                        content = self._convertDocumentDataToString(document_data, self._getFileExtension(document_name))
                        
                        # Validate content before creating file
                        if not content or content.strip() == "":
                            logger.warning(f"Empty content for document {document_name}, skipping")
                            continue
                        
                        # Create file in database
                        file_id = self.service.createFile(
                            fileName=document_name,
                            mimeType=mime_type,
                            content=content,
                            base64encoded=False
                        )
                        
                        if not file_id:
                            logger.error(f"Failed to create file for document {document_name}")
                            continue
                        
                        # Create ChatDocument object
                        document = self.service.createDocument(
                            fileName=document_name,
                            mimeType=mime_type,
                            content=content,
                            base64encoded=False
                        )
                        
                        if document:
                            processed_documents.append(document)
                            logger.info(f"Created document: {document_name} with file ID: {file_id} and MIME type: {mime_type}")
                        else:
                            logger.error(f"Failed to create ChatDocument object for {document_name}")
                        
                    except Exception as e:
                        logger.error(f"Error processing document {doc_data.get('documentName', 'unknown')}: {str(e)}")
                        continue
                
                # Update message with processed documents
                message_data["documents"] = processed_documents
            
            # Create message using interface
            message = self.chatInterface.createWorkflowMessage(message_data)
            if message:
                workflow.messages.append(message)
                logger.info(f"Created action message for {action.execMethod}.{action.execAction} with {len(message_data.get('documents', []))} documents")
            else:
                logger.error(f"Failed to create workflow message for action {action.execMethod}.{action.execAction}")
                
        except Exception as e:
            logger.error(f"Error creating action message: {str(e)}")
    
    def _getFileExtension(self, filename: str) -> str:
        """Extract file extension from filename"""
        return self.service.getFileExtension(filename)
    
    def _getMimeType(self, extension: str) -> str:
        """Get MIME type based on file extension"""
        return self.service.getMimeTypeFromExtension(extension)
    
    def _detectMimeTypeFromContent(self, content: Any, filename: str) -> str:
        """
        Detect MIME type from content and filename using service center.
        Only returns a detected MIME type if it's better than application/octet-stream.
        
        Args:
            content: Content data (string, dict, or other)
            filename: Name of the file
            
        Returns:
            str: Detected MIME type or original if detection failed
        """
        try:
            # Convert content to bytes for MIME type detection
            if isinstance(content, str):
                file_bytes = content.encode('utf-8')
            elif isinstance(content, dict):
                import json
                file_bytes = json.dumps(content, ensure_ascii=False).encode('utf-8')
            else:
                file_bytes = str(content).encode('utf-8')
            
            # Use service center's MIME type detection
            detected_mime_type = self.service.detectContentTypeFromData(file_bytes, filename)
            if detected_mime_type != "application/octet-stream":
                return detected_mime_type
            return "application/octet-stream"
        except Exception as e:
            logger.warning(f"Error in MIME type detection for {filename}: {str(e)}")
            return 'application/octet-stream'
    
    def _detectMimeTypeFromDocument(self, document: Any, filename: str) -> str:
        """
        Detect MIME type from document object using service center.
        Only returns a detected MIME type if it's better than application/octet-stream.
        
        Args:
            document: Document object with content attribute
            filename: Name of the file
            
        Returns:
            str: Detected MIME type or original if detection failed
        """
        try:
            # Get document content as bytes for MIME type detection
            content = getattr(document, 'content', '')
            if isinstance(content, str):
                file_bytes = content.encode('utf-8')
            else:
                file_bytes = str(content).encode('utf-8')
            
            # Use service center's MIME type detection
            detected_mime_type = self.service.detectContentTypeFromData(file_bytes, filename)
            if detected_mime_type != "application/octet-stream":
                return detected_mime_type
            return "application/octet-stream"
        except Exception as e:
            logger.warning(f"Error in MIME type detection for document {filename}: {str(e)}")
            return 'application/octet-stream'
    
    def _convertDocumentDataToString(self, document_data: Dict[str, Any], file_extension: str) -> str:
        """Convert document data to string content based on file type with enhanced processing"""
        try:
            # Handle None or empty data
            if document_data is None:
                return ""
            
            # Handle string data directly
            if isinstance(document_data, str):
                return document_data
            
            # Handle dictionary data
            if isinstance(document_data, dict):
                # For JSON files, return formatted JSON
                if file_extension == 'json':
                    return json.dumps(document_data, indent=2, ensure_ascii=False)
                
                # For text files, try to extract text content
                elif file_extension in ['txt', 'md', 'html', 'css', 'js', 'py']:
                    # Look for common text content fields
                    text_fields = ['content', 'text', 'data', 'result', 'summary', 'extracted_content', 'table_data']
                    for field in text_fields:
                        if field in document_data:
                            content = document_data[field]
                            if isinstance(content, str):
                                return content
                            elif isinstance(content, (dict, list)):
                                return json.dumps(content, indent=2, ensure_ascii=False)
                    
                    # If no text field found, convert entire dict to JSON
                    return json.dumps(document_data, indent=2, ensure_ascii=False)
                
                # For CSV files, try to extract table data
                elif file_extension == 'csv':
                    # Look for CSV-specific fields
                    csv_fields = ['table_data', 'csv_data', 'rows', 'data']
                    for field in csv_fields:
                        if field in document_data:
                            content = document_data[field]
                            if isinstance(content, str):
                                return content
                            elif isinstance(content, list):
                                # Convert list of rows to CSV format
                                if content and isinstance(content[0], (list, dict)):
                                    import csv
                                    import io
                                    output = io.StringIO()
                                    if isinstance(content[0], dict):
                                        # List of dictionaries
                                        if content:
                                            fieldnames = content[0].keys()
                                            writer = csv.DictWriter(output, fieldnames=fieldnames)
                                            writer.writeheader()
                                            writer.writerows(content)
                                    else:
                                        # List of lists
                                        writer = csv.writer(output)
                                        writer.writerows(content)
                                    return output.getvalue()
                    
                    # Fallback to JSON if no CSV data found
                    return json.dumps(document_data, indent=2, ensure_ascii=False)
                
                # For other file types, convert to JSON
                else:
                    return json.dumps(document_data, indent=2, ensure_ascii=False)
            
            # Handle list data
            elif isinstance(document_data, list):
                if file_extension == 'csv':
                    # Convert list to CSV format
                    import csv
                    import io
                    output = io.StringIO()
                    if document_data and isinstance(document_data[0], dict):
                        # List of dictionaries
                        fieldnames = document_data[0].keys()
                        writer = csv.DictWriter(output, fieldnames=fieldnames)
                        writer.writeheader()
                        writer.writerows(document_data)
                    else:
                        # List of lists
                        writer = csv.writer(output)
                        writer.writerows(document_data)
                    return output.getvalue()
                else:
                    return json.dumps(document_data, indent=2, ensure_ascii=False)
            
            # Handle other data types
            else:
                return str(document_data)
                
        except Exception as e:
            logger.error(f"Error converting document data to string: {str(e)}")
            return str(document_data)
    
    async def _performTaskReview(self, review_context: Dict[str, Any]) -> Dict[str, Any]:
        """Perform AI-based task review with enhanced retry logic"""
        try:
            # Prepare prompt for result review
            prompt = self._createResultReviewPrompt(review_context)
            
            # Call AI with circuit breaker
            response = await self._callAIWithCircuitBreaker(prompt, "result_review")
            
            # Parse review result
            review = self._parseReviewResponse(response)
            
            # Add default values for missing fields
            review.setdefault('status', 'unknown')
            review.setdefault('reason', 'No reason provided')
            review.setdefault('quality_score', 5)
            
            # Enhanced retry logic based on result quality
            if review.get('status') == 'retry':
                # Analyze the specific issues for better retry guidance
                action_results = review_context.get('action_results', [])
                if action_results:
                    # Check for common issues that warrant retry
                    # Only consider empty results a problem if there are no documents produced
                    has_empty_results = any(
                        not result.get('result', '').strip() and 
                        not result.get('documents', []) and 
                        not result.get('documents_metadata', [])
                        for result in action_results 
                        if result.get('status') == 'completed'
                    )
                    
                    has_incomplete_metadata = any(
                        any(doc.get('filename') == 'unknown' for doc in result.get('documents_metadata', []))
                        for result in action_results
                        if result.get('status') == 'completed'
                    )
                    
                    if has_empty_results:
                        review['improvements'] = (review.get('improvements', '') + 
                                               " Ensure the document extraction returns actual content, not empty results. " +
                                               "Check if the AI prompt is specific enough to extract meaningful data.")
                    
                    if has_incomplete_metadata:
                        review['improvements'] = (review.get('improvements', '') + 
                                               " Ensure proper document metadata is extracted including filename, size, and mime type. " +
                                               "The document processing should provide complete file information.")
                    
                    # If we have specific issues, adjust quality score
                    if has_empty_results or has_incomplete_metadata:
                        review['quality_score'] = max(1, review.get('quality_score', 5) - 2)
            
            return review
            
        except Exception as e:
            logger.error(f"Error performing task review: {str(e)}")
            return {
                'status': 'success',  # Default to success to avoid blocking workflow
                'reason': f'Review failed: {str(e)}',
                'quality_score': 5,
                'confidence': 0.5
            }
    
    def _getPreviousResultsFromActions(self, task_actions: List[TaskAction]) -> List[str]:
        """Get list of previous results from completed actions and workflow messages"""
        results = []
        
        # Get results from action objects
        for action in task_actions:
            if action.execResultLabel and action.isSuccessful():
                results.append(action.execResultLabel)
        
        # Get results from workflow messages (for actions that have been executed)
        if hasattr(self, 'workflow') and self.workflow and self.workflow.messages:
            for message in self.workflow.messages:
                if (message.role == 'assistant' and 
                    message.status == 'step' and 
                    message.documentsLabel and 
                    message.documentsLabel not in results):
                    results.append(message.documentsLabel)
        
        return results
    
    def _calculateTaskQualityMetrics(self, task_step: Dict[str, Any], action_results: List[Dict[str, Any]]) -> Dict[str, Any]:
        """Calculate quality metrics for task step results"""
        try:
            quality_score = 0
            confidence = 0
            
            # Count successful actions
            successful_actions = sum(1 for result in action_results if result.get('status') == 'completed')
            total_actions = len(action_results)
            
            if total_actions > 0:
                success_rate = successful_actions / total_actions
                quality_score = int(success_rate * 10)  # Scale to 0-10
                confidence = min(success_rate, 1.0)
            
            return {
                'score': quality_score,
                'confidence': confidence,
                'successful_actions': successful_actions,
                'total_actions': total_actions
            }
            
        except Exception as e:
            logger.error(f"Error calculating task quality metrics: {str(e)}")
            return {'score': 0, 'confidence': 0, 'successful_actions': 0, 'total_actions': 0}
    
    # ===== BASIC HELPER METHODS =====
    
    def _getAvailableDocuments(self, workflow: ChatWorkflow) -> List[str]:
        """Get list of available documents in the workflow"""
        documents = []
        for message in workflow.messages:
            for doc in message.documents:
                documents.append(doc.filename)
        return documents

    def _getPreviousResults(self, task: TaskItem) -> List[str]:
        """Get list of previous results from completed actions"""
        results = []
        for action in task.actionList:
            if action.execResultLabel:
                results.append(action.execResultLabel)
        return results

    def _parseTaskPlanResponse(self, response: str) -> Dict[str, Any]:
        """Parse AI response into task plan structure"""
        try:
            # Extract JSON from response
            json_start = response.find('{')
            json_end = response.rfind('}') + 1
            if json_start == -1 or json_end == 0:
                raise ValueError("No JSON found in response")
            
            json_str = response[json_start:json_end]
            task_plan = json.loads(json_str)
            
            # Validate structure
            if 'tasks' not in task_plan:
                raise ValueError("Task plan missing 'tasks' field")
            
            return task_plan
            
        except Exception as e:
            logger.error(f"Error parsing task plan response: {str(e)}")
            return {'tasks': []}

    def _parseActionResponse(self, response: str) -> List[Dict[str, Any]]:
        """Parse AI response into action list"""
        try:
            # Extract JSON from response
            json_start = response.find('{')
            json_end = response.rfind('}') + 1
            if json_start == -1 or json_end == 0:
                raise ValueError("No JSON found in response")
            
            json_str = response[json_start:json_end]
            action_data = json.loads(json_str)
            
            # Validate structure
            if 'actions' not in action_data:
                raise ValueError("Action response missing 'actions' field")
            
            return action_data['actions']
            
        except Exception as e:
            logger.error(f"Error parsing action response: {str(e)}")
            return []

    def _parseReviewResponse(self, response: str) -> Dict[str, Any]:
        """Parse AI response into review result"""
        try:
            # Extract JSON from response
            json_start = response.find('{')
            json_end = response.rfind('}') + 1
            if json_start == -1 or json_end == 0:
                raise ValueError("No JSON found in response")
            
            json_str = response[json_start:json_end]
            review = json.loads(json_str)
            
            # Validate structure
            if 'status' not in review:
                raise ValueError("Review response missing 'status' field")
            
            return review
            
        except Exception as e:
            logger.error(f"Error parsing review response: {str(e)}")
            return {'status': 'failed', 'reason': f'Parse error: {str(e)}'}

    async def _callAI(self, prompt: str, context: str) -> str:
        """Call AI service with prompt"""
        try:
            # Use the existing AI call mechanism through service
            if hasattr(self, 'service') and self.service:
                # Ensure service is properly initialized
                if hasattr(self.service, 'callAiTextBasic'):
                    response = await self.service.callAiTextBasic(prompt)
                    return response
                else:
                    raise Exception("Service does not have callAiTextBasic method")
            else:
                raise Exception("No service available for AI calls")
                
        except Exception as e:
            logger.error(f"Error calling AI for {context}: {str(e)}")
            raise 

    # ===== WORKFLOW FEEDBACK GENERATION =====
    
    async def generateWorkflowFeedback(self, workflow: ChatWorkflow) -> str:
        """Generate feedback message for workflow completion"""
        try:
            # Count messages by role
            user_messages = [msg for msg in workflow.messages if msg.role == 'user']
            assistant_messages = [msg for msg in workflow.messages if msg.role == 'assistant']
            
            # Generate summary feedback
            feedback = f"Workflow completed.\n\n"
            feedback += f"Processed {len(user_messages)} user inputs and generated {len(assistant_messages)} responses.\n"
            
            # Add final status
            if workflow.status == "completed":
                feedback += "All tasks completed successfully."
            elif workflow.status == "partial":
                feedback += "Some tasks completed with partial success."
            else:
                feedback += f"Workflow status: {workflow.status}"
            
            return feedback
            
        except Exception as e:
            logger.error(f"Error generating workflow feedback: {str(e)}")
            return "Workflow processing completed."

    # ===== UNIFIED WORKFLOW EXECUTION =====
    
    async def executeUnifiedWorkflow(self, userInput: str, workflow: ChatWorkflow) -> Dict[str, Any]:
        """Execute a unified workflow with all phases"""
        try:
            logger.info(f"Starting unified workflow execution for workflow {workflow.id}")
            start_time = time.time()
            
            # Initialize chat manager with workflow
            await self.initialize(workflow)
            
            # Process file IDs if provided
            documents = []
            if hasattr(userInput, 'listFileId') and userInput.listFileId:
                documents = await self.processFileIds(userInput.listFileId)
                logger.info(f"Processed {len(documents)} documents")
            
            # Calculate and update user input stats
            user_input_size = self.service.calculateUserInputSize(userInput)
            self.service.updateWorkflowStats(eventLabel="userinput", bytesReceived=user_input_size)
            
            # Phase 1: High-Level Task Planning
            logger.info("--- PHASE 1: HIGH-LEVEL TASK PLANNING ---")
            task_plan = await self.planHighLevelTasks(userInput.prompt, workflow)
            
            # Update stats for task planning
            task_plan_size = self.service.calculateObjectSize(task_plan)
            self.service.updateWorkflowStats(eventLabel="taskplan", bytesSent=task_plan_size)
            
            # Create user-friendly task plan log
            tasks_count = len(task_plan.get('tasks', []))
            task_descriptions = "\n".join([f"- {task.get('description', 'No description')}" for task in task_plan.get('tasks', [])])
            self.chatInterface.createWorkflowLog({
                "workflowId": workflow.id,
                "message": f"Planning completed: {tasks_count} tasks identified\n{task_descriptions}",
                "type": "info",
                "status": "running",
                "progress": 15,
                "agentName": "System"
            })
            
            # Log task plan details (without document content)
            task_plan_log = {
                'overview': task_plan.get('overview', ''),
                'tasks_count': len(task_plan.get('tasks', [])),
                'tasks': []
            }
            for task in task_plan.get('tasks', []):
                task_log = {
                    'id': task.get('id', ''),
                    'description': task.get('description', ''),
                    'dependencies': task.get('dependencies', []),
                    'expected_outputs': task.get('expected_outputs', []),
                    'success_criteria': task.get('success_criteria', []),
                    'required_documents_count': len(task.get('required_documents', [])),
                    'estimated_complexity': task.get('estimated_complexity', '')
                }
                task_plan_log['tasks'].append(task_log)
            logger.debug(f"TASK PLAN CREATED: {json.dumps(task_plan_log, indent=2, ensure_ascii=False)}")
            
            # Execute each task step with retry logic
            workflow_results = []
            previous_results = []
            max_retries = 3  # Maximum retries per task
            
            for i, task_step in enumerate(task_plan['tasks']):
                task_description = task_step.get('description', 'Unknown')
                logger.info(f"=== PROCESSING TASK {i+1}/{len(task_plan['tasks'])}: {task_description} ===")
                
                # Create user-friendly task start log
                progress = 20 + (i * 60 // len(task_plan['tasks']))
                self.chatInterface.createWorkflowLog({
                    "workflowId": workflow.id,
                    "message": f"Executing task {i+1}/{len(task_plan['tasks'])}: {task_description}",
                    "type": "info",
                    "status": "running",
                    "progress": progress,
                    "agentName": "System"
                })
                
                # Retry loop for each task
                task_success = False
                retry_count = 0
                task_actions = []
                action_results = []
                review_result = {}
                handover_data = {}
                previous_action_results = []  # Track previous action results for retry context
                previous_review_feedback = ""  # Track previous review feedback for retry context
                
                while not task_success and retry_count < max_retries:
                    if retry_count > 0:
                        logger.info(f"--- RETRY {retry_count}/{max_retries} FOR TASK {i+1} ---")
                        # Create user-friendly retry log
                        self.chatInterface.createWorkflowLog({
                            "workflowId": workflow.id,
                            "message": f"Retrying task {i+1} (attempt {retry_count}/{max_retries})",
                            "type": "warning",
                            "status": "running",
                            "progress": progress,
                            "agentName": "System"
                        })
                    
                    try:
                        # Phase 2: Define Task Actions
                        logger.info(f"--- PHASE 2: DEFINING ACTIONS FOR TASK {i+1} ---")
                        
                        # Enhanced context for retries - include previous results and feedback
                        enhanced_previous_results = previous_results.copy() if previous_results else []
                        if retry_count > 0 and previous_action_results:
                            # Add previous action results to context
                            for result in previous_action_results:
                                if result.get('resultLabel'):
                                    enhanced_previous_results.append(result.get('resultLabel'))
                        
                        # Create enhanced context with retry information
                        context = {
                            'task_step': task_step,
                            'workflow': workflow,
                            'workflow_id': workflow.id,
                            'available_documents': self._getAvailableDocuments(workflow),
                            'previous_results': enhanced_previous_results,
                            'improvements': previous_review_feedback if retry_count > 0 else None,
                            'retry_count': retry_count,
                            'previous_action_results': previous_action_results if retry_count > 0 else [],
                            'previous_review_result': review_result if retry_count > 0 else None
                        }
                        
                        task_actions = await self.defineTaskActions(task_step, workflow, enhanced_previous_results, context)
                        if not task_actions:
                            logger.warning(f"No actions defined for task {i+1}, skipping")
                            break
                        
                        # Log task actions (convert to serializable format with metadata only)
                        task_actions_serializable = []
                        for action in task_actions:
                            # Extract only metadata from parameters, not document content
                            parameters_metadata = {}
                            if hasattr(action, 'execParameters') and action.execParameters:
                                for key, value in action.execParameters.items():
                                    if key == 'documentList':
                                        # Log document list as count and labels only
                                        if isinstance(value, list):
                                            parameters_metadata[key] = {
                                                'count': len(value),
                                                'labels': [str(v).split(':')[-1] if ':' in str(v) else str(v) for v in value]
                                            }
                                        else:
                                            parameters_metadata[key] = str(value)
                                    elif key == 'aiPrompt':
                                        # Truncate AI prompts to avoid logging large content
                                        parameters_metadata[key] = str(value)[:100] + '...' if len(str(value)) > 100 else str(value)
                                    else:
                                        parameters_metadata[key] = str(value)
                            
                            action_dict = {
                                'execMethod': action.execMethod,
                                'execAction': action.execAction,
                                'execParameters': parameters_metadata,
                                'execResultLabel': action.execResultLabel,
                                'status': action.status.value if hasattr(action.status, 'value') else str(action.status)
                            }
                            task_actions_serializable.append(action_dict)
                        logger.debug(f"TASK {i+1} ACTIONS CREATED: {json.dumps(task_actions_serializable, indent=2, ensure_ascii=False)}")
                        
                        # Phase 3: Execute Task Actions
                        logger.info(f"--- PHASE 3: EXECUTING TASK {i+1} ACTIONS ---")
                        action_results = await self.executeTaskActions(task_actions, workflow)
                        
                        # Update stats for action execution
                        # Action stats are already handled by the service center during AI calls
                        
                        # Create user-friendly action completion log with quality metrics
                        successful_actions = sum(1 for result in action_results if result.get('status') == 'completed')
                        total_actions = len(action_results)
                        
                        if total_actions > 0:
                            if successful_actions == total_actions:
                                log_type = "success"
                            elif successful_actions == 0:
                                log_type = "error"
                            else:
                                log_type = "warning"
                            self.chatInterface.createWorkflowLog({
                                "workflowId": workflow.id,
                                "message": f"Successful actions: {successful_actions}/{total_actions}",
                                "type": log_type,
                                "status": "running",
                                "progress": progress + 10
                            })
                        
                        # Log action results (with metadata only)
                        action_results_metadata = []
                        for result in action_results:
                            # Extract document metadata only
                            documents_metadata = []
                            for doc in result.get('documents', []):
                                if hasattr(doc, 'filename'):
                                    documents_metadata.append({
                                        'filename': doc.filename,
                                        'fileSize': getattr(doc, 'fileSize', 0),
                                        'mimeType': getattr(doc, 'mimeType', 'unknown')
                                    })
                                elif isinstance(doc, dict):
                                    documents_metadata.append({
                                        'filename': doc.get('filename', 'unknown'),
                                        'fileSize': doc.get('fileSize', 0),
                                        'mimeType': doc.get('mimeType', 'unknown')
                                    })
                            
                            result_metadata = {
                                'status': result.get('status', ''),
                                'result_summary': result.get('result', '')[:200] + '...' if len(result.get('result', '')) > 200 else result.get('result', ''),
                                'error': result.get('error', ''),
                                'resultLabel': result.get('resultLabel', ''),
                                'documents_count': len(documents_metadata),
                                'documents_metadata': documents_metadata,
                                'actionId': result.get('actionId', ''),
                                'actionMethod': result.get('actionMethod', ''),
                                'actionName': result.get('actionName', '')
                            }
                            action_results_metadata.append(result_metadata)
                        logger.debug(f"TASK {i+1} ACTION RESULTS: {json.dumps(action_results_metadata, indent=2, ensure_ascii=False)}")
                        
                        # Phase 4: Review Task Completion
                        logger.info(f"--- PHASE 4: REVIEWING TASK {i+1} COMPLETION ---")
                        review_result = await self.reviewTaskCompletion(task_step, task_actions, action_results, workflow)
                        
                        # Update stats for task review
                        # Task review stats are already handled by the service center during AI calls
                        
                        # Create user-friendly review log with quality metrics
                        quality_metrics = review_result.get('quality_metrics', {})
                        quality_score = quality_metrics.get('score', 0)
                        confidence = quality_metrics.get('confidence', 0)
                        
                        review_status = review_result.get('status', 'unknown')
                        if review_status == 'success':
                            self.chatInterface.createWorkflowLog({
                                "workflowId": workflow.id,
                                "message": f"🎯 Task completed successfully with quality score {quality_score} and confidence {confidence}",
                                "type": "success",
                                "status": "running",
                                "progress": progress + 20
                            })
                        elif review_status == 'retry':
                            # Extract improvement details
                            improvements = review_result.get('improvements', '')
                            reason = review_result.get('reason', '')
                            unmet_criteria = review_result.get('unmet_criteria', [])
                            
                            # Build detailed message
                            retry_details = []
                            if reason:
                                retry_details.append(f"Reason: {reason}")
                            if improvements:
                                retry_details.append(f"Improvements: {improvements}")
                            if unmet_criteria:
                                retry_details.append(f"Missing criteria: {', '.join(unmet_criteria[:3])}{'...' if len(unmet_criteria) > 3 else ''}")
                            
                            retry_message = f"🔄 Task needs improvement"
                            if retry_details:
                                retry_message += f"\n{chr(10).join(retry_details)}"
                            
                            self.chatInterface.createWorkflowLog({
                                "workflowId": workflow.id,
                                "message": retry_message,
                                "type": "warning",
                                "status": "running",
                                "progress": progress + 15
                            })
                        else:
                            # Extract failure details
                            reason = review_result.get('reason', '')
                            unmet_criteria = review_result.get('unmet_criteria', [])
                            missing_outputs = review_result.get('missing_outputs', [])
                            
                            # Build detailed failure message
                            failure_details = []
                            if reason:
                                failure_details.append(f"Reason: {reason}")
                            if unmet_criteria:
                                failure_details.append(f"Unmet criteria: {', '.join(unmet_criteria[:3])}{'...' if len(unmet_criteria) > 3 else ''}")
                            if missing_outputs:
                                failure_details.append(f"Missing outputs: {', '.join(missing_outputs[:3])}{'...' if len(missing_outputs) > 3 else ''}")
                            
                            failure_message = f"❌ Task failed"
                            if failure_details:
                                failure_message += f"\n{chr(10).join(failure_details)}"
                            
                            self.chatInterface.createWorkflowLog({
                                "workflowId": workflow.id,
                                "message": failure_message,
                                "type": "error",
                                "status": "running",
                                "progress": progress + 15
                            })
                        
                        # Log review result (with metadata only)
                        review_result_metadata = {
                            'status': review_result.get('status', ''),
                            'reason': review_result.get('reason', ''),
                            'improvements': review_result.get('improvements', ''),
                            'quality_score': review_result.get('quality_score', 0),
                            'missing_outputs_count': len(review_result.get('missing_outputs', [])),
                            'met_criteria_count': len(review_result.get('met_criteria', [])),
                            'unmet_criteria_count': len(review_result.get('unmet_criteria', []))
                        }
                        logger.debug(f"TASK {i+1} REVIEW RESULT: {json.dumps(review_result_metadata, indent=2, ensure_ascii=False)}")
                        
                        # Phase 5: Prepare Task Handover
                        logger.info(f"--- PHASE 5: PREPARING TASK {i+1} HANDOVER ---")
                        handover_data = await self.prepareTaskHandover(task_step, task_actions, review_result, workflow)
                        
                        # Log handover data (with metadata only)
                        handover_data_metadata = {
                            'task_step_id': handover_data.get('task_step', {}).get('id', ''),
                            'task_actions_count': len(handover_data.get('task_actions', [])),
                            'review_status': handover_data.get('review_result', {}).get('status', ''),
                            'next_task_ready': handover_data.get('next_task_ready', False),
                            'available_results_count': len(handover_data.get('available_results', []))
                        }
                        logger.debug(f"TASK {i+1} HANDOVER DATA: {json.dumps(handover_data_metadata, indent=2, ensure_ascii=False)}")
                        
                        # Check if task is successful or needs retry
                        review_status = review_result.get('status', 'unknown')
                        if review_status == 'success':
                            task_success = True
                            logger.info(f"Task {i+1} completed successfully")
                        elif review_status == 'retry':
                            # Store current results and feedback for next retry
                            previous_action_results = action_results.copy()
                            previous_review_feedback = review_result.get('improvements', '')
                            
                            retry_count += 1
                            if retry_count > max_retries:
                                logger.error(f"Task {i+1} failed after {max_retries} retries")
                                task_success = False
                            else:
                                logger.info(f"Task {i+1} needs retry (attempt {retry_count}/{max_retries})")
                                logger.info(f"Previous feedback: {previous_review_feedback}")
                                # Add delay before retry
                                await asyncio.sleep(2)
                                continue
                        elif review_status == 'failed':
                            logger.error(f"Task {i+1} failed permanently")
                            task_success = False
                            break
                        else:
                            logger.warning(f"Unknown review status '{review_status}' for task {i+1}")
                            task_success = False
                            break
                        
                    except Exception as e:
                        logger.error(f"Error processing task {i+1} (attempt {retry_count + 1}): {str(e)}")
                        retry_count += 1
                        if retry_count >= max_retries:
                            logger.error(f"Task {i+1} failed after {max_retries} retries due to exceptions")
                            task_success = False
                            break
                        else:
                            logger.info(f"Retrying task {i+1} after exception")
                            await asyncio.sleep(2)
                            continue
                
                # Collect results regardless of success/failure
                workflow_results.append({
                    'task_step': task_step,
                    'task_actions': task_actions,
                    'action_results': action_results,
                    'review_result': review_result,
                    'handover_data': handover_data,
                    'retry_count': retry_count,
                    'task_success': task_success
                })
                
                # Update previous results for next task if successful
                if task_success and handover_data.get('next_task_ready', False):
                    previous_results = handover_data.get('available_results', [])
                else:
                    # If task failed, stop workflow
                    logger.warning(f"Task {i+1} not successful, stopping workflow")
                    break
            
            # Final workflow summary
            successful_tasks = sum(1 for result in workflow_results if result.get('task_success', False))
            total_tasks = len(task_plan['tasks'])
            
            # Final workflow stats are already handled by the service center during AI calls
            
            # Calculate total processing time
            total_processing_time = time.time() - start_time
            
            # Create final user-friendly completion log
            if successful_tasks == total_tasks:
                self.chatInterface.createWorkflowLog({
                    "workflowId": workflow.id,
                    "message": f"🎉 Workflow completed ({successful_tasks}/{total_tasks} tasks)",
                    "type": "success",
                    "status": "completed",
                    "progress": 100
                })
            elif successful_tasks > 0:
                self.chatInterface.createWorkflowLog({
                    "workflowId": workflow.id,
                    "message": f"⚠️ Workflow  partially completed ({successful_tasks}/{total_tasks} tasks)",
                    "type": "warning",
                    "status": "completed",
                    "progress": 100
                })
            else:
                self.chatInterface.createWorkflowLog({
                    "workflowId": workflow.id,
                    "message": f"❌ Workflow failed ({successful_tasks}/{total_tasks} tasks)",
                    "type": "error",
                    "status": "failed",
                    "progress": 100
                })
            
            # Create serializable workflow results (with metadata only)
            workflow_results_serializable = []
            for result in workflow_results:
                # Extract action results metadata
                action_results_metadata = []
                for action_result in result.get('action_results', []):
                    documents_metadata = []
                    for doc in action_result.get('documents', []):
                        if hasattr(doc, 'filename'):
                            documents_metadata.append({
                                'filename': doc.filename,
                                'fileSize': getattr(doc, 'fileSize', 0),
                                'mimeType': getattr(doc, 'mimeType', 'unknown')
                            })
                        elif isinstance(doc, dict):
                            documents_metadata.append({
                                'filename': doc.get('filename', 'unknown'),
                                'fileSize': doc.get('fileSize', 0),
                                'mimeType': doc.get('mimeType', 'unknown')
                            })
                    
                    action_result_metadata = {
                        'status': action_result.get('status', ''),
                        'result_summary': action_result.get('result', '')[:200] + '...' if len(action_result.get('result', '')) > 200 else action_result.get('result', ''),
                        'error': action_result.get('error', ''),
                        'resultLabel': action_result.get('resultLabel', ''),
                        'documents_count': len(documents_metadata),
                        'documents_metadata': documents_metadata,
                        'actionId': action_result.get('actionId', ''),
                        'actionMethod': action_result.get('actionMethod', ''),
                        'actionName': action_result.get('actionName', ''),
                        'success_indicator': 'documents' if len(documents_metadata) > 0 else 'text_result' if action_result.get('result', '').strip() else 'none'
                    }
                    action_results_metadata.append(action_result_metadata)
                
                serializable_result = {
                    'task_step': result['task_step'],
                    'action_results': action_results_metadata,
                    'review_result': result['review_result'],
                    'handover_data': {
                        'task_step_id': result['handover_data'].get('task_step', {}).get('id', ''),
                        'task_actions_count': len(result['handover_data'].get('task_actions', [])),
                        'review_status': result['handover_data'].get('review_result', {}).get('status', ''),
                        'next_task_ready': result['handover_data'].get('next_task_ready', False),
                        'available_results_count': len(result['handover_data'].get('available_results', []))
                    },
                    'retry_count': result.get('retry_count', 0),
                    'task_success': result.get('task_success', False)
                }
                # Convert task_actions to serializable format with metadata only
                if 'task_actions' in result:
                    task_actions_serializable = []
                    for action in result['task_actions']:
                        # Extract only metadata from parameters
                        parameters_metadata = {}
                        if hasattr(action, 'execParameters') and action.execParameters:
                            for key, value in action.execParameters.items():
                                if key == 'documentList':
                                    if isinstance(value, list):
                                        parameters_metadata[key] = {
                                            'count': len(value),
                                            'labels': [str(v).split(':')[-1] if ':' in str(v) else str(v) for v in value]
                                        }
                                    else:
                                        parameters_metadata[key] = str(value)
                                elif key == 'aiPrompt':
                                    parameters_metadata[key] = str(value)[:100] + '...' if len(str(value)) > 100 else str(value)
                                else:
                                    parameters_metadata[key] = str(value)
                        
                        action_dict = {
                            'execMethod': action.execMethod,
                            'execAction': action.execAction,
                            'execParameters': parameters_metadata,
                            'execResultLabel': action.execResultLabel,
                            'status': action.status.value if hasattr(action.status, 'value') else str(action.status)
                        }
                        task_actions_serializable.append(action_dict)
                    serializable_result['task_actions'] = task_actions_serializable
                workflow_results_serializable.append(serializable_result)
            
            workflow_summary = {
                'status': 'completed' if successful_tasks == total_tasks else 'partial',
                'successful_tasks': successful_tasks,
                'total_tasks': total_tasks,
                'workflow_results_count': len(workflow_results_serializable),
                'final_results_count': len(previous_results)
            }
            
            logger.info(f"=== UNIFIED WORKFLOW COMPLETED: {successful_tasks}/{total_tasks} tasks successful ===")
            logger.debug(f"FINAL WORKFLOW SUMMARY: {json.dumps(workflow_summary, indent=2, ensure_ascii=False)}")
            return workflow_summary
            
        except Exception as e:
            logger.error(f"Error in unified workflow execution: {str(e)}")
            # Create error log for user
            self.chatInterface.createWorkflowLog({
                "workflowId": workflow.id,
                "message": f"Workflow execution failed: {str(e)}",
                "type": "error",
                "status": "failed",
                "progress": 100,
                "agentName": "System"
            })
            return {
                'status': 'failed',
                'error': str(e),
                'phase': 'execution'
            }