gateway/modules/workflows/workflowManager.py
2025-10-24 23:57:17 +02:00

721 lines
32 KiB
Python

from typing import Dict, Any, List, Optional
import logging
import uuid
import asyncio
import json
from modules.datamodels.datamodelChat import (
UserInputRequest,
ChatMessage,
ChatWorkflow,
ChatDocument
)
from modules.datamodels.datamodelChat import TaskContext
from modules.workflows.processing.workflowProcessor import WorkflowProcessor, WorkflowStoppedException
logger = logging.getLogger(__name__)
class WorkflowManager:
"""Manager for workflow processing and coordination"""
def __init__(self, services):
self.services = services
self.workflowProcessor = None
# Exported functions
async def workflowStart(self, userInput: UserInputRequest, workflowId: Optional[str] = None, workflowMode: str = "React") -> ChatWorkflow:
"""Starts a new workflow or continues an existing one, then launches processing."""
try:
# Debug log to check workflowMode parameter
logger.info(f"WorkflowManager received workflowMode: {workflowMode}")
currentTime = self.services.utils.timestampGetUtc()
if workflowId:
workflow = self.services.workflow.getWorkflow(workflowId)
if not workflow:
raise ValueError(f"Workflow {workflowId} not found")
# Store workflow in services for reference
self.services.currentWorkflow = workflow
if workflow.status == "running":
logger.info(f"Stopping running workflow {workflowId} before processing new prompt")
workflow.status = "stopped"
workflow.lastActivity = currentTime
self.services.workflow.updateWorkflow(workflowId, {
"status": "stopped",
"lastActivity": currentTime
})
self.services.workflow.storeLog(workflow, {
"message": "Workflow stopped for new prompt",
"type": "info",
"status": "stopped",
"progress": 100
})
newRound = workflow.currentRound + 1
self.services.workflow.updateWorkflow(workflowId, {
"status": "running",
"lastActivity": currentTime,
"currentRound": newRound,
"workflowMode": workflowMode # Update workflow mode for existing workflows
})
# Reflect updates on the in-memory object without reloading
workflow.status = "running"
workflow.lastActivity = currentTime
workflow.currentRound = newRound
workflow.workflowMode = workflowMode
self.services.workflow.storeLog(workflow, {
"message": f"Workflow resumed (round {workflow.currentRound}) with mode: {workflowMode}",
"type": "info",
"status": "running",
"progress": 0
})
else:
workflowData = {
"name": "New Workflow",
"status": "running",
"startedAt": currentTime,
"lastActivity": currentTime,
"currentRound": 1,
"currentTask": 0,
"currentAction": 0,
"totalTasks": 0,
"totalActions": 0,
"mandateId": self.services.user.mandateId,
"messageIds": [],
"workflowMode": workflowMode,
"maxSteps": 5 if workflowMode == "React" else 1, # Set maxSteps for React mode
}
workflow = self.services.workflow.createWorkflow(workflowData)
logger.info(f"Created workflow with mode: {getattr(workflow, 'workflowMode', 'NOT_SET')}")
logger.info(f"Workflow data passed: {workflowData.get('workflowMode', 'NOT_IN_DATA')}")
self.services.currentWorkflow = workflow
# Start workflow processing asynchronously
asyncio.create_task(self._workflowProcess(userInput, workflow))
return workflow
except Exception as e:
logger.error(f"Error starting workflow: {str(e)}")
raise
async def workflowStop(self, workflowId: str) -> ChatWorkflow:
"""Stops a running workflow."""
try:
workflow = self.services.workflow.getWorkflow(workflowId)
if not workflow:
raise ValueError(f"Workflow {workflowId} not found")
workflow.status = "stopped"
workflow.lastActivity = self.services.utils.timestampGetUtc()
self.services.workflow.updateWorkflow(workflowId, {
"status": "stopped",
"lastActivity": workflow.lastActivity
})
self.services.workflow.storeLog(workflow, {
"message": "Workflow stopped",
"type": "warning",
"status": "stopped",
"progress": 100
})
return workflow
except Exception as e:
logger.error(f"Error stopping workflow: {str(e)}")
raise
# Main processor
async def _workflowProcess(self, userInput: UserInputRequest, workflow: ChatWorkflow) -> None:
"""Process a workflow with user input"""
try:
# Store the current user prompt in services for easy access throughout the workflow
self.services.rawUserPrompt = userInput.prompt
self.services.currentUserPrompt = userInput.prompt
# Update the workflow service with the current workflow context
self.services.workflow.setWorkflowContext(workflow)
self.workflowProcessor = WorkflowProcessor(self.services, workflow)
await self._sendFirstMessage(userInput, workflow)
task_plan = await self._planTasks(userInput, workflow)
await self._executeTasks(task_plan, workflow)
await self._processWorkflowResults(workflow)
except WorkflowStoppedException:
self._handleWorkflowStop(workflow)
except Exception as e:
self._handleWorkflowError(workflow, e)
# Helper functions
async def _sendFirstMessage(self, userInput: UserInputRequest, workflow: ChatWorkflow) -> None:
"""Send first message to start workflow"""
try:
self.workflowProcessor._checkWorkflowStopped(workflow)
# Create initial message using interface
# For first user message, include round info in the user context label
round_num = workflow.currentRound
task_num = 0
action_num = 0
context_label = f"round{round_num}_usercontext"
messageData = {
"workflowId": workflow.id,
"role": "user",
"message": userInput.prompt,
"status": "first",
"sequenceNr": 1,
"publishedAt": self.services.utils.timestampGetUtc(),
"documentsLabel": context_label,
"documents": [],
# Add workflow context fields
"roundNumber": workflow.currentRound,
"taskNumber": 0,
"actionNumber": 0,
# Add progress status
"taskProgress": "pending",
"actionProgress": "pending"
}
# Analyze the user's input to detect language, normalize request, extract intent, and offload bulky context into documents
created_docs = []
try:
analyzerPrompt = (
"You are an input analyzer. From the user's message, perform ALL of the following in one pass:\n"
"1) detectedLanguage: detect ISO 639-1 language code (e.g., de, en).\n"
"2) normalizedRequest: full, explicit restatement of the user's request in the detected language; do NOT summarize; preserve ALL constraints and details.\n"
"3) intent: concise single-paragraph core request in the detected language for high-level routing.\n"
"4) contextItems: supportive data blocks to attach as separate documents if significantly larger than the intent (large literal content, long lists/tables, code/JSON blocks, transcripts, CSV fragments, detailed specs). Keep URLs in the intent unless they embed large pasted content.\n\n"
"Rules:\n"
"- If total content (intent + data) is < 10% of model max tokens, do not extract; return empty contextItems and keep intent compact and self-contained.\n"
"- If content exceeds that threshold, move bulky parts into contextItems; keep intent short and clear.\n"
"- Preserve critical references (URLs, filenames) in intent.\n"
"- Normalize to the primary detected language if mixed-language.\n\n"
"Return ONLY JSON (no markdown) with this shape:\n"
"{\n"
" \"detectedLanguage\": \"de|en|fr|it|...\",\n"
" \"normalizedRequest\": \"Full explicit instruction in detected language\",\n"
" \"intent\": \"Concise normalized request...\",\n"
" \"contextItems\": [\n"
" {\n"
" \"title\": \"User context 1\",\n"
" \"mimeType\": \"text/plain\",\n"
" \"content\": \"Full extracted content block here\"\n"
" }\n"
" ]\n"
"}\n\n"
f"User message:\n{self.services.ai.sanitizePromptContent(userInput.prompt, 'userinput')}"
)
# Call AI analyzer (planning call - will use static parameters)
aiResponse = await self.services.ai.callAiPlanning(prompt=analyzerPrompt, placeholders=None)
detectedLanguage = None
normalizedRequest = None
intentText = userInput.prompt
contextItems = []
# Parse analyzer response (JSON expected)
try:
jsonStart = aiResponse.find('{') if aiResponse else -1
jsonEnd = aiResponse.rfind('}') + 1 if aiResponse else 0
if jsonStart != -1 and jsonEnd > jsonStart:
parsed = json.loads(aiResponse[jsonStart:jsonEnd])
detectedLanguage = parsed.get('detectedLanguage') or None
normalizedRequest = parsed.get('normalizedRequest') or None
if parsed.get('intent'):
intentText = parsed.get('intent')
contextItems = parsed.get('contextItems') or []
except Exception:
contextItems = []
# Update services state
if detectedLanguage and isinstance(detectedLanguage, str):
self._setUserLanguage(detectedLanguage)
try:
setattr(self.services, 'currentUserLanguage', detectedLanguage)
except Exception:
pass
self.services.currentUserPrompt = intentText or userInput.prompt
try:
if normalizedRequest:
setattr(self.services, 'currentUserPromptNormalized', normalizedRequest)
if contextItems is not None:
setattr(self.services, 'currentUserContextItems', contextItems)
except Exception:
pass
# Create documents for context items
if contextItems and isinstance(contextItems, list):
for idx, item in enumerate(contextItems):
try:
title = item.get('title') if isinstance(item, dict) else None
mime = item.get('mimeType') if isinstance(item, dict) else None
content = item.get('content') if isinstance(item, dict) else None
if not content:
continue
fileName = (title or f"user_context_{idx+1}.txt").strip()
mimeType = (mime or "text/plain").strip()
# Create file in component storage
content_bytes = content.encode('utf-8')
file_item = self.services.interfaceDbComponent.createFile(
name=fileName,
mimeType=mimeType,
content=content_bytes
)
# Persist file data
self.services.interfaceDbComponent.createFileData(file_item.id, content_bytes)
# Collect file info
file_info = self.services.workflow.getFileInfo(file_item.id)
from modules.datamodels.datamodelChat import ChatDocument
doc = ChatDocument(
fileId=file_item.id,
fileName=file_info.get("fileName", fileName) if file_info else fileName,
fileSize=file_info.get("size", len(content_bytes)) if file_info else len(content_bytes),
mimeType=file_info.get("mimeType", mimeType) if file_info else mimeType
)
created_docs.append(doc)
except Exception:
continue
except Exception as e:
logger.warning(f"Prompt analysis failed or skipped: {str(e)}")
# Process user-uploaded documents (fileIds) and combine with context documents
if userInput.listFileId:
try:
user_docs = await self._processFileIds(userInput.listFileId, None)
if user_docs:
created_docs.extend(user_docs)
except Exception as e:
logger.warning(f"Failed to process user fileIds: {e}")
# Finally, persist and bind the first message with combined documents (context + user)
self.services.workflow.storeMessageWithDocuments(workflow, messageData, created_docs)
except Exception as e:
logger.error(f"Error sending first message: {str(e)}")
raise
async def _planTasks(self, userInput: UserInputRequest, workflow: ChatWorkflow):
"""Generate task plan for workflow execution"""
handling = self.workflowProcessor
# Generate task plan first (shared for both modes)
task_plan = await handling.generateTaskPlan(userInput.prompt, workflow)
if not task_plan or not task_plan.tasks:
raise Exception("No tasks generated in task plan.")
workflow_mode = getattr(workflow, 'workflowMode', 'Actionplan')
logger.info(f"Workflow object attributes: {workflow.__dict__ if hasattr(workflow, '__dict__') else 'No __dict__'}")
logger.info(f"Executing workflow mode={workflow_mode} with {len(task_plan.tasks)} tasks")
return task_plan
async def _executeTasks(self, task_plan, workflow: ChatWorkflow) -> None:
"""Execute all tasks in the task plan and update workflow status."""
handling = self.workflowProcessor
total_tasks = len(task_plan.tasks)
all_task_results: List = []
previous_results: List[str] = []
for idx, task_step in enumerate(task_plan.tasks):
current_task_index = idx + 1
logger.info(f"Task {current_task_index}/{total_tasks}: {task_step.objective}")
# Build TaskContext (mode-specific behavior is inside WorkflowProcessor)
task_context = TaskContext(
task_step=task_step,
workflow=workflow,
workflow_id=workflow.id,
available_documents=None,
available_connections=None,
previous_results=previous_results,
previous_handover=None,
improvements=[],
retry_count=0,
previous_action_results=[],
previous_review_result=None,
is_regeneration=False,
failure_patterns=[],
failed_actions=[],
successful_actions=[],
criteria_progress={
'met_criteria': set(),
'unmet_criteria': set(),
'attempt_history': []
}
)
task_result = await handling.executeTask(task_step, workflow, task_context, current_task_index, total_tasks)
handover_data = await handling.prepareTaskHandover(task_step, [], task_result, workflow)
all_task_results.append({
'task_step': task_step,
'task_result': task_result,
'handover_data': handover_data
})
if task_result.success and task_result.feedback:
previous_results.append(task_result.feedback)
# Mark workflow as completed; error/stop cases update status elsewhere
workflow.status = "completed"
return None
async def _processWorkflowResults(self, workflow: ChatWorkflow) -> None:
"""Process workflow results based on workflow status and create appropriate messages"""
try:
try:
self.workflowProcessor._checkWorkflowStopped(workflow)
except WorkflowStoppedException:
logger.info(f"Workflow {workflow.id} was stopped during result processing")
# Create final stopped message
stopped_message = {
"workflowId": workflow.id,
"role": "assistant",
"message": "🛑 Workflow stopped by user",
"status": "last",
"sequenceNr": len(workflow.messages) + 1,
"publishedAt": self.services.utils.timestampGetUtc(),
"documentsLabel": "workflow_stopped",
"documents": [],
# Add workflow context fields
"roundNumber": workflow.currentRound,
"taskNumber": 0,
"actionNumber": 0,
# Add progress status
"taskProgress": "stopped",
"actionProgress": "stopped"
}
self.services.workflow.storeMessageWithDocuments(workflow, stopped_message, [])
# Update workflow status to stopped
workflow.status = "stopped"
workflow.lastActivity = self.services.utils.timestampGetUtc()
self.services.workflow.updateWorkflow(workflow.id, {
"status": "stopped",
"lastActivity": workflow.lastActivity
})
return
if workflow.status == 'stopped':
# Create stopped message
stopped_message = {
"workflowId": workflow.id,
"role": "assistant",
"message": "🛑 Workflow stopped by user",
"status": "last",
"sequenceNr": len(workflow.messages) + 1,
"publishedAt": self.services.utils.timestampGetUtc(),
"documentsLabel": "workflow_stopped",
"documents": [],
# Add workflow context fields
"roundNumber": workflow.currentRound,
"taskNumber": 0,
"actionNumber": 0,
# Add progress status
"taskProgress": "stopped",
"actionProgress": "stopped"
}
self.services.workflow.storeMessageWithDocuments(workflow, stopped_message, [])
# Update workflow status to stopped
workflow.status = "stopped"
workflow.lastActivity = self.services.utils.timestampGetUtc()
self.services.workflow.updateWorkflow(workflow.id, {
"status": "stopped",
"lastActivity": workflow.lastActivity,
"totalTasks": workflow.totalTasks,
"totalActions": workflow.totalActions
})
# Add stopped log entry
self.services.workflow.storeLog(workflow, {
"message": "Workflow stopped by user",
"type": "warning",
"status": "stopped",
"progress": 100
})
return
elif workflow.status == 'failed':
# Create error message
error_message = {
"workflowId": workflow.id,
"role": "assistant",
"message": f"Workflow failed: {'Unknown error'}",
"status": "last",
"sequenceNr": len(workflow.messages) + 1,
"publishedAt": self.services.utils.timestampGetUtc(),
"documentsLabel": "workflow_failure",
"documents": [],
# Add workflow context fields
"roundNumber": workflow.currentRound,
"taskNumber": 0,
"actionNumber": 0,
# Add progress status
"taskProgress": "fail",
"actionProgress": "fail"
}
self.services.workflow.storeMessageWithDocuments(workflow, error_message, [])
# Update workflow status to failed
workflow.status = "failed"
workflow.lastActivity = self.services.utils.timestampGetUtc()
self.services.workflow.updateWorkflow(workflow.id, {
"status": "failed",
"lastActivity": workflow.lastActivity,
"totalTasks": workflow.totalTasks,
"totalActions": workflow.totalActions
})
# Add failed log entry
self.services.workflow.storeLog(workflow, {
"message": "Workflow failed: Unknown error",
"type": "error",
"status": "failed",
"progress": 100
})
return
# For successful workflows, send detailed completion message
await self._sendLastMessage(workflow)
except Exception as e:
logger.error(f"Error processing workflow results: {str(e)}")
# Create error message
error_message = {
"workflowId": workflow.id,
"role": "assistant",
"message": f"Error processing workflow results: {str(e)}",
"status": "last",
"sequenceNr": len(workflow.messages) + 1,
"publishedAt": self.services.utils.timestampGetUtc(),
"documentsLabel": "workflow_error",
"documents": [],
# Add workflow context fields
"roundNumber": workflow.currentRound,
"taskNumber": 0,
"actionNumber": 0,
# Add progress status
"taskProgress": "fail",
"actionProgress": "fail"
}
self.services.workflow.storeMessageWithDocuments(workflow, error_message, [])
# Update workflow status to failed
workflow.status = "failed"
workflow.lastActivity = self.services.utils.timestampGetUtc()
self.services.workflow.updateWorkflow(workflow.id, {
"status": "failed",
"lastActivity": workflow.lastActivity,
"totalTasks": workflow.totalTasks,
"totalActions": workflow.totalActions
})
async def _sendLastMessage(self, workflow: ChatWorkflow) -> None:
"""Send last message to complete workflow (only for successful workflows)"""
try:
# Safety check: ensure this is only called for successful workflows
if workflow.status in ['stopped', 'failed']:
logger.warning(f"Attempted to send last message for {workflow.status} workflow {workflow.id}")
return
# Generate feedback
feedback = await self._generateWorkflowFeedback(workflow)
# Create last message using interface
messageData = {
"workflowId": workflow.id,
"role": "assistant",
"message": feedback,
"status": "last",
"sequenceNr": len(workflow.messages) + 1,
"publishedAt": self.services.utils.timestampGetUtc(),
"documentsLabel": "workflow_feedback",
"documents": [],
# Add workflow context fields
"roundNumber": workflow.currentRound,
"taskNumber": 0,
"actionNumber": 0,
# Add progress status
"taskProgress": "success",
"actionProgress": "success"
}
# Create message using interface
self.services.workflow.storeMessageWithDocuments(workflow, messageData, [])
# Update workflow status to completed
workflow.status = "completed"
workflow.lastActivity = self.services.utils.timestampGetUtc()
# Update workflow in database
self.services.workflow.updateWorkflow(workflow.id, {
"status": "completed",
"lastActivity": workflow.lastActivity
})
# Add completion log entry
self.services.workflow.storeLog(workflow, {
"message": "Workflow completed",
"type": "success",
"status": "completed",
"progress": 100
})
except Exception as e:
logger.error(f"Error sending last message: {str(e)}")
raise
async def _generateWorkflowFeedback(self, workflow: ChatWorkflow) -> str:
"""Generate feedback message for workflow completion"""
try:
self.workflowProcessor._checkWorkflowStopped(workflow)
# Count messages by role
user_messages = [msg for msg in workflow.messages if msg.role == 'user']
assistant_messages = [msg for msg in workflow.messages if msg.role == 'assistant']
# Generate summary feedback
feedback = f"Workflow completed.\n\n"
feedback += f"Processed {len(user_messages)} user inputs and generated {len(assistant_messages)} responses.\n"
# Add final status
if workflow.status == "completed":
feedback += "All tasks completed successfully."
elif workflow.status == "partial":
feedback += "Some tasks completed with partial success."
else:
feedback += f"Workflow status: {workflow.status}"
return feedback
except Exception as e:
logger.error(f"Error generating workflow feedback: {str(e)}")
return "Workflow processing completed."
def _handleWorkflowStop(self, workflow: ChatWorkflow) -> None:
"""Handle workflow stop exception"""
logger.info("Workflow stopped by user")
# Update workflow status to stopped
workflow.status = "stopped"
workflow.lastActivity = self.services.utils.timestampGetUtc()
self.services.workflow.updateWorkflow(workflow.id, {
"status": "stopped",
"lastActivity": workflow.lastActivity,
"totalTasks": workflow.totalTasks,
"totalActions": workflow.totalActions
})
# Create final stopped message
stopped_message = {
"workflowId": workflow.id,
"role": "assistant",
"message": "🛑 Workflow stopped by user",
"status": "last",
"sequenceNr": len(workflow.messages) + 1,
"publishedAt": self.services.utils.timestampGetUtc(),
"documentsLabel": "workflow_stopped",
"documents": [],
# Add workflow context fields
"roundNumber": workflow.currentRound,
"taskNumber": 0,
"actionNumber": 0,
# Add progress status
"taskProgress": "pending",
"actionProgress": "pending"
}
self.services.workflow.storeMessageWithDocuments(workflow, stopped_message, [])
# Add log entry
self.services.workflow.storeLog(workflow, {
"message": "Workflow stopped by user",
"type": "warning",
"status": "stopped",
"progress": 100
})
def _handleWorkflowError(self, workflow: ChatWorkflow, error: Exception) -> None:
"""Handle workflow error exception"""
logger.error(f"Workflow processing error: {str(error)}")
# Update workflow status to failed
workflow.status = "failed"
workflow.lastActivity = self.services.utils.timestampGetUtc()
self.services.workflow.updateWorkflow(workflow.id, {
"status": "failed",
"lastActivity": workflow.lastActivity,
"totalTasks": workflow.totalTasks,
"totalActions": workflow.totalActions
})
# Create error message
error_message = {
"workflowId": workflow.id,
"role": "assistant",
"message": f"Workflow processing failed: {str(error)}",
"status": "last",
"sequenceNr": len(workflow.messages) + 1,
"publishedAt": self.services.utils.timestampGetUtc(),
"documentsLabel": "workflow_error",
"documents": [],
# Add workflow context fields
"roundNumber": workflow.currentRound,
"taskNumber": 0,
"actionNumber": 0,
# Add progress status
"taskProgress": "fail",
"actionProgress": "fail"
}
self.services.workflow.storeMessageWithDocuments(workflow, error_message, [])
# Add error log entry
self.services.workflow.storeLog(workflow, {
"message": f"Workflow failed: {str(error)}",
"type": "error",
"status": "failed",
"progress": 100
})
raise
async def _processFileIds(self, fileIds: List[str], messageId: str = None) -> List[ChatDocument]:
"""Process file IDs from existing files and return ChatDocument objects"""
documents = []
for fileId in fileIds:
try:
# Get file info from unified workflow service
fileInfo = self.services.workflow.getFileInfo(fileId)
if fileInfo:
# Create document directly with all file attributes
document = ChatDocument(
id=str(uuid.uuid4()),
messageId=messageId or "", # Use provided messageId or empty string as fallback
fileId=fileId,
fileName=fileInfo.get("fileName", "unknown"),
fileSize=fileInfo.get("size", 0),
mimeType=fileInfo.get("mimeType", "application/octet-stream")
)
documents.append(document)
logger.info(f"Processed file ID {fileId} -> {document.fileName}")
else:
logger.warning(f"No file info found for file ID {fileId}")
except Exception as e:
logger.error(f"Error processing file ID {fileId}: {str(e)}")
return documents
def _setUserLanguage(self, language: str) -> None:
"""Set user language for the service center"""
self.services.user.language = language