PowerOn/gateway
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

enhanced document handover with primary search over full string and secondary search over round/task/action number only

Browse source
This commit is contained in:
ValueOn AG 2025-08-25 22:36:21 +02:00
parent bd0d964e93
commit 0895975478
15 changed files with 726 additions and 382 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
modules/chat/documents/documentGeneration.py
View file

@ -18,7 +18,8 @@ class DocumentGenerator:
def processActionResultDocuments(self, action_result, action, workflow) -> List[Dict[str, Any]]:
"""
Main function to process documents from an action result.
Process documents produced by AI actions and convert them to ChatDocument format.
This function handles AI-generated document data, not document references.
Returns a list of processed document dictionaries.
"""
try:
@ -31,18 +32,7 @@ class DocumentGenerator:
logger.info(f"Processing {len(documents)} documents from action_result.documents")
# Check if documents are references (strings starting with "docItem:") or actual document objects
if documents and isinstance(documents[0], str) and documents[0].startswith("docItem:"):
# These are document references, resolve them to actual documents
logger.info(f"Resolving {len(documents)} document references to actual documents")
try:
actual_documents = self.service.getChatDocumentsFromDocumentList(documents)
logger.info(f"Resolved {len(actual_documents)} actual documents from references")
documents = actual_documents
except Exception as e:
logger.error(f"Error resolving document references: {str(e)}")
return []
# Process each document from the AI action result
processed_documents = []
for doc in documents:
processed_doc = self.processSingleDocument(doc, action)

572
modules/chat/handling/handlingTasks.py
View file

@ -60,7 +60,9 @@ class HandlingTasks:
# Check workflow status before generating task plan
self._checkWorkflowStopped()
logger.info(f"Generating task plan for workflow {workflow.id}")
logger.info(f"=== STARTING TASK PLAN GENERATION ===")
logger.info(f"Workflow ID: {workflow.id}")
logger.info(f"User Input: {userInput}")
available_docs = self.service.getAvailableDocuments(workflow)
# Set initial workflow context
@ -95,12 +97,39 @@ class HandlingTasks:
is_regeneration=False,
failure_patterns=[],
failed_actions=[],
successful_actions=[]
successful_actions=[],
criteria_progress={
'met_criteria': set(),
'unmet_criteria': set(),
'attempt_history': []
}
)
prompt = await self.service.callAiTextAdvanced(
createTaskPlanningPrompt(task_planning_context, self.service)
)
# Generate the task planning prompt
task_planning_prompt = createTaskPlanningPrompt(task_planning_context, self.service)
# Log the full task planning prompt being sent to AI for debugging
logger.info("=== TASK PLANNING PROMPT SENT TO AI ===")
logger.info(f"User Input: {userInput}")
logger.info(f"Available Documents: {len(available_docs) if available_docs else 0}")
logger.info("=== FULL TASK PLANNING PROMPT ===")
logger.info(task_planning_prompt)
logger.info("=== END TASK PLANNING PROMPT ===")
prompt = await self.service.callAiTextAdvanced(task_planning_prompt)
# Check if AI response is valid
if not prompt:
raise ValueError("AI service returned no response for task planning")
# Log the full AI response for task planning
logger.info("=== TASK PLANNING AI RESPONSE RECEIVED ===")
logger.info(f"Response length: {len(prompt) if prompt else 0}")
logger.info(f"Response preview: {prompt[:500] if prompt else 'None'}...")
logger.info("=== FULL TASK PLANNING AI RESPONSE ===")
logger.info(prompt)
logger.info("=== END TASK PLANNING AI RESPONSE ===")
# Inline _parseTaskPlanResponse logic
try:
json_start = prompt.find('{')
@ -109,6 +138,7 @@ class HandlingTasks:
raise ValueError("No JSON found in response")
json_str = prompt[json_start:json_end]
task_plan_dict = json.loads(json_str)
if 'tasks' not in task_plan_dict:
raise ValueError("Task plan missing 'tasks' field")
except Exception as e:
@ -121,12 +151,29 @@ class HandlingTasks:
logger.error(f"Parsed Task Plan: {json.dumps(task_plan_dict, indent=2)}")
raise Exception("AI-generated task plan failed validation - AI is required for task planning")
if not task_plan_dict.get('tasks'):
raise ValueError("Task plan contains no tasks")
tasks = []
for task_dict in task_plan_dict.get('tasks', []):
for i, task_dict in enumerate(task_plan_dict.get('tasks', [])):
if not isinstance(task_dict, dict):
logger.warning(f"Skipping invalid task {i+1}: not a dictionary")
continue
# Map old 'description' field to new 'objective' field
if 'description' in task_dict and 'objective' not in task_dict:
task_dict['objective'] = task_dict.pop('description')
tasks.append(TaskStep(**task_dict))
try:
task = TaskStep(**task_dict)
tasks.append(task)
except Exception as e:
logger.warning(f"Skipping invalid task {i+1}: {str(e)}")
continue
if not tasks:
raise ValueError("No valid tasks could be created from AI response")
task_plan = TaskPlan(
overview=task_plan_dict.get('overview', ''),
tasks=tasks
@ -134,26 +181,13 @@ class HandlingTasks:
# Set workflow totals for progress tracking
total_tasks = len(tasks)
if total_tasks == 0:
raise ValueError("Task plan contains no valid tasks")
self.service.setWorkflowTotals(total_tasks=total_tasks)
logger.info(f"Task plan generated successfully with {len(tasks)} tasks")
# Log the generated tasks
for i, task in enumerate(tasks):
logger.info(f" Task {i+1}: {task.objective}")
if task.success_criteria:
logger.info(f" Success criteria: {task.success_criteria}")
# Log the complete task plan
logger.info("=== GENERATED TASK PLAN ===")
logger.info(f"Overview: {task_plan.overview}")
logger.info(f"Total tasks: {len(tasks)}")
# Log the RAW AI-generated task plan JSON for debugging
logger.info("=== RAW AI TASK PLAN JSON ===")
logger.info(f"AI Response with task plan: {prompt}")
logger.info("=== END RAW AI TASK PLAN JSON ===")
# PHASE 3: Create chat message containing the task plan
await self.createTaskPlanMessage(task_plan, workflow)
@ -212,135 +246,45 @@ class HandlingTasks:
logger.error(f"Error creating task plan message: {str(e)}")
async def createDocumentContextMessage(self, documents: List, workflow):
"""Create a chat message with enhanced document context and workflow labeling"""
"""Create a chat message with document context and workflow labeling"""
try:
from .promptFactory import createDocumentContextPrompt
# Get user language from service
user_language = self.service.user.language if self.service and self.service.user else 'en'
# Get current workflow context and stats
workflow_context = self.service.getWorkflowContext()
workflow_stats = self.service.getWorkflowStats()
# Build context for the document context prompt
context = {
'documents': documents,
'workflow_context': {
'currentRound': workflow_context.get('currentRound', 1),
'totalTasks': workflow_stats.get('totalTasks', 0),
'currentTask': workflow_context.get('currentTask', 0),
'totalActions': workflow_stats.get('totalActions', 0),
'currentAction': workflow_context.get('currentAction', 0),
'workflowStatus': workflow_stats.get('workflowStatus', 'unknown'),
'workflowId': workflow_stats.get('workflowId', 'unknown')
},
'user_language': user_language
}
# Create a simple document context message without AI dependency
message_text = f"📄 **Document Context**\n\n"
message_text += f"**Total Documents:** {len(documents)}\n\n"
# Generate enhanced document context using AI
prompt = createDocumentContextPrompt(context)
response = await self.service.callAiTextAdvanced(prompt)
# Add workflow context information
current_round = workflow_context.get('currentRound', 1)
current_task = workflow_context.get('currentTask', 0)
total_tasks = workflow_stats.get('totalTasks', 0)
current_action = workflow_context.get('currentAction', 0)
total_actions = workflow_stats.get('totalActions', 0)
# Parse the AI response
try:
json_start = response.find('{')
json_end = response.find('}') + 1
if json_start != -1 and json_end > 0:
json_str = response[json_start:json_end]
doc_context = json.loads(json_str)
# Build message from AI response
message_text = f"📄 **Document Context**\n\n"
message_text += f"**Summary:** {doc_context.get('documentSummary', 'No summary available')}\n\n"
message_text += f"**Workflow Progress:** {doc_context.get('workflowProgress', 'No progress info')}\n\n"
# Add workflow context information
current_round = workflow_context.get('currentRound', 1)
current_task = workflow_context.get('currentTask', 0)
total_tasks = workflow_stats.get('totalTasks', 0)
current_action = workflow_context.get('currentAction', 0)
total_actions = workflow_stats.get('totalActions', 0)
message_text += f"**Workflow Context:**\n"
message_text += f"- Round: {current_round}\n"
if total_tasks > 0:
message_text += f"- Task: {current_task}/{total_tasks}\n"
else:
message_text += f"- Task: {current_task}\n"
if total_actions > 0:
message_text += f"- Action: {current_action}/{total_actions}\n"
else:
message_text += f"- Action: {current_action}\n"
message_text += f"- Status: {workflow_stats.get('workflowStatus', 'unknown')}\n\n"
# Add overall user message if available
overall_message = doc_context.get('overallUserMessage')
if overall_message:
message_text += f"💬 {overall_message}\n\n"
# Add document details
document_details = doc_context.get('documentDetails', [])
if document_details:
message_text += "**Document Details:**\n"
for doc_detail in document_details:
message_text += f"- {doc_detail.get('workflowLabel', 'Unknown')}: {doc_detail.get('fileName', 'Unknown file')}\n"
user_msg = doc_detail.get('userMessage')
if user_msg:
message_text += f" 💬 {user_msg}\n"
message_text += "\n"
else:
# Fallback if AI response parsing fails
message_text = f"📄 **Document Context**\n\n"
message_text += f"**Total Documents:** {len(documents)}\n\n"
# Add workflow context information even in fallback
current_round = workflow_context.get('currentRound', 1)
current_task = workflow_context.get('currentTask', 0)
total_tasks = workflow_stats.get('totalTasks', 0)
current_action = workflow_context.get('currentAction', 0)
total_actions = workflow_stats.get('totalActions', 0)
message_text += f"**Workflow Context:**\n"
message_text += f"- Round: {current_round}\n"
if total_tasks > 0:
message_text += f"- Task: {current_task}/{total_tasks}\n"
else:
message_text += f"- Task: {current_task}\n"
if total_actions > 0:
message_text += f"- Action: {current_action}/{total_actions}\n"
else:
message_text += f"- Action: {current_action}\n"
message_text += f"- Status: {workflow_stats.get('workflowStatus', 'unknown')}\n\n"
message_text += "Document context information is available for processing."
except Exception as e:
logger.error(f"Error parsing document context AI response: {str(e)}")
# Fallback message with workflow context
message_text = f"📄 **Document Context**\n\n"
message_text += f"**Total Documents:** {len(documents)}\n\n"
# Add workflow context information in fallback
current_round = workflow_context.get('currentRound', 1)
current_task = workflow_context.get('currentTask', 0)
total_tasks = workflow_stats.get('totalTasks', 0)
current_action = workflow_context.get('currentAction', 0)
total_actions = workflow_stats.get('totalActions', 0)
message_text += f"**Workflow Context:**\n"
message_text += f"- Round: {current_round}\n"
if total_tasks > 0:
message_text += f"- Task: {current_task}/{total_tasks}\n"
else:
message_text += f"- Task: {current_task}\n"
if total_actions > 0:
message_text += f"- Action: {current_action}/{total_actions}\n"
else:
message_text += f"- Action: {current_action}\n"
message_text += f"- Status: {workflow_stats.get('workflowStatus', 'unknown')}\n\n"
message_text += "Document context information is available for processing."
message_text += f"**Workflow Context:**\n"
message_text += f"- Round: {current_round}\n"
if total_tasks > 0:
message_text += f"- Task: {current_task}/{total_tasks}\n"
else:
message_text += f"- Task: {current_task}\n"
if total_actions > 0:
message_text += f"- Action: {current_action}/{total_actions}\n"
else:
message_text += f"- Action: {current_action}\n"
message_text += f"- Status: {workflow_stats.get('workflowStatus', 'unknown')}\n\n"
# Add document list
if documents:
message_text += "**Available Documents:**\n"
for i, doc in enumerate(documents[:5]): # Show first 5 documents
message_text += f"- {doc.fileName if hasattr(doc, 'fileName') else f'Document {i+1}'}\n"
if len(documents) > 5:
message_text += f"- ... and {len(documents) - 5} more documents\n"
message_text += "\n"
message_text += "Document context information is available for processing."
# Create workflow message
message_data = {
@ -351,7 +295,7 @@ class HandlingTasks:
"sequenceNr": len(workflow.messages) + 1,
"publishedAt": get_utc_timestamp(),
"documentsLabel": "document_context",
"documents": documents,
"documents": [], # Empty documents for context message
# Add workflow context fields
"roundNumber": workflow_context.get('currentRound', 1),
"taskNumber": workflow_context.get('currentTask', 0),
@ -374,11 +318,61 @@ class HandlingTasks:
# Check workflow status before generating actions
self._checkWorkflowStopped()
logger.info(f"Generating actions for task: {task_step.objective}")
retry_info = f" (Retry #{enhanced_context.retry_count})" if enhanced_context and enhanced_context.retry_count > 0 else ""
logger.info(f"Generating actions for task: {task_step.objective}{retry_info}")
# Log criteria progress if this is a retry
if enhanced_context and hasattr(enhanced_context, 'criteria_progress') and enhanced_context.criteria_progress is not None:
progress = enhanced_context.criteria_progress
logger.info(f"Retry attempt {enhanced_context.retry_count} - Criteria progress:")
if progress.get('met_criteria'):
logger.info(f" Met criteria: {', '.join(progress['met_criteria'])}")
if progress.get('unmet_criteria'):
logger.warning(f" Unmet criteria: {', '.join(progress['unmet_criteria'])}")
# Show improvement trends
if progress.get('attempt_history'):
recent_attempts = progress['attempt_history'][-2:] # Last 2 attempts
if len(recent_attempts) >= 2:
prev_score = recent_attempts[0].get('quality_score', 0)
curr_score = recent_attempts[1].get('quality_score', 0)
if curr_score > prev_score:
logger.info(f" Quality improving: {prev_score} -> {curr_score}")
elif curr_score < prev_score:
logger.warning(f" Quality declining: {prev_score} -> {curr_score}")
else:
logger.info(f" Quality stable: {curr_score}")
# Enhanced retry context logging
if enhanced_context and enhanced_context.retry_count > 0:
logger.info("=== RETRY CONTEXT FOR ACTION GENERATION ===")
logger.info(f"Retry Count: {enhanced_context.retry_count}")
logger.info(f"Previous Improvements: {enhanced_context.improvements}")
logger.info(f"Previous Review Result: {enhanced_context.previous_review_result}")
logger.info(f"Failure Patterns: {enhanced_context.failure_patterns}")
logger.info(f"Failed Actions: {enhanced_context.failed_actions}")
logger.info(f"Successful Actions: {enhanced_context.successful_actions}")
logger.info("=== END RETRY CONTEXT ===")
available_docs = self.service.getAvailableDocuments(workflow)
available_connections = self.service.getConnectionReferenceList()
# Log available resources for debugging
logger.info("=== AVAILABLE RESOURCES FOR ACTION GENERATION ===")
logger.info(f"Available Documents: {len(available_docs) if available_docs else 0}")
if available_docs:
for i, doc in enumerate(available_docs[:5]): # Show first 5
logger.info(f" Doc {i+1}: {doc}")
if len(available_docs) > 5:
logger.info(f" ... and {len(available_docs) - 5} more documents")
logger.info(f"Available Connections: {len(available_connections) if available_connections else 0}")
if available_connections:
for i, conn in enumerate(available_connections[:5]): # Show first 5
logger.info(f" Conn {i+1}: {conn}")
if len(available_connections) > 5:
logger.info(f" ... and {len(available_connections) - 5} more connections")
logger.info("=== END AVAILABLE RESOURCES ===")
# Create proper context object for action definition
if enhanced_context and isinstance(enhanced_context, TaskContext):
# Use existing TaskContext if provided
@ -397,7 +391,8 @@ class HandlingTasks:
is_regeneration=enhanced_context.is_regeneration or False,
failure_patterns=enhanced_context.failure_patterns or [],
failed_actions=enhanced_context.failed_actions or [],
successful_actions=enhanced_context.successful_actions or []
successful_actions=enhanced_context.successful_actions or [],
criteria_progress=enhanced_context.criteria_progress
)
else:
# Create new context from scratch
@ -416,66 +411,106 @@ class HandlingTasks:
is_regeneration=False,
failure_patterns=[],
failed_actions=[],
successful_actions=[]
successful_actions=[],
criteria_progress=None
)
# Check workflow status before calling AI service
self._checkWorkflowStopped()
prompt = await self.service.callAiTextAdvanced(
await createActionDefinitionPrompt(action_context, self.service)
)
# Log the final action context being sent to AI
logger.info("=== FINAL ACTION CONTEXT FOR AI ===")
logger.info(f"Task Step ID: {action_context.task_step.id if action_context.task_step else 'None'}")
logger.info(f"Task Step Objective: {action_context.task_step.objective if action_context.task_step else 'None'}")
logger.info(f"Workflow ID: {action_context.workflow_id}")
logger.info(f"Available Documents Count: {len(action_context.available_documents) if action_context.available_documents else 0}")
logger.info(f"Available Connections Count: {len(action_context.available_connections) if action_context.available_connections else 0}")
logger.info(f"Previous Results Count: {len(action_context.previous_results) if action_context.previous_results else 0}")
logger.info(f"Retry Count: {action_context.retry_count}")
logger.info(f"Is Regeneration: {action_context.is_regeneration}")
logger.info("=== END ACTION CONTEXT ===")
# Generate the action definition prompt
action_prompt = await createActionDefinitionPrompt(action_context, self.service)
# Log the full prompt being sent to AI for debugging
logger.info("=== ACTION DEFINITION PROMPT SENT TO AI ===")
logger.info(f"Task: {task_step.objective}")
logger.info(f"Retry Count: {action_context.retry_count}")
logger.info(f"Previous Results: {action_context.previous_results}")
logger.info(f"Improvements: {action_context.improvements}")
logger.info(f"Previous Review Result: {action_context.previous_review_result}")
logger.info(f"Criteria Progress: {action_context.criteria_progress}")
logger.info("=== FULL PROMPT ===")
logger.info(action_prompt)
logger.info("=== END PROMPT ===")
prompt = await self.service.callAiTextAdvanced(action_prompt)
# Check if AI response is valid
if not prompt:
raise ValueError("AI service returned no response")
# Log the full AI response for debugging
logger.info("=== FULL AI RESPONSE ===")
logger.info(prompt)
logger.info("=== END AI RESPONSE ===")
# Inline parseActionResponse logic here
json_start = prompt.find('{')
json_end = prompt.rfind('}') + 1
if json_start == -1 or json_end == 0:
raise ValueError("No JSON found in response")
json_str = prompt[json_start:json_end]
try:
action_data = json.loads(json_str)
except Exception as e:
logger.error(f"Error parsing action response JSON: {str(e)}")
action_data = {}
if 'actions' not in action_data:
raise ValueError("Action response missing 'actions' field")
actions = action_data['actions']
if not actions:
raise ValueError("Action response contains empty actions list")
if not isinstance(actions, list):
raise ValueError(f"Action response 'actions' field is not a list: {type(actions)}")
if not self._validateActions(actions, action_context):
logger.error("Generated actions failed validation")
raise Exception("AI-generated actions failed validation - AI is required for action generation")
# Convert to TaskAction objects
task_actions = [self.createTaskAction({
"execMethod": a.get('method', 'unknown'),
"execAction": a.get('action', 'unknown'),
"execParameters": a.get('parameters', {}),
"execResultLabel": a.get('resultLabel', ''),
"expectedDocumentFormats": a.get('expectedDocumentFormats', None),
"status": TaskStatus.PENDING,
# Extract user-friendly message if available
"userMessage": a.get('userMessage', None)
}) for a in actions]
task_actions = []
for i, a in enumerate(actions):
if not isinstance(a, dict):
logger.warning(f"Skipping invalid action {i+1}: not a dictionary")
continue
task_action = self.createTaskAction({
"execMethod": a.get('method', 'unknown'),
"execAction": a.get('action', 'unknown'),
"execParameters": a.get('parameters', {}),
"execResultLabel": a.get('resultLabel', ''),
"expectedDocumentFormats": a.get('expectedDocumentFormats', None),
"status": TaskStatus.PENDING,
# Extract user-friendly message if available
"userMessage": a.get('userMessage', None)
})
if task_action:
task_actions.append(task_action)
else:
logger.warning(f"Skipping invalid action {i+1}: failed to create TaskAction")
valid_actions = [ta for ta in task_actions if ta]
logger.info(f"Generated {len(valid_actions)} actions for task: {task_step.objective}")
# Log the generated actions
for i, action in enumerate(valid_actions):
logger.info(f" Action {i+1}: {action.execMethod}.{action.execAction}")
if action.expectedDocumentFormats:
logger.info(f" Expected formats: {action.expectedDocumentFormats}")
if action.execParameters.get('documentList'):
logger.info(f" Input documents: {action.execParameters['documentList']}")
# Log the complete action plan
logger.info("=== GENERATED ACTION PLAN ===")
logger.info(f"Task: {task_step.objective}")
logger.info(f"Total actions: {len(valid_actions)}")
# Log the RAW AI-generated action plan JSON for debugging
logger.info("=== RAW AI ACTION PLAN JSON ===")
logger.info(f"AI Response with parsed actions: {prompt}")
logger.info("=== END RAW AI ACTION PLAN JSON ===")
if not valid_actions:
raise ValueError("No valid actions could be created from AI response")
return valid_actions
except Exception as e:
logger.error(f"Error in generateTaskActions: {str(e)}")
@ -488,7 +523,7 @@ class HandlingTasks:
# Update workflow context for this task
if task_index is not None:
self.service.setWorkflowContext(task_number=task_index)
self.service.incrementWorkflowContext('task')
# Remove the increment call that causes double-increment bug
# Create database log entry for task start in format expected by frontend
if task_index is not None:
@ -569,7 +604,7 @@ class HandlingTasks:
# Update workflow context for this action
action_number = action_idx + 1
self.service.setWorkflowContext(action_number=action_number)
self.service.incrementWorkflowContext('action')
# Remove the increment call that causes double-increment bug
# Log action start in format expected by frontend
logger.info(f"Task {task_index} - Starting action {action_number}/{total_actions}")
@ -643,10 +678,23 @@ class HandlingTasks:
# Create a task completion message for the user
task_progress = f"{task_index}/{total_tasks}" if total_tasks is not None else str(task_index)
# Enhanced completion message with criteria details
completion_message = f"🎯 Task {task_progress} Completed Successfully!\n\nObjective: {task_step.objective}\n\nFeedback: {feedback or 'Task completed successfully'}"
# Add criteria status if available
if hasattr(review_result, 'met_criteria') and review_result.met_criteria:
completion_message += f"\n\n✅ **Success Criteria Met:**\n"
for criterion in review_result.met_criteria:
completion_message += f"• {criterion}\n"
if hasattr(review_result, 'quality_score'):
completion_message += f"\n📊 **Quality Score:** {review_result.quality_score}/10"
task_completion_message = {
"workflowId": workflow.id,
"role": "assistant",
"message": f"🎯 Task {task_progress} Completed Successfully!\n\nObjective: {task_step.objective}\n\nFeedback: {feedback or 'Task completed successfully'}",
"message": completion_message,
"status": "step",
"sequenceNr": len(workflow.messages) + 1,
"publishedAt": get_utc_timestamp(),
@ -674,11 +722,19 @@ class HandlingTasks:
feedback=feedback,
error=None
)
elif review_result.status == 'retry' and state.canRetry():
logger.warning(f"Task step '{task_step.objective}' requires retry: {review_result.improvements}")
# Enhanced logging of criteria status
if review_result.met_criteria:
logger.info(f"Met criteria: {', '.join(review_result.met_criteria)}")
if review_result.unmet_criteria:
logger.warning(f"Unmet criteria: {', '.join(review_result.unmet_criteria)}")
state.incrementRetryCount()
# Update retry context with retry information
# Update retry context with retry information and criteria tracking
if retry_context:
retry_context.retry_count = state.retry_count
retry_context.improvements = review_result.improvements
@ -688,6 +744,47 @@ class HandlingTasks:
retry_context.failure_patterns = state.getFailurePatterns()
retry_context.failed_actions = state.failed_actions
retry_context.successful_actions = state.successful_actions
# Track criteria progress across retries
if not hasattr(retry_context, 'criteria_progress'):
retry_context.criteria_progress = {
'met_criteria': set(),
'unmet_criteria': set(),
'attempt_history': []
}
# Update criteria progress - convert lists to sets for deduplication
if review_result.met_criteria:
retry_context.criteria_progress['met_criteria'].update(review_result.met_criteria)
if review_result.unmet_criteria:
retry_context.criteria_progress['unmet_criteria'].update(review_result.unmet_criteria)
# Record this attempt's criteria status
attempt_record = {
'attempt': state.retry_count,
'met_criteria': review_result.met_criteria or [],
'unmet_criteria': review_result.unmet_criteria or [],
'quality_score': review_result.quality_score,
'improvements': review_result.improvements or []
}
retry_context.criteria_progress['attempt_history'].append(attempt_record)
logger.info(f"Criteria progress after {state.retry_count} attempts:")
logger.info(f" Total met: {len(retry_context.criteria_progress['met_criteria'])}")
logger.info(f" Total unmet: {len(retry_context.criteria_progress['unmet_criteria'])}")
if retry_context.criteria_progress['met_criteria']:
logger.info(f" Met criteria: {', '.join(retry_context.criteria_progress['met_criteria'])}")
if retry_context.criteria_progress['unmet_criteria']:
logger.info(f" Unmet criteria: {', '.join(retry_context.criteria_progress['unmet_criteria'])}")
# Log retry summary for debugging
logger.info(f"=== RETRY #{state.retry_count} SUMMARY ===")
logger.info(f"Task: {task_step.objective}")
logger.info(f"Quality Score: {review_result.quality_score}/10")
logger.info(f"Status: {review_result.status}")
logger.info(f"Improvements Needed: {review_result.improvements}")
logger.info(f"Reason: {review_result.reason}")
logger.info("=== END RETRY SUMMARY ===")
continue
else:
@ -698,8 +795,18 @@ class HandlingTasks:
error_message += f"Objective: {task_step.objective}\n\n"
# Add specific error details if available
if error:
error_message += f"Error: {error}\n\n"
if review_result and hasattr(review_result, 'reason') and review_result.reason:
error_message += f"Reason: {review_result.reason}\n\n"
# Add criteria progress information if available
if retry_context and hasattr(retry_context, 'criteria_progress'):
progress = retry_context.criteria_progress
error_message += f"📊 **Progress Summary:**\n"
if progress.get('met_criteria'):
error_message += f"✅ Met criteria: {', '.join(progress['met_criteria'])}\n"
if progress.get('unmet_criteria'):
error_message += f"❌ Unmet criteria: {', '.join(progress['unmet_criteria'])}\n"
error_message += "\n"
# Add retry information
error_message += f"Attempts: {attempt+1}\n"
@ -733,14 +840,14 @@ class HandlingTasks:
else:
logger.error(f"Failed to create user-facing retry message for failed task: {task_step.objective}")
except Exception as e:
logger.error(f"Error creating user-facing retry message: {str(e)}")
logger.error(f"Error creating user-facing retry message: {str(e)}")
return TaskResult(
taskId=task_step.id,
status=TaskStatus.FAILED,
success=False,
feedback=feedback,
error=error
error=review_result.reason if review_result and hasattr(review_result, 'reason') else "Task failed after retry attempts"
)
logger.error(f"=== TASK {task_index or '?'} FAILED AFTER ALL RETRIES: {task_step.objective} ===")
@ -749,8 +856,10 @@ class HandlingTasks:
error_message += f"Objective: {task_step.objective}\n\n"
# Add specific error details if available
if error and error != "Task failed after all retries.":
error_message += f"Error: {error}\n\n"
if retry_context and hasattr(retry_context, 'previous_review_result') and retry_context.previous_review_result:
reason = retry_context.previous_review_result.get('reason', '')
if reason and reason != "Task failed after all retries.":
error_message += f"Reason: {reason}\n\n"
# Add retry information
error_message += f"Retries attempted: {retry_context.retry_count if retry_context else 'Unknown'}\n"
@ -799,6 +908,11 @@ class HandlingTasks:
# Check workflow status before reviewing task completion
self._checkWorkflowStopped()
logger.info(f"=== STARTING TASK COMPLETION REVIEW ===")
logger.info(f"Task: {task_step.objective}")
logger.info(f"Actions executed: {len(task_actions) if task_actions else 0}")
logger.info(f"Action results: {len(action_results) if action_results else 0}")
# Create proper context object for result review
review_context = ReviewContext(
task_step=task_step,
@ -827,13 +941,32 @@ class HandlingTasks:
# Use promptFactory for review prompt
prompt = createResultReviewPrompt(review_context, self.service)
# Log the full result review prompt being sent to AI for debugging
logger.info("=== RESULT REVIEW PROMPT SENT TO AI ===")
logger.info(f"Task: {task_step.objective}")
logger.info(f"Action Results Count: {len(review_context.action_results) if review_context.action_results else 0}")
logger.info(f"Task Actions Count: {len(review_context.task_actions) if review_context.task_actions else 0}")
logger.info("=== FULL RESULT REVIEW PROMPT ===")
logger.info(prompt)
logger.info("=== END RESULT REVIEW PROMPT ===")
response = await self.service.callAiTextAdvanced(prompt)
# Log the full AI response for result review
logger.info("=== RESULT REVIEW AI RESPONSE RECEIVED ===")
logger.info(f"Response length: {len(response) if response else 0}")
logger.info("=== FULL RESULT REVIEW AI RESPONSE ===")
logger.info(response)
logger.info("=== END RESULT REVIEW AI RESPONSE ===")
# Inline parseReviewResponse logic here
json_start = response.find('{')
json_end = response.rfind('}') + 1
if json_start == -1 or json_end == 0:
raise ValueError("No JSON found in review response")
json_str = response[json_start:json_end]
try:
review = json.loads(json_str)
except Exception as e:
@ -888,6 +1021,12 @@ class HandlingTasks:
else:
logger.error(f"VALIDATION FAILED - Task failed: {review_result.reason}")
logger.info(f"=== TASK COMPLETION REVIEW FINISHED ===")
logger.info(f"Final Status: {review_result.status}")
logger.info(f"Quality Score: {review_result.quality_score}/10")
logger.info(f"Improvements: {review_result.improvements}")
logger.info("=== END REVIEW ===")
return review_result
except Exception as e:
logger.error(f"Error in reviewTaskCompletion: {str(e)}")
@ -897,20 +1036,33 @@ class HandlingTasks:
quality_score=0
)
async def prepareTaskHandover(self, task_step, task_actions, review_result, workflow):
async def prepareTaskHandover(self, task_step, task_actions, task_result, workflow):
try:
# Check workflow status before preparing task handover
self._checkWorkflowStopped()
# Log handover status summary
status = review_result.status if review_result else 'unknown'
met = review_result.met_criteria if review_result and review_result.met_criteria else []
status = task_result.status if task_result else 'unknown'
# Handle both TaskResult and ReviewResult objects
if hasattr(task_result, 'met_criteria'):
# This is a ReviewResult object
met = task_result.met_criteria if task_result.met_criteria else []
review_result = task_result.to_dict()
else:
# This is a TaskResult object
met = []
review_result = {
'status': task_result.status if task_result else 'unknown',
'reason': task_result.error if task_result and hasattr(task_result, 'error') else None,
'success': task_result.success if task_result else False
}
handover_data = {
'task_id': task_step.id,
'task_description': task_step.objective,
'actions': [action.to_dict() for action in task_actions],
'review_result': review_result.to_dict(),
'review_result': review_result,
'workflow_id': workflow.id,
'handover_time': get_utc_timestamp()
}
@ -1029,7 +1181,7 @@ class HandlingTasks:
await self.createActionMessage(action, result, workflow, message_result_label, created_documents, task_step, task_index)
# Log action results
logger.info(f"✓ Action completed successfully")
logger.info(f"Action completed successfully")
# Create database log entry for action completion
if total_actions is not None:
@ -1060,7 +1212,7 @@ class HandlingTasks:
logger.info("Output: No documents created")
else:
action.setError(result.error or "Action execution failed")
logger.error(f"✗ Action failed: {result.error}")
logger.error(f"Action failed: {result.error}")
# ⚠️ IMPORTANT: Create error message for failed actions so user can see what went wrong
await self.createActionMessage(action, result, workflow, result_label, [], task_step, task_index)
@ -1114,10 +1266,6 @@ class HandlingTasks:
if result_label is None:
result_label = action.execResultLabel
# Use provided documents or process them if not provided
if created_documents is None:
created_documents = self.documentGenerator.createDocumentsFromActionResult(result, action, workflow)
# Log delivered documents
if created_documents:
logger.info(f"Result label: {result_label} - {len(created_documents)} documents")
@ -1226,10 +1374,10 @@ class HandlingTasks:
"actionName": action.execAction,
"documentsLabel": result_label,
"documents": created_documents,
# Add workflow context fields
# Add workflow context fields - extract from result_label to match document reference
"roundNumber": workflow_context.get('currentRound', 1),
"taskNumber": task_index,
"actionNumber": workflow_context.get('currentAction', 0)
"actionNumber": self._extractActionNumberFromLabel(result_label) if result_label else workflow_context.get('currentAction', 0)
}
# Add user-friendly message if available
@ -1314,6 +1462,24 @@ class HandlingTasks:
logger.error(f"Error validating task plan: {str(e)}")
return False
def _extractActionNumberFromLabel(self, label: str) -> int:
"""Extract action number from a document label like 'round1_task1_action1_diagram_analysis'"""
try:
if not label or not isinstance(label, str):
return 0
# Parse label format: round{round}_task{task}_action{action}_{context}
if '_action' in label:
action_part = label.split('_action')[1]
if action_part and '_' in action_part:
action_number = action_part.split('_')[0]
return int(action_number)
return 0
except Exception as e:
logger.warning(f"Could not extract action number from label '{label}': {str(e)}")
return 0
def _validateActions(self, actions: List[Dict[str, Any]], context) -> bool:
try:
if not isinstance(actions, list):

63
modules/chat/handling/promptFactory.py
View file

@ -152,18 +152,19 @@ Previous review feedback:
You are an action generation AI that creates specific actions to accomplish a task step with user-friendly messages.
DOCUMENT REFERENCE TYPES:
- docItem: Reference to a single document. Format: "docItem:<id>:<label>"
- docList: Reference to a group of documents under a label. Format: <label> (e.g., "round1_task2_action3_results").
- Each docList label maps to a list of docItem references (see AVAILABLE DOCUMENTS).
- A label like "round1_task2_action3_results" refers to the output of action 3 in task 2 of round 1.
- docItem: Reference to a single document
- docList: Reference to a group of documents
- round{{round_number}}_task{{task_number}}_action{{action_number}}_{{context}}: Reference to resulting document list from previous action
USAGE GUIDE:
- Use docItem when you need a specific document: "docItem:doc_123:component_diagram.pdf"
- Use docList when you need all documents in a group: "docList:msg_456:AnalysisResults"
- Use round/task/action format when referencing outputs from previous actions: "round1_task1_action2_AnalysisResults"
CRITICAL DOCUMENT REFERENCE RULES:
- ONLY use the exact labels listed in AVAILABLE DOCUMENTS below
- NEVER invent new labels or use message IDs
- NEVER use formats like "msg_xxx:documents" or "task_X_results" (these will fail)
- ONLY use the exact labels shown in AVAILABLE DOCUMENTS
- ONLY use the exact labels listed in AVAILABLE DOCUMENTS below, or result labels from previous actions
- When generating multiple actions, you may only use as input documents those that are already present in AVAILABLE DOCUMENTS or produced by actions that come earlier in the list. Do NOT use as input any document label that will be produced by a later action.
- If AVAILABLE DOCUMENTS shows "NO DOCUMENTS AVAILABLE", you CANNOT create document extraction actions. Instead, create actions that generate new content or inform the user that documents are needed.
- If AVAILABLE DOCUMENTS shows "NO DOCUMENTS AVAILABLE", you CANNOT create document extraction actions. Instead, create actions that generate new content or inform the user that documents are needed, if you miss something.
TASK STEP: {context.task_step.objective if context.task_step else 'No task step specified'} (ID: {context.task_step.id if context.task_step else 'unknown'})
@ -192,16 +193,15 @@ AVAILABLE DOCUMENTS:
{available_documents_str}
DOCUMENT REFERENCE EXAMPLES:
✅ CORRECT: Use exact labels from AVAILABLE DOCUMENTS above
- "round1_task2_action3_results"
- "round1_task1_action1_input"
- "docItem:doc_abc:round1_task2_action3_webpage_content.html"
- "docList:msg123:round1_task2_action3_results" (supported format, but use actual labels instead)
✅ CORRECT: Use exact references from AVAILABLE DOCUMENTS above or result labels from previous actions
- "docList:msg_456:diagram_analysis_results" (access all documents in a list)
- "docItem:doc_123:component_diagram.pdf" (access specific document)
- "round1_task2_action3_contextinfo" (document list from previous action)
❌ INCORRECT: These will cause errors
- "msg_xxx:documents" (invalid format - missing docList/docItem prefix)
- "task_2_results" (not a valid label - use exact labels from AVAILABLE DOCUMENTS)
- Inventing message IDs instead of using actual document labels
- "task_2_results" (not a valid reference - use exact references from AVAILABLE DOCUMENTS)
- Inventing document IDs not produces from a preceeding action
PREVIOUS RESULTS: {previous_results_str}
IMPROVEMENTS NEEDED: {improvements_str}
@ -236,15 +236,16 @@ DOCUMENT ROUTING GUIDANCE:
INSTRUCTIONS:
- Generate actions to accomplish this task step using available documents, connections, and previous results
- Use docItem for single documents and docList labels for groups of documents as shown in AVAILABLE DOCUMENTS
- Use docItem for single documents and docList for groups of documents as shown in AVAILABLE DOCUMENTS
- If AVAILABLE DOCUMENTS shows "NO DOCUMENTS AVAILABLE", you cannot create document extraction actions. Instead, create actions that generate new content or inform the user that documents are needed.
- Always pass documentList as a LIST of references (docItem and/or docList) - this list CANNOT be empty for document extraction actions
- For referencing documents from previous actions, use the format "round{{round_number}}_task{{task_number}}_action{{action_number}}_{{context}}"
- For resultLabel, use the format: "round{{round_number}}_task{{task_id}}_action{{action_number}}_{{short_label}}" where:
- {{round_number}} = the current round number (e.g., 1)
- {{task_id}} = the current task's id (e.g., 1)
- {{action_number}} = the sequence number of the action within the task (e.g., 2)
- {{short_label}} = a short, descriptive label for the output (e.g., "analysis_results")
Example: "round1_task1_action2_analysis_results"
- {{short_label}} = a short, descriptive label for the output (e.g., "AnalysisResults")
Example: "round1_task1_action2_AnalysisResults"
- If this is a retry, ensure the new actions address the specific issues from previous attempts
- Follow the JSON structure below. All fields are required.
@ -255,10 +256,10 @@ REQUIRED JSON STRUCTURE:
"method": "method_name", // Use only the method name (e.g., "document")
"action": "action_name", // Use only the action name (e.g., "extract")
"parameters": {{
"documentList": ["docItem:doc_abc:round1_task1_action2_results", "round1_task1_action1_input"],
"documentList": ["docItem:doc_abc:round1_task1_action2_AnalysisResults", "round1_task1_action1_input"],
"aiPrompt": "Comprehensive AI prompt describing what to accomplish"
}},
"resultLabel": "round1_task1_action3_analysis_results",
"resultLabel": "round1_task1_action3_AnalysisResults",
"expectedDocumentFormats": [ // OPTIONAL: Specify expected document formats when needed
{{
"extension": ".txt",
@ -276,7 +277,7 @@ FIELD REQUIREMENTS:
- "method": Must be from AVAILABLE METHODS
- "action": Must be valid for the method
- "parameters": Method-specific, must include documentList as a list if required by the signature
- "resultLabel": Must follow the format above (e.g., "round1_task1_action3_analysis_results")
- "resultLabel": Must follow the format above (e.g., "round1_task1_action3_AnalysisResults")
- "expectedDocumentFormats": OPTIONAL - Only specify when you need to control output format
- Use when you need specific file types (e.g., CSV for data, JSON for structured output)
- Omit when format is flexible (e.g., folder queries with mixed file types)
@ -292,7 +293,7 @@ EXAMPLES OF GOOD ACTIONS:
"method": "document",
"action": "extract",
"parameters": {{
"documentList": ["docItem:doc_57520394-6b6d-41c2-b641-bab3fc6d7f4b:round1_task1_action1_candidate_profile.txt"],
"documentList": ["docItem:doc_57520394-6b6d-41c2-b641-bab3fc6d7f4b:candidate_profile.txt"],
"aiPrompt": "Extract and analyze the candidate's qualifications, experience, skills, and suitability for the product designer position. Identify key strengths, relevant experience, technical skills, and any areas of concern. Provide a comprehensive assessment that can be used for evaluation."
}},
"resultLabel": "round1_task1_action2_candidate_analysis",
@ -312,12 +313,12 @@ EXAMPLES OF GOOD ACTIONS:
"method": "document",
"action": "extract",
"parameters": {{
"documentList": ["round1_task1_action2_candidate_analysis", "round1_task1_action3_candidate_analysis", "round1_task1_action4_candidate_analysis"],
"aiPrompt": "Compare all three candidate profiles and create an evaluation matrix. Rate each candidate on technical skills, experience level, cultural fit, portfolio quality, and communication skills. Provide clear rankings and recommendations for the product designer position."
"documentList": ["docList:msg_456:candidate_analysis_results"],
"aiPrompt": "Compare all candidate profiles and create an evaluation matrix. Rate each candidate on technical skills, experience level, cultural fit, portfolio quality, and communication skills. Provide clear rankings and recommendations for the product designer position."
}},
"resultLabel": "round1_task1_action5_evaluation_matrix",
"description": "Create comprehensive evaluation matrix comparing all candidates",
"userMessage": "Ich vergleiche alle drei Kandidatenprofile und erstelle eine umfassende Bewertungsmatrix mit klaren Empfehlungen."
"userMessage": "Ich vergleiche alle Kandidatenprofile und erstelle eine umfassende Bewertungsmatrix mit klaren Empfehlungen."
}}
3. Data extraction with specific CSV format and user message:
@ -325,7 +326,7 @@ EXAMPLES OF GOOD ACTIONS:
"method": "document",
"action": "extract",
"parameters": {{
"documentList": ["docItem:doc_abc:round1_task1_action1_table_data.pdf"],
"documentList": ["docItem:doc_abc:table_data.pdf"],
"aiPrompt": "Extract all table data and convert to structured CSV format with proper headers and data types. IMPORTANT: Deliver pure CSV data without any markdown formatting, code blocks, or additional text. Output only the CSV content with proper headers and data rows."
}},
"resultLabel": "round1_task1_action2_structured_data",
@ -345,7 +346,7 @@ EXAMPLES OF GOOD ACTIONS:
"method": "document",
"action": "generateReport",
"parameters": {{
"documentList": ["round1_task1_action2_candidate_analysis", "round1_task1_action3_candidate_analysis", "round1_task1_action4_candidate_analysis"],
"documentList": ["docList:msg_456:candidate_analysis_results"],
"title": "Comprehensive Candidate Evaluation Report"
}},
"resultLabel": "round1_task1_action6_summary_report",
@ -400,7 +401,9 @@ IMPORTANT NOTES:
- If AVAILABLE DOCUMENTS shows "NO DOCUMENTS AVAILABLE", use example 6 above to create a status report action instead of document extraction.
- Always include a user-friendly userMessage for each action in the user's language ({user_language}).
- The examples above show German user messages as reference - adapt the language to match the USER LANGUAGE specified above."""
logging.debug(f"[ACTION PLAN PROMPT] Enhanced Document Context:\n{available_documents_str}\nUser Connections Section:\n{available_connections_str}\nAvailable Methods (detailed):\n{available_methods_str}")
return prompt
def createResultReviewPrompt(context: ReviewContext, service) -> str:
@ -473,7 +476,7 @@ REVIEW INSTRUCTIONS:
REQUIRED JSON STRUCTURE:
{{
"status": "success|partial|failed",
"status": "success|retry|failed",
"reason": "Brief explanation of the status",
"improvements": ["improvement1", "improvement2"],
"quality_score": 8, // 1-10 scale
@ -487,7 +490,7 @@ REQUIRED JSON STRUCTURE:
FIELD REQUIREMENTS:
- "status": Overall task completion status
- "success": All criteria met, high-quality outputs
- "partial": Some criteria met, outputs need improvement
- "retry": Some criteria met, outputs need improvement and retry
- "failed": Most criteria unmet, significant issues
- "reason": Clear explanation of why this status was assigned
- "improvements": List of specific, actionable improvements

7
modules/chat/managerChat.py
View file

@ -65,7 +65,12 @@ class ChatManager:
is_regeneration=False,
failure_patterns=[],
failed_actions=[],
successful_actions=[]
successful_actions=[],
criteria_progress={
'met_criteria': set(),
'unmet_criteria': set(),
'attempt_history': []
}
)
# Execute task (this handles action generation, execution, and review internally)

209
modules/chat/serviceCenter.py
View file

@ -433,7 +433,7 @@ class ServiceCenter:
return 0
def getEnhancedDocumentContext(self) -> str:
"""Get enhanced document context formatted for action planning prompts with technically clear labels"""
"""Get enhanced document context formatted for action planning prompts with proper docList and docItem references"""
try:
document_list = self.getDocumentReferenceList()
@ -444,14 +444,32 @@ class ServiceCenter:
if document_list["chat"]:
context += "CURRENT ROUND DOCUMENTS:\n"
for exchange in document_list["chat"]:
context += f"- {exchange.documentsLabel} contains {', '.join(exchange.documents)}\n"
# Generate docList reference for the exchange (using message ID)
doc_list_ref = f"docList:{exchange.documentsLabel}"
context += f"- {doc_list_ref} contains:\n"
# Generate docItem references for each document in the list
for doc_ref in exchange.documents:
if doc_ref.startswith("docItem:"):
context += f" - {doc_ref}\n"
else:
# Convert to proper docItem format if needed
context += f" - docItem:{doc_ref}\n"
context += "\n"
# Process history exchanges (previous rounds)
if document_list["history"]:
context += "WORKFLOW HISTORY DOCUMENTS:\n"
for exchange in document_list["history"]:
context += f"- {exchange.documentsLabel} contains {', '.join(exchange.documents)}\n"
# Generate docList reference for the exchange (using message ID)
doc_list_ref = f"docList:{exchange.documentsLabel}"
context += f"- {doc_list_ref} contains:\n"
# Generate docItem references for each document in the list
for doc_ref in exchange.documents:
if doc_ref.startswith("docItem:"):
context += f" - {doc_ref}\n"
else:
# Convert to proper docItem format if needed
context += f" - docItem:{doc_ref}\n"
context += "\n"
if not document_list["chat"] and not document_list["history"]:
@ -516,86 +534,121 @@ class ServiceCenter:
return None
def getDocumentReferenceFromChatDocument(self, document: ChatDocument, message: ChatMessage) -> str:
"""Get document reference using new label format: round+task+action+filename.extension"""
"""Get document reference using document ID and filename."""
try:
# Generate new document label
label = self.generateDocumentLabel(document, message)
return f"docItem:{document.id}:{label}"
# Use document ID and filename for simple reference
return f"docItem:{document.id}:{document.fileName}"
except Exception as e:
logger.error(f"Critical error creating document reference for document {document.id}: {str(e)}")
# Re-raise the error to prevent workflow from continuing with invalid data
raise
def getDocumentListReferenceFromChatMessage(self, message: ChatMessage) -> str:
"""Get document list reference using message ID and label."""
try:
# Use message ID and documentsLabel for document list reference
label = getattr(message, 'documentsLabel', f"message_{message.id}")
return f"docList:{message.id}:{label}"
except Exception as e:
logger.error(f"Critical error creating document list reference for message {message.id}: {str(e)}")
# Re-raise the error to prevent workflow from continuing with invalid data
raise
def getChatDocumentsFromDocumentList(self, documentList: List[str]) -> List[ChatDocument]:
"""Get ChatDocuments from a list of document references using new label format."""
"""Get ChatDocuments from a list of document references using all three formats."""
try:
all_documents = []
for doc_ref in documentList:
# Parse reference format
parts = doc_ref.split(':', 2) # Split into max 3 parts
if len(parts) < 3:
logger.debug(f"Invalid document reference format: {doc_ref}")
continue
ref_type = parts[0]
ref_id = parts[1]
ref_label = parts[2]
if ref_type == "docItem":
# Handle ChatDocument reference: docItem:<id>:<new_label>
for message in self.workflow.messages:
if message.documents:
for doc in message.documents:
if doc.id == ref_id:
all_documents.append(doc)
break
if any(doc.id == ref_id for doc in message.documents):
break
elif ref_type == "docList":
# Handle document list reference: docList:<message_id>:<new_label>
try:
message_id = ref_id
label = ref_label
# Find message by ID
target_message = None
if doc_ref.startswith("docItem:"):
# docItem:<id>:<filename> - extract ID and find document
parts = doc_ref.split(':')
if len(parts) >= 2:
doc_id = parts[1]
# Find the document by ID
for message in self.workflow.messages:
if message.documents:
for doc in message.documents:
if doc.id == doc_id:
all_documents.append(doc)
break
elif doc_ref.startswith("docList:"):
# docList:<messageId>:<label> - extract message ID and find document list
parts = doc_ref.split(':')
if len(parts) >= 2:
message_id = parts[1]
# Find the message by ID and get all its documents
for message in self.workflow.messages:
if str(message.id) == message_id:
target_message = message
if message.documents:
all_documents.extend(message.documents)
break
if target_message and target_message.documents:
# Parse new label format: round1_task2_action3_filename.ext
if label.startswith("round"):
# New format - extract context and find matching documents
label_parts = label.split('_', 3)
if len(label_parts) >= 4:
round_num = int(label_parts[0].replace('round', ''))
task_num = int(label_parts[1].replace('task', ''))
action_num = int(label_parts[2].replace('action', ''))
filename = label_parts[3]
# Check if message context matches
msg_round = target_message.roundNumber if hasattr(target_message, 'roundNumber') else 1
msg_task = target_message.taskNumber if hasattr(target_message, 'taskNumber') else 0
msg_action = target_message.actionNumber if hasattr(target_message, 'actionNumber') else 0
if (msg_round == round_num and
msg_task == task_num and
msg_action == action_num):
# Add documents that match the filename
for doc in target_message.documents:
if doc.fileName == filename:
all_documents.append(doc)
else:
# Direct label reference (round1_task2_action3_contextinfo)
# Search for messages with matching documentsLabel to find the actual documents
if doc_ref.startswith("round"):
# Parse round/task/action to find the corresponding document list
label_parts = doc_ref.split('_', 3)
if len(label_parts) >= 4:
round_num = int(label_parts[0].replace('round', ''))
task_num = int(label_parts[1].replace('task', ''))
action_num = int(label_parts[2].replace('action', ''))
context_info = label_parts[3]
logger.debug(f"Resolving round reference: round{round_num}_task{task_num}_action{action_num}_{context_info}")
logger.debug(f"Looking for messages with documentsLabel matching: {doc_ref}")
# Find messages with matching documentsLabel (this is the correct way!)
# In case of retries, we want the NEWEST message (most recent publishedAt)
matching_messages = []
for message in self.workflow.messages:
msg_documents_label = getattr(message, 'documentsLabel', '')
# Check if this message's documentsLabel matches our reference
if msg_documents_label == doc_ref:
# Found a matching message, collect it for comparison
matching_messages.append(message)
logger.debug(f"Found message {message.id} with matching documentsLabel: {msg_documents_label}")
# If we found matching messages, take the newest one (highest publishedAt)
if matching_messages:
# Sort by publishedAt descending (newest first)
matching_messages.sort(key=lambda msg: getattr(msg, 'publishedAt', 0), reverse=True)
newest_message = matching_messages[0]
logger.debug(f"Found {len(matching_messages)} matching messages, using newest: {newest_message.id} (publishedAt: {getattr(newest_message, 'publishedAt', 'unknown')})")
logger.debug(f"Newest message has {len(newest_message.documents) if newest_message.documents else 0} documents")
if newest_message.documents:
all_documents.extend(newest_message.documents)
logger.debug(f"Added {len(newest_message.documents)} documents from newest message {newest_message.id}")
else:
logger.debug(f"No documents found in newest message {newest_message.id}")
else:
logger.debug(f"Label does not follow new format: {label}")
continue
except Exception as e:
logger.error(f"Error processing docList reference {doc_ref}: {str(e)}")
continue
logger.debug(f"No messages found with documentsLabel: {doc_ref}")
# Fallback: also check if any message has this documentsLabel as a prefix
logger.debug(f"Trying fallback search for messages with documentsLabel containing: {doc_ref}")
fallback_messages = []
for message in self.workflow.messages:
msg_documents_label = getattr(message, 'documentsLabel', '')
if msg_documents_label and msg_documents_label.startswith(doc_ref):
fallback_messages.append(message)
logger.debug(f"Found fallback message {message.id} with documentsLabel: {msg_documents_label}")
if fallback_messages:
# Sort by publishedAt descending (newest first)
fallback_messages.sort(key=lambda msg: getattr(msg, 'publishedAt', 0), reverse=True)
newest_fallback = fallback_messages[0]
logger.debug(f"Using fallback message {newest_fallback.id} with documentsLabel: {getattr(newest_fallback, 'documentsLabel', 'unknown')}")
if newest_fallback.documents:
all_documents.extend(newest_fallback.documents)
logger.debug(f"Added {len(newest_fallback.documents)} documents from fallback message {newest_fallback.id}")
else:
logger.debug(f"No documents found in fallback message {newest_fallback.id}")
else:
logger.debug(f"No fallback messages found either")
logger.debug(f"Resolved {len(all_documents)} documents from document list: {documentList}")
return all_documents
except Exception as e:
logger.error(f"Error getting documents from document list: {str(e)}")
@ -615,13 +668,21 @@ class ServiceCenter:
logger.debug(f"getConnectionReferenceList: User connections type: {type(user_connections)}")
logger.debug(f"getConnectionReferenceList: User connections length: {len(user_connections) if user_connections else 0}")
refreshed_count = 0
for conn in user_connections:
# Get enhanced connection reference with state information
enhanced_ref = self.getConnectionReferenceFromUserConnection(conn)
logger.debug(f"getConnectionReferenceList: Enhanced ref for connection {conn.id}: {enhanced_ref}")
connections.append(enhanced_ref)
# Count refreshed tokens
if "refreshed" in enhanced_ref:
refreshed_count += 1
# Sort by connection reference
logger.debug(f"getConnectionReferenceList: Final connections list: {connections}")
if refreshed_count > 0:
logger.info(f"Refreshed {refreshed_count} connection tokens while building action planning prompt")
return sorted(connections)
def getConnectionReferenceFromUserConnection(self, connection: UserConnection) -> str:
@ -630,8 +691,9 @@ class ServiceCenter:
token = None
token_status = "unknown"
try:
# Use getConnectionToken to find token for this specific connection
token = self.interfaceApp.getConnectionToken(connection.id)
# Use getConnectionToken with auto_refresh=True to automatically refresh expired tokens
logger.debug(f"Getting connection token for connection {connection.id} with auto_refresh=True")
token = self.interfaceApp.getConnectionToken(connection.id, auto_refresh=True)
if token:
if hasattr(token, 'expiresAt') and token.expiresAt:
current_time = get_utc_timestamp()
@ -640,7 +702,12 @@ class ServiceCenter:
if current_time > token.expiresAt:
token_status = "expired"
else:
token_status = "valid"
# Check if this token was recently refreshed (within last 5 minutes)
time_since_creation = current_time - token.createdAt if hasattr(token, 'createdAt') else 0
if time_since_creation < 300: # 5 minutes
token_status = "valid (refreshed)"
else:
token_status = "valid"
else:
token_status = "no_expiration"
else:

81
modules/interfaces/interfaceAppObjects.py
View file

@ -708,7 +708,7 @@ class AppObjects:
"message": f"Error checking username availability: {str(e)}"
}
def saveAccessToken(self, token: Token) -> None:
def saveAccessToken(self, token: Token, replace_existing: bool = True) -> None:
"""Save an access token for the current user (must NOT have connectionId)"""
try:
# Validate that this is NOT a connection token
@ -728,6 +728,28 @@ class AppObjects:
if not token.createdAt:
token.createdAt = get_utc_timestamp()
# If replace_existing is True, delete old access tokens for this user and authority first
if replace_existing:
try:
old_tokens = self.db.getRecordset("tokens", recordFilter={
"userId": self.currentUser.id,
"authority": token.authority,
"connectionId": None # Ensure we only delete access tokens
})
deleted_count = 0
for old_token in old_tokens:
if old_token["id"] != token.id: # Don't delete the new token if it already exists
self.db.recordDelete("tokens", old_token["id"])
deleted_count += 1
logger.debug(f"Deleted old access token {old_token['id']} for user {self.currentUser.id} and authority {token.authority}")
if deleted_count > 0:
logger.info(f"Replaced {deleted_count} old access tokens for user {self.currentUser.id} and authority {token.authority}")
except Exception as e:
logger.warning(f"Failed to delete old access tokens for user {self.currentUser.id} and authority {token.authority}: {str(e)}")
# Continue with saving the new token even if deletion fails
# Convert to dict and ensure all fields are properly set
token_dict = token.dict()
# Ensure userId is set to current user
@ -743,7 +765,7 @@ class AppObjects:
logger.error(f"Error saving access token: {str(e)}")
raise
def saveConnectionToken(self, token: Token) -> None:
def saveConnectionToken(self, token: Token, replace_existing: bool = True) -> None:
"""Save a connection token (must have connectionId)"""
try:
# Validate that this IS a connection token
@ -763,6 +785,26 @@ class AppObjects:
if not token.createdAt:
token.createdAt = get_utc_timestamp()
# If replace_existing is True, delete old tokens for this connectionId first
if replace_existing:
try:
old_tokens = self.db.getRecordset("tokens", recordFilter={
"connectionId": token.connectionId
})
deleted_count = 0
for old_token in old_tokens:
if old_token["id"] != token.id: # Don't delete the new token if it already exists
self.db.recordDelete("tokens", old_token["id"])
deleted_count += 1
logger.debug(f"Deleted old token {old_token['id']} for connectionId {token.connectionId}")
if deleted_count > 0:
logger.info(f"Replaced {deleted_count} old tokens for connectionId {token.connectionId}")
except Exception as e:
logger.warning(f"Failed to delete old tokens for connectionId {token.connectionId}: {str(e)}")
# Continue with saving the new token even if deletion fails
# Convert to dict and ensure all fields are properly set
token_dict = token.dict()
# Ensure userId is set to current user
@ -809,9 +851,8 @@ class AppObjects:
# Try to refresh the token
refreshed_token = token_manager.refresh_token(latest_token)
if refreshed_token:
# Save the new token and delete the old one
# Save the new token (which will automatically replace old ones)
self.saveAccessToken(refreshed_token)
self.deleteAccessToken(authority)
return refreshed_token
else:
@ -840,6 +881,18 @@ class AppObjects:
"connectionId": connectionId
})
# Debug: Log what we found
logger.debug(f"getConnectionToken: Found {len(tokens)} tokens for connectionId {connectionId}")
if tokens:
for i, token in enumerate(tokens):
logger.debug(f"getConnectionToken: Token {i}: id={token.get('id')}, expiresAt={token.get('expiresAt')}, createdAt={token.get('createdAt')}")
else:
# Debug: Check if there are any tokens at all in the database
all_tokens = self.db.getRecordset("tokens", recordFilter={})
logger.debug(f"getConnectionToken: No tokens found for connectionId {connectionId}. Total tokens in database: {len(all_tokens)}")
if all_tokens:
logger.debug(f"getConnectionToken: Sample tokens: {[{'id': t.get('id'), 'connectionId': t.get('connectionId'), 'authority': t.get('authority')} for t in all_tokens[:3]]}")
if not tokens:
logger.warning(f"No connection token found for connectionId: {connectionId}")
return None
@ -848,26 +901,34 @@ class AppObjects:
tokens.sort(key=lambda x: x.get("expiresAt", 0), reverse=True)
latest_token = Token(**tokens[0])
# Check if token is expired
if latest_token.expiresAt and latest_token.expiresAt < get_utc_timestamp():
# Check if token is expired or expires within 30 minutes
current_time = get_utc_timestamp()
thirty_minutes = 30 * 60 # 30 minutes in seconds
if latest_token.expiresAt and latest_token.expiresAt < (current_time + thirty_minutes):
if auto_refresh:
logger.debug(f"getConnectionToken: Token expires soon, attempting refresh. expiresAt: {latest_token.expiresAt}, current_time: {current_time}")
# Import TokenManager here to avoid circular imports
from modules.security.tokenManager import TokenManager
token_manager = TokenManager()
# Try to refresh the token
logger.debug(f"getConnectionToken: Calling token_manager.refresh_token for token {latest_token.id}")
refreshed_token = token_manager.refresh_token(latest_token)
if refreshed_token:
# Save the new token and delete the old one
logger.debug(f"getConnectionToken: Token refresh successful, saving new token {refreshed_token.id}")
# Save the new token (which will automatically replace old ones)
self.saveConnectionToken(refreshed_token)
self.deleteConnectionTokenByConnectionId(connectionId)
logger.info(f"Proactively refreshed connection token for connectionId {connectionId} (expired in {latest_token.expiresAt - current_time} seconds)")
return refreshed_token
else:
logger.warning(f"Failed to refresh expired connection token for connectionId {connectionId}")
logger.warning(f"getConnectionToken: Token refresh failed for connectionId {connectionId}")
return None
else:
logger.warning(f"Connection token for connectionId {connectionId} is expired (expiresAt: {latest_token.expiresAt})")
logger.warning(f"Connection token for connectionId {connectionId} expires soon (expiresAt: {latest_token.expiresAt})")
return None
return latest_token

18
modules/interfaces/interfaceChatModel.py
View file

@ -750,6 +750,9 @@ class TaskContext(BaseModel, ModelMixin):
failed_actions: Optional[list] = []
successful_actions: Optional[list] = []
# Criteria progress tracking for retries
criteria_progress: Optional[dict] = None
def getDocumentReferences(self) -> List[str]:
"""Get all available document references"""
docs = self.available_documents or []
@ -826,4 +829,19 @@ class WorkflowResult(BaseModel, ModelMixin):
error: Optional[str] = None
phase: Optional[str] = None
# Register labels for WorkflowResult
register_model_labels(
"WorkflowResult",
{"en": "Workflow Result", "fr": "Résultat du workflow"},
{
"status": {"en": "Status", "fr": "Statut"},
"completed_tasks": {"en": "Completed Tasks", "fr": "Tâches terminées"},
"total_tasks": {"en": "Total Tasks", "fr": "Total des tâches"},
"execution_time": {"en": "Execution Time", "fr": "Temps d'exécution"},
"final_results_count": {"en": "Final Results Count", "fr": "Nombre de résultats finaux"},
"error": {"en": "Error", "fr": "Erreur"},
"phase": {"en": "Phase", "fr": "Phase"}
}
)

24
modules/interfaces/interfaceChatObjects.py
View file

@ -278,7 +278,29 @@ class ChatObjects:
# Sort messages by publishedAt timestamp to ensure chronological order
messages.sort(key=lambda x: x.get("publishedAt", x.get("timestamp", "0")))
return [ChatMessage(**msg) for msg in messages]
# Convert messages to ChatMessage objects with proper document handling
chat_messages = []
for msg in messages:
# Ensure documents field is properly converted to ChatDocument objects
if "documents" in msg and msg["documents"]:
try:
# Convert each document back to ChatDocument object
documents = []
for doc in msg["documents"]:
if isinstance(doc, dict):
documents.append(ChatDocument(**doc))
else:
documents.append(doc)
msg["documents"] = documents
except Exception as e:
logger.warning(f"Error converting documents for message {msg.get('id', 'unknown')}: {e}")
msg["documents"] = []
else:
msg["documents"] = []
chat_messages.append(ChatMessage(**msg))
return chat_messages
def createWorkflowMessage(self, messageData: Dict[str, Any]) -> ChatMessage:
"""Creates a message for a workflow if user has access."""

14
modules/methods/methodOutlook.py
View file

@ -164,11 +164,11 @@ class MethodOutlook(MethodBase):
return True
elif response.status_code == 403:
logger.error("❌ Permission denied - connection lacks necessary mail permissions")
logger.error("Permission denied - connection lacks necessary mail permissions")
logger.error("Required scopes: Mail.ReadWrite, Mail.Send, Mail.ReadWrite.Shared")
return False
else:
logger.warning(f"⚠️ Permission check returned status {response.status_code}")
logger.warning(f"Permission check returned status {response.status_code}")
return False
except Exception as e:
@ -732,13 +732,13 @@ class MethodOutlook(MethodBase):
message["attachments"].append(attachment)
else:
logger.warning(f"⚠️ No content found for attachment: {doc.fileName}")
logger.warning(f"No content found for attachment: {doc.fileName}")
except Exception as e:
logger.error(f"❌ Error reading attachment file {doc.fileName}: {str(e)}")
logger.error(f"Error reading attachment file {doc.fileName}: {str(e)}")
else:
logger.warning(f"⚠️ Attachment document has no fileId: {doc.fileName}")
else:
logger.warning(f"⚠️ No attachment documents found for reference: {attachment_ref}")
logger.warning(f"Attachment document has no fileId: {doc.fileName}")
else:
logger.warning(f"No attachment documents found for reference: {attachment_ref}")
# Create the draft message
# First, get the Drafts folder ID to ensure the draft is created there

3
modules/routes/routeSecurityGoogle.py
View file

@ -507,9 +507,8 @@ async def refresh_token(
refreshed_token = token_manager.refresh_token(current_token)
if refreshed_token:
# Save the new connection token and delete the old one
# Save the new connection token (which will automatically replace old ones)
appInterface.saveConnectionToken(refreshed_token)
appInterface.deleteConnectionTokenByConnectionId(google_connection.id)
# Update the connection's expiration time
google_connection.expiresAt = float(refreshed_token.expiresAt)

3
modules/routes/routeSecurityMsft.py
View file

@ -515,9 +515,8 @@ async def refresh_token(
refreshed_token = token_manager.refresh_token(current_token)
if refreshed_token:
# Save the new connection token and delete the old one
# Save the new connection token (which will automatically replace old ones)
appInterface.saveConnectionToken(refreshed_token)
appInterface.deleteConnectionTokenByConnectionId(msft_connection.id)
# Update the connection's expiration time
msft_connection.expiresAt = float(refreshed_token.expiresAt)

15
modules/security/tokenManager.py
View file

@ -30,12 +30,16 @@ class TokenManager:
def refresh_microsoft_token(self, refresh_token: str, user_id: str, old_token: Token) -> Optional[Token]:
"""Refresh Microsoft OAuth token using refresh token"""
try:
logger.debug(f"refresh_microsoft_token: Starting Microsoft token refresh for user {user_id}")
logger.debug(f"refresh_microsoft_token: Configuration check - client_id: {bool(self.msft_client_id)}, client_secret: {bool(self.msft_client_secret)}")
if not self.msft_client_id or not self.msft_client_secret:
logger.error("Microsoft OAuth configuration not found")
return None
# Microsoft token refresh endpoint
token_url = f"https://login.microsoftonline.com/{self.msft_tenant_id}/oauth2/v2.0/token"
logger.debug(f"refresh_microsoft_token: Using token URL: {token_url}")
# Prepare refresh request
data = {
@ -45,13 +49,17 @@ class TokenManager:
"refresh_token": refresh_token,
"scope": "Mail.ReadWrite Mail.Send Mail.ReadWrite.Shared User.Read"
}
logger.debug(f"refresh_microsoft_token: Refresh request data prepared (refresh_token length: {len(refresh_token) if refresh_token else 0})")
# Make refresh request
with httpx.Client(timeout=30.0) as client:
logger.debug(f"refresh_microsoft_token: Making HTTP request to Microsoft OAuth endpoint")
response = client.post(token_url, data=data)
logger.debug(f"refresh_microsoft_token: HTTP response status: {response.status_code}")
if response.status_code == 200:
token_data = response.json()
logger.debug(f"refresh_microsoft_token: Token refresh successful, creating new token")
# Create new token
new_token = Token(
@ -65,7 +73,7 @@ class TokenManager:
createdAt=get_utc_timestamp()
)
logger.debug(f"refresh_microsoft_token: New token created with ID: {new_token.id}")
return new_token
else:
logger.error(f"Failed to refresh Microsoft token: {response.status_code} - {response.text}")
@ -126,14 +134,19 @@ class TokenManager:
def refresh_token(self, old_token: Token) -> Optional[Token]:
"""Refresh an expired token using the appropriate OAuth service"""
try:
logger.debug(f"refresh_token: Starting refresh for token {old_token.id}, authority: {old_token.authority}")
logger.debug(f"refresh_token: Token details: userId={old_token.userId}, connectionId={old_token.connectionId}, hasRefreshToken={bool(old_token.tokenRefresh)}")
if not old_token.tokenRefresh:
logger.warning(f"No refresh token available for {old_token.authority}")
return None
# Route to appropriate refresh method
if old_token.authority == AuthAuthority.MSFT:
logger.debug(f"refresh_token: Refreshing Microsoft token")
return self.refresh_microsoft_token(old_token.tokenRefresh, old_token.userId, old_token)
elif old_token.authority == AuthAuthority.GOOGLE:
logger.debug(f"refresh_token: Refreshing Google token")
return self.refresh_google_token(old_token.tokenRefresh, old_token.userId, old_token)
else:
logger.warning(f"Unknown authority for token refresh: {old_token.authority}")

3
notes/changelog.txt
View file

@ -66,6 +66,8 @@ Can you analyse again with my inputs and present a revised plan.
- add a prompt --> then shall be visible in the workflow to select
- msft connection bei 2 verschiedene users
- chat 3x ausführen mit verschiedenen mailempfängern, test ob round greift
- manual task retry - triggered
- check method outlook: alles
- check method sharepoint: alles
- check method webcrawler: alles
@ -75,7 +77,6 @@ Can you analyse again with my inputs and present a revised plan.
********************

70
test_documentExtraction.py
View file

@ -63,52 +63,52 @@ def check_dependencies():
# Check for required dependencies
try:
import bs4
logger.info("✓ beautifulsoup4 is available")
logger.info("beautifulsoup4 is available")
except ImportError:
missing_deps.append("beautifulsoup4")
logger.error("✗ beautifulsoup4 is missing")
logger.error("beautifulsoup4 is missing")
try:
import PyPDF2
logger.info("✓ PyPDF2 is available")
logger.info("PyPDF2 is available")
except ImportError:
missing_deps.append("PyPDF2")
logger.error("✗ PyPDF2 is missing")
logger.error("PyPDF2 is missing")
try:
import fitz
logger.info("✓ PyMuPDF (fitz) is available")
logger.info("PyMuPDF (fitz) is available")
except ImportError:
missing_deps.append("PyMuPDF")
logger.error("✗ PyMuPDF (fitz) is missing")
logger.error("PyMuPDF (fitz) is missing")
try:
import docx
logger.info("✓ python-docx is available")
logger.info("python-docx is available")
except ImportError:
missing_deps.append("python-docx")
logger.error("✗ python-docx is missing")
logger.error("python-docx is missing")
try:
import openpyxl
logger.info("✓ openpyxl is available")
logger.info("openpyxl is available")
except ImportError:
missing_deps.append("openpyxl")
logger.error("✗ openpyxl is missing")
logger.error("openpyxl is missing")
try:
import pptx
logger.info("✓ python-pptx is available")
logger.info("python-pptx is available")
except ImportError:
missing_deps.append("python-pptx")
logger.error("✗ python-pptx is missing")
logger.error("python-pptx is missing")
try:
from PIL import Image
logger.info("✓ Pillow (PIL) is available")
logger.info("Pillow (PIL) is available")
except ImportError:
missing_deps.append("Pillow")
logger.error("✗ Pillow (PIL) is missing")
logger.error("Pillow (PIL) is missing")
if missing_deps:
logger.error("\n" + "="*60)
@ -132,7 +132,7 @@ def check_dependencies():
logger.error("="*60)
return False
logger.info("✓ All required dependencies are available!")
logger.info("All required dependencies are available!")
return True
def check_module_imports():
@ -146,14 +146,14 @@ def check_module_imports():
from modules.interfaces.interfaceAppModel import User, UserConnection
from modules.interfaces.interfaceChatModel import ChatWorkflow, TaskItem
logger.info("✓ All required modules imported successfully")
logger.info("All required modules imported successfully")
return True
except ImportError as e:
logger.error(f"✗ Failed to import required modules: {e}")
logger.error(f"Failed to import required modules: {e}")
logger.error("Make sure you're running this script from the gateway directory")
return False
except Exception as e:
logger.error(f"✗ Unexpected error importing modules: {e}")
logger.error(f"Unexpected error importing modules: {e}")
return False
def create_mock_service_center():
@ -195,11 +195,11 @@ def create_mock_service_center():
# Create service center
service_center = ServiceCenter(mock_user, mock_workflow)
logger.info("✓ ServiceCenter created successfully with proper objects")
logger.info("ServiceCenter created successfully with proper objects")
return service_center
except Exception as e:
logger.error(f"✗ Failed to create ServiceCenter: {e}")
logger.error(f"Failed to create ServiceCenter: {e}")
return None
class DocumentExtractionTester:
@ -243,10 +243,10 @@ class DocumentExtractionTester:
# Verify directory was created
if self.output_dir.exists():
logger.info(f"✓ Output directory created/verified: {self.output_dir}")
logger.info(f"Output directory created/verified: {self.output_dir}")
logger.info(f"Output directory absolute path: {self.output_dir.absolute()}")
else:
logger.error(f"✗ Failed to create output directory: {self.output_dir}")
logger.error(f"Failed to create output directory: {self.output_dir}")
# Log configuration
logger.info(f"Configuration: AI processing = {'ENABLED' if self.enable_ai else 'DISABLED'}")
@ -264,20 +264,20 @@ class DocumentExtractionTester:
if test_file.exists():
actual_size = test_file.stat().st_size
logger.info(f"✓ Basic file writing test passed: {test_file} (size: {actual_size} bytes)")
logger.info(f"Basic file writing test passed: {test_file} (size: {actual_size} bytes)")
# Test reading the file back
with open(test_file, 'r', encoding='utf-8') as f:
content = f.read()
logger.info(f"✓ File read test passed: content length = {len(content)}")
logger.info(f"File read test passed: content length = {len(content)}")
# Clean up test file
test_file.unlink()
logger.info("✓ Test file cleaned up")
logger.info("Test file cleaned up")
else:
logger.error(f"✗ Basic file writing test failed: {test_file}")
logger.error(f"Basic file writing test failed: {test_file}")
except Exception as e:
logger.error(f"✗ Basic file writing test failed with error: {e}")
logger.error(f"Basic file writing test failed with error: {e}")
import traceback
traceback.print_exc()
@ -329,10 +329,10 @@ class DocumentExtractionTester:
# Now create DocumentExtraction with the service center
from modules.chat.documents.documentExtraction import DocumentExtraction
self.extractor = DocumentExtraction(self.service_center)
logger.info("✓ DocumentExtraction initialized successfully with ServiceCenter")
logger.info("DocumentExtraction initialized successfully with ServiceCenter")
return True
except Exception as e:
logger.error(f"✗ Failed to initialize DocumentExtraction: {e}")
logger.error(f"Failed to initialize DocumentExtraction: {e}")
return False
def get_files_to_process(self) -> List[Path]:
@ -457,14 +457,14 @@ class DocumentExtractionTester:
# Verify file was created
if output_file.exists():
actual_size = output_file.stat().st_size
logger.info(f"✓ File created successfully: {output_fileName} (expected: {content_size} bytes, actual: {actual_size} bytes)")
logger.info(f"File created successfully: {output_fileName} (expected: {content_size} bytes, actual: {actual_size} bytes)")
else:
logger.error(f"✗ File was not created: {output_file}")
logger.error(f"File was not created: {output_file}")
result['output_files'].append(output_fileName)
result['content_items'] += 1
except Exception as write_error:
logger.error(f"✗ Error writing file {output_fileName}: {write_error}")
logger.error(f"Error writing file {output_fileName}: {write_error}")
import traceback
traceback.print_exc()
else:
@ -665,13 +665,13 @@ class DocumentExtractionTester:
try:
if await self.process_single_file(file_path):
successful += 1
logger.info(f"✓ File {i+1} processed successfully")
logger.info(f"File {i+1} processed successfully")
else:
failed += 1
logger.error(f"✗ File {i+1} processing failed")
logger.error(f"File {i+1} processing failed")
except Exception as e:
failed += 1
logger.error(f"✗ Exception processing file {i+1}: {e}")
logger.error(f"Exception processing file {i+1}: {e}")
import traceback
traceback.print_exc()

10
test_excel_processing.py
View file

@ -61,11 +61,11 @@ async def test_excel_processing():
)
service_center = ServiceCenter(mock_user, mock_workflow)
logger.info("✓ ServiceCenter created successfully")
logger.info("ServiceCenter created successfully")
# Create DocumentExtraction instance
extractor = DocumentExtraction(service_center)
logger.info("✓ DocumentExtraction created successfully")
logger.info("DocumentExtraction created successfully")
# Test with a sample Excel file if available
test_file_path = "d:/temp/test-extraction/test.xlsx"
@ -90,7 +90,7 @@ async def test_excel_processing():
enableAI=False
)
logger.info(f"✓ Excel processing completed successfully!")
logger.info(f"Excel processing completed successfully!")
logger.info(f"Generated {len(result.contents)} content items:")
for i, content_item in enumerate(result.contents):
@ -131,7 +131,7 @@ async def test_excel_processing():
wb.properties.creator = "Test User"
wb.properties.subject = "Test Subject"
logger.info("✓ Test workbook created successfully")
logger.info("Test workbook created successfully")
logger.info(f" Title: {wb.properties.title}")
logger.info(f" Creator: {wb.properties.creator}")
logger.info(f" Subject: {wb.properties.subject}")
@ -158,7 +158,7 @@ async def test_excel_processing():
enableAI=False
)
logger.info(f"✓ Test workbook processing completed successfully!")
logger.info(f"Test workbook processing completed successfully!")
logger.info(f"Generated {len(result.contents)} content items:")
for i, content_item in enumerate(result.contents):