1229 lines
No EOL
49 KiB
Python
1229 lines
No EOL
49 KiB
Python
"""
|
|
ChatManager Module for managing AI-Chat workflows.
|
|
Implements a compact and modular architecture for processing
|
|
user requests, agent execution, and result formatting.
|
|
"""
|
|
|
|
import os
|
|
import logging
|
|
import json
|
|
import re
|
|
import uuid
|
|
import base64
|
|
from datetime import datetime
|
|
from typing import Dict, Any, List, Optional, Union
|
|
|
|
# Required imports
|
|
from connectors.connector_aichat_openai import ChatService
|
|
from modules.chat_registry import get_agent_registry
|
|
from modules.lucydom_interface import get_lucydom_interface, GLOBAL_SETTINGS
|
|
from modules.chat_content_extraction import get_document_contents
|
|
|
|
# Configure logger
|
|
logger = logging.getLogger(__name__)
|
|
|
|
class ChatManager:
|
|
"""
|
|
Manages the processing of chat requests, agent execution, and
|
|
the integration of results into the workflow.
|
|
"""
|
|
|
|
def __init__(self, mandate_id: int, user_id: int):
|
|
"""
|
|
Initializes the ChatManager with mandate and user context.
|
|
|
|
Args:
|
|
mandate_id: ID of the current mandate
|
|
user_id: ID of the current user
|
|
"""
|
|
self.mandate_id = mandate_id
|
|
self.user_id = user_id
|
|
self.ai_service = ChatService()
|
|
self.lucy_interface = get_lucydom_interface(mandate_id, user_id)
|
|
self.agent_registry = get_agent_registry()
|
|
self.agent_registry.set_ai_service(self.ai_service)
|
|
|
|
# Set AI service in lucy interface for language support
|
|
self.lucy_interface.set_ai_service(self.ai_service)
|
|
|
|
### Chat Management
|
|
|
|
async def chat_run(self, user_input: Dict[str, Any], workflow_id: Optional[str] = None) -> Dict[str, Any]:
|
|
"""
|
|
Main function for integrating user requests into the workflow.
|
|
|
|
Args:
|
|
user_input: Dictionary with user request and file IDs
|
|
workflow_id: Optional - ID of the workflow (None for new workflows)
|
|
|
|
Returns:
|
|
Workflow object with updated state
|
|
"""
|
|
# 1. Initialize workflow or load existing one
|
|
workflow = self.workflow_init(workflow_id)
|
|
self.log_add(workflow, "Starting workflow processing", level="info", progress=0)
|
|
|
|
# 2. Transform user input into a message object and save in workflow
|
|
message_user = await self.chat_message_to_workflow("user", "", user_input, workflow)
|
|
|
|
# 3. Create project manager prompt and analyze response
|
|
self.log_add(workflow, "Analyzing request and planning work", level="info", progress=10)
|
|
project_manager_response = await self.chat_prompt(message_user, workflow)
|
|
obj_final_documents = project_manager_response.get("obj_final_documents", [])
|
|
obj_workplan = project_manager_response.get("obj_workplan", [])
|
|
obj_user_response = project_manager_response.get("obj_user_response", "")
|
|
|
|
# Get detected language and set it in the lucy interface
|
|
user_language = project_manager_response.get("user_language", "en")
|
|
self.lucy_interface.set_user_language(user_language)
|
|
|
|
# 4. Save the response as a message in the workflow and add log entries
|
|
response_message = {
|
|
"role": "assistant",
|
|
"agent_name": "project_manager",
|
|
"content": obj_user_response
|
|
}
|
|
self.message_add(workflow, response_message)
|
|
|
|
self.log_add(workflow, f"Planned outputs: {len(obj_final_documents)} documents", level="info", progress=20)
|
|
self.log_add(workflow, f"Work plan created with {len(obj_workplan)} steps", level="info", progress=25)
|
|
|
|
# 5. Execute agents according to work plan
|
|
obj_results = []
|
|
if obj_workplan:
|
|
total_tasks = len(obj_workplan)
|
|
for task_index, task in enumerate(obj_workplan):
|
|
agent_name = task.get("agent", "unknown")
|
|
progress_value = 30 + int((task_index / total_tasks) * 60) # Progress from 30% to 90%
|
|
|
|
progress_msg = f"Running task {task_index+1}/{total_tasks}: {agent_name}"
|
|
self.log_add(workflow, progress_msg, level="info", progress=progress_value)
|
|
|
|
task_results = await self.agent_processing(task, workflow)
|
|
obj_results.extend(task_results)
|
|
|
|
# Log completion of this task
|
|
self.log_add(
|
|
workflow,
|
|
f"Completed task {task_index+1}/{total_tasks}: {agent_name}",
|
|
level="info",
|
|
progress=progress_value + (60/total_tasks)/2
|
|
)
|
|
|
|
# 6. Create the final response with relevant documents from obj_final_documents
|
|
self.log_add(workflow, "Creating final response", level="info", progress=90)
|
|
final_message = await self.chat_final_message(obj_user_response, obj_final_documents, obj_results)
|
|
self.message_add(workflow, final_message)
|
|
|
|
# 7. Finalize the workflow
|
|
self.workflow_finish(workflow)
|
|
self.log_add(workflow, "Workflow completed successfully", level="info", progress=100)
|
|
|
|
return workflow
|
|
|
|
async def chat_prompt(self, message_user: Dict[str, Any], workflow: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""
|
|
Creates the prompt for the project manager and processes the response.
|
|
|
|
Args:
|
|
message_user: Message object with user request
|
|
workflow: Current workflow object
|
|
|
|
Returns:
|
|
Project manager's response with obj_final_documents, obj_workplan and obj_user_response
|
|
"""
|
|
# Get available agents with their capabilities
|
|
available_agents = self.agent_profiles()
|
|
|
|
# Create a workflow summary
|
|
workflow_summary = await self.workflow_summarize(workflow, message_user)
|
|
|
|
# Create a list of currently available documents from user input or previously generated documents
|
|
available_documents = self.available_documents_get(workflow, message_user)
|
|
available_docs_str = json.dumps(available_documents, indent=2)
|
|
|
|
# Create the prompt for the project manager with language detection requirement
|
|
prompt = f"""
|
|
Based on the user request and the provided documents, please analyze the requirements and create a processing plan.
|
|
Also, identify the language of the user's request and include it in your response.
|
|
|
|
<userrequest>
|
|
{message_user.get('content')}
|
|
</userrequest>
|
|
|
|
# Previous conversation history:
|
|
|
|
{workflow_summary}
|
|
|
|
|
|
# Available documents (currently in workflow):
|
|
|
|
{available_docs_str}
|
|
|
|
|
|
# Available agents and their capabilities:
|
|
|
|
{self.parse_json2text(available_agents)}
|
|
|
|
|
|
Please analyze the request and create:
|
|
|
|
1. A list of required result documents (obj_final_documents)
|
|
2. A plan for executing agents (obj_workplan)
|
|
3. A clear response to the user explaining what you're doing (obj_user_response)
|
|
4. Identified language of the user's request (user_language)
|
|
|
|
## IMPORTANT RULES FOR THE WORKPLAN:
|
|
1. Each input document must either already exist (provided by the user or previously created by an agent) or be created by an agent before it's used.
|
|
2. If necessary, convert input documents to a suitable format using agents when the type doesn't match.
|
|
3. Do not define document inputs that don't exist or haven't been generated beforehand.
|
|
4. Create a logical sequence - earlier agents can create documents that are later used as inputs.
|
|
5. If the user has provided documents but hasn't clearly stated what they want, try to act according to the context.
|
|
|
|
Your answer must be strictly in the JSON_OUTPUT format, with no additions before or after the JSON object.
|
|
|
|
JSON_OUTPUT = {{
|
|
"obj_final_documents": [
|
|
FILEREF
|
|
],
|
|
"obj_workplan": [
|
|
{{
|
|
"agent": "agent_name", # Name of an available agent
|
|
"prompt": "Specific instructions to the agent, that he knows what to do with which documents and which output to provide."
|
|
"output_documents": [
|
|
"label":"document label in the format 'filename.ext'",
|
|
"prompt":"AI prompt to describe the content of the file"
|
|
],
|
|
"input_documents": [
|
|
"label":"document label in the format 'filename.ext'",
|
|
"file_id":id, # if refering to an existing document, provide file_id to select the correct file
|
|
"content_part":"", # provide empty string, if all document contents to consider, otherwise the content_part of the document to focus on
|
|
"prompt":"AI prompt to describe what data to extract from the file."
|
|
], # If no input documents are needed, include "input_documents" as an empty list
|
|
}}
|
|
# Multiple agent tasks can be added here and should build logically on each other
|
|
],
|
|
"obj_user_response": "Information to the user about how his request will be solved.",
|
|
"user_language": "en" # Language code (e.g., en, de, fr, es) based on the user's request
|
|
}}
|
|
|
|
## RULES for input_documents:
|
|
1. The user request refers to documents where "file_source" in available documents is "user". Those documents are in the focus for input
|
|
2. In case of redundant label in available documents, use document with highest sequence_nr if not specified differently
|
|
|
|
## STRICT RULES FOR document "label":
|
|
1. Every document label MUST include a proper file extension that matches the content type.
|
|
2. Use standard extensions like:
|
|
- ".txt" for text files
|
|
- ".md" for markdown files
|
|
- ".csv" for comma-separated values
|
|
- ".json" for JSON data
|
|
- ".html" for HTML content
|
|
- ".jpg" or ".png" for images
|
|
- ".docx" for Word documents
|
|
- ".xlsx" for Excel files
|
|
- ".pdf" for PDF documents
|
|
3. Use descriptive filenames that indicate the document's purpose (e.g., "analysis_report.txt" rather than just "report.txt")
|
|
4. If you use label for an existing file
|
|
"""
|
|
|
|
# Call the AI service through lucy_interface for language support
|
|
logger.debug(f"Planning prompt: {prompt}")
|
|
project_manager_output = await self.lucy_interface.call_ai([
|
|
{
|
|
"role": "system",
|
|
"content": "You are an experienced project manager who analyzes user requests and creates work plans. You pay very careful attention to ensure that all document dependencies are correct and that no non-existent documents are defined as inputs. The output follows strictly the specified format."
|
|
},
|
|
{
|
|
"role": "user",
|
|
"content": prompt
|
|
}
|
|
])
|
|
|
|
# Parse the JSON response
|
|
return self.parse_json_response(project_manager_output)
|
|
|
|
async def chat_message_to_workflow(self, role: str, agent_name: str, chat_message: Dict[str, Any], workflow: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""
|
|
Integrates user inputs into a Message object including files with complete contents.
|
|
|
|
Args:
|
|
role: Role of the message sender ('user' or 'assistant')
|
|
agent_name: Name of the agent, if message is from an agent
|
|
chat_message: Input data with "prompt"=str, "list_file_id"=[]
|
|
workflow: Current workflow object
|
|
|
|
Returns:
|
|
Message object with content and documents including contents
|
|
"""
|
|
logger.info(f"Message from {role} {agent_name} sent with {len(chat_message.get('list_file_id', []))} documents")
|
|
logger.debug(f"message = {self.parse_json2text(chat_message)}.")
|
|
|
|
# Check message content
|
|
message_content = chat_message.get("prompt", "")
|
|
if isinstance(message_content, dict) and "content" in message_content:
|
|
message_content = message_content["content"]
|
|
|
|
# If message content is empty, no chat
|
|
if role=="user" and (message_content is None or message_content.strip() == ""):
|
|
logger.warning(f"Empty message, no chat")
|
|
message_content = "(No user input received)"
|
|
|
|
# Process additional files with complete contents
|
|
additional_fileids = chat_message.get("list_file_id", [])
|
|
additional_files = await self.process_file_ids(additional_fileids)
|
|
|
|
# Create message object
|
|
message_object = {
|
|
"role": role,
|
|
"agent_name": agent_name,
|
|
"content": message_content,
|
|
"documents": additional_files
|
|
}
|
|
|
|
message_object = self.message_add(workflow, message_object)
|
|
logger.debug(f"message_user = {self.parse_json2text(message_object)}.")
|
|
return message_object
|
|
|
|
async def chat_final_message(self, obj_user_response: str, obj_final_documents: List[Dict[str, Any]], obj_results: List[Dict[str, Any]]) -> Dict[str, Any]:
|
|
"""
|
|
Creates the final response message with review of proposed and delivered.
|
|
|
|
Args:
|
|
obj_user_response: Initial text response to the user
|
|
obj_final_documents: List of expected response documents
|
|
obj_results: List of generated result documents
|
|
|
|
Returns:
|
|
Complete message object with content and relevant documents
|
|
"""
|
|
# Find documents that match the obj_final_documents requirements
|
|
matching_documents = []
|
|
|
|
for answer_spec in obj_final_documents:
|
|
answer_label = answer_spec.get("label")
|
|
|
|
# Find matching document in results
|
|
for doc in obj_results:
|
|
doc_name = self.get_filename(doc)
|
|
# Check if this document matches the answer specification
|
|
if doc_name == answer_label:
|
|
content_ref = []
|
|
for c in doc.get("contents"):
|
|
content_ref.append(c.get("summary"))
|
|
doc_ref = {
|
|
"label": doc_name,
|
|
"content_summary": content_ref
|
|
}
|
|
matching_documents.append(doc_ref)
|
|
break
|
|
|
|
# Use the lucy_interface for language-aware AI calls
|
|
final_prompt = await self.lucy_interface.call_ai([
|
|
{"role": "system", "content": "You are a project manager, who delivers results to a user."},
|
|
{"role": "user", "content": f"""
|
|
Give the final short feedback to the user with reference to the initial statement (obj_user_response). Provide a list of delivered files (files_delivered). If in the list of delivered files (files_delivered) some files from the original list (files_promised) are not available, then just give a comment on this, otherwise task is completed.
|
|
|
|
Here the data:
|
|
obj_user_response = {self.parse_json2text(obj_user_response)}
|
|
files_promised = {self.parse_json2text(matching_documents)}
|
|
files_delivered = {self.parse_json2text(obj_user_response)}
|
|
"""
|
|
}
|
|
], produce_user_answer=True)
|
|
|
|
# Create basic message structure with proper fields
|
|
logger.debug(f"FINAL PROMPT = {self.parse_json2text(final_prompt)}.")
|
|
final_message = {
|
|
"role": "assistant",
|
|
"agent_name": "project_manager",
|
|
"content": final_prompt,
|
|
"documents": [] # DO NOT include the results documents, already with agents
|
|
}
|
|
|
|
logger.debug(f"FINAL MESSAGE = {self.parse_json2text(final_message)}.")
|
|
return final_message
|
|
|
|
|
|
### Workflow
|
|
|
|
def workflow_init(self, workflow_id: Optional[str] = None) -> Dict[str, Any]:
|
|
"""
|
|
Initializes a workflow or loads an existing one with round counting.
|
|
|
|
Args:
|
|
workflow_id: Optional - ID of the workflow to load
|
|
|
|
Returns:
|
|
Initialized workflow object
|
|
"""
|
|
current_time = datetime.now().isoformat()
|
|
|
|
if workflow_id is None or not self.lucy_interface.get_workflow(workflow_id):
|
|
# Create new workflow
|
|
new_workflow_id = str(uuid.uuid4()) if workflow_id is None else workflow_id
|
|
workflow = {
|
|
"id": new_workflow_id,
|
|
"mandate_id": self.mandate_id,
|
|
"user_id": self.user_id,
|
|
"name": f"Workflow {new_workflow_id[:8]}",
|
|
"started_at": current_time,
|
|
"messages": [], # Empty list - will be filled with references
|
|
"message_ids": [], # Initialize empty message_ids list
|
|
"logs": [],
|
|
"data_stats": {},
|
|
"current_round": 1,
|
|
"status": "running",
|
|
"last_activity": current_time,
|
|
}
|
|
|
|
# Save to database - only the workflow metadata
|
|
workflow_db = {
|
|
"id": workflow["id"],
|
|
"mandate_id": workflow["mandate_id"],
|
|
"user_id": workflow["user_id"],
|
|
"name": workflow["name"],
|
|
"started_at": workflow["started_at"],
|
|
"status": workflow["status"],
|
|
"data_stats": workflow["data_stats"],
|
|
"current_round": workflow["current_round"],
|
|
"last_activity": workflow["last_activity"],
|
|
"message_ids": workflow["message_ids"] # Include message_ids
|
|
}
|
|
self.lucy_interface.create_workflow(workflow_db)
|
|
|
|
self.log_add(workflow, GLOBAL_SETTINGS["workflow_status_messages"]["init"], level="info", progress=0)
|
|
return workflow
|
|
else:
|
|
# Load existing workflow
|
|
workflow = self.lucy_interface.load_workflow_state(workflow_id)
|
|
|
|
# Ensure message_ids exists
|
|
if "message_ids" not in workflow:
|
|
# Initialize from existing messages
|
|
workflow["message_ids"] = [msg["id"] for msg in workflow.get("messages", [])]
|
|
|
|
# Update in database
|
|
self.lucy_interface.update_workflow(workflow_id, {"message_ids": workflow["message_ids"]})
|
|
|
|
# Update status and increment round counter
|
|
workflow["status"] = "running"
|
|
workflow["last_activity"] = current_time
|
|
|
|
# Increment current_round if it exists, otherwise set it to 1
|
|
if "current_round" in workflow:
|
|
workflow["current_round"] += 1
|
|
else:
|
|
workflow["current_round"] = 1
|
|
|
|
# Update in database - only the relevant workflow fields
|
|
workflow_update = {
|
|
"status": workflow["status"],
|
|
"last_activity": workflow["last_activity"],
|
|
"current_round": workflow["current_round"]
|
|
}
|
|
self.lucy_interface.update_workflow(workflow_id, workflow_update)
|
|
|
|
self.log_add(workflow, GLOBAL_SETTINGS["workflow_status_messages"]["running"], level="info", progress=0)
|
|
return workflow
|
|
|
|
def workflow_finish(self, workflow: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""
|
|
Finalizes a workflow and sets the status to 'completed'.
|
|
|
|
Args:
|
|
workflow: Workflow object
|
|
|
|
Returns:
|
|
Updated workflow object
|
|
"""
|
|
# Prepare workflow update data
|
|
workflow_update = {
|
|
"status": "completed",
|
|
"last_activity": datetime.now().isoformat(),
|
|
}
|
|
|
|
# Update the workflow object in memory
|
|
workflow["status"] = workflow_update["status"]
|
|
workflow["last_activity"] = workflow_update["last_activity"]
|
|
|
|
# Save workflow state to database - only relevant fields, not the messages list
|
|
self.lucy_interface.update_workflow(workflow["id"], workflow_update)
|
|
|
|
self.log_add(workflow, GLOBAL_SETTINGS["workflow_status_messages"]["completed"], level="info", progress=100)
|
|
return workflow
|
|
|
|
async def workflow_summarize(self, workflow: Dict[str, Any], message_user: Dict[str, Any]) -> str:
|
|
"""
|
|
Creates a summary of the workflow without the current user message.
|
|
|
|
Args:
|
|
workflow: Workflow object
|
|
message_user: Current user message
|
|
|
|
Returns:
|
|
Summary of the workflow
|
|
"""
|
|
if not workflow or "messages" not in workflow or not workflow["messages"]:
|
|
return "" # first message
|
|
|
|
# Go through messages in reverse order (newest first)
|
|
messages = sorted(workflow["messages"], key=lambda m: m.get("sequence_no", 0), reverse=False)
|
|
|
|
summary_parts = []
|
|
for message in messages:
|
|
if True: # including user message, excluding would be: message["id"] != message_user["id"]:
|
|
message_summary = await self.message_summarize(message)
|
|
summary_parts.append(message_summary)
|
|
|
|
return "\n\n".join(summary_parts)
|
|
|
|
|
|
|
|
### Agents
|
|
|
|
def agent_profiles(self) -> List[Dict[str, Any]]:
|
|
"""
|
|
Gets information about all available agents.
|
|
|
|
Returns:
|
|
List with information about all available agents
|
|
"""
|
|
return self.agent_registry.get_agent_infos()
|
|
|
|
async def agent_input_documents(self, doc_input_list: List[Dict[str, Any]], workflow: Dict[str, Any]) -> List[Dict[str, Any]]:
|
|
"""
|
|
Prepares input documents for an agent, sorted with newest first.
|
|
|
|
Args:
|
|
doc_input_list: List of required input documents as specified by the project manager
|
|
workflow: Workflow object
|
|
|
|
Returns:
|
|
Prepared input documents for the agent, sorted with newest first
|
|
"""
|
|
prepared_inputs = []
|
|
|
|
# Sort workflow messages by sequence number (descending)
|
|
sorted_messages = sorted(
|
|
workflow.get("messages", []),
|
|
key=lambda m: m.get("sequence_no", 0),
|
|
reverse=True
|
|
)
|
|
|
|
for doc_spec in doc_input_list:
|
|
doc_filename = doc_spec.get("label","")
|
|
doc_file_id = doc_spec.get("file_id","")
|
|
|
|
found_doc = None
|
|
# Search for the document in sorted workflow messages (newest first)
|
|
for message in sorted_messages:
|
|
for doc in message.get("documents", []):
|
|
if (doc_file_id!="" and doc_file_id==doc.get("file_id")) or (doc_filename!="" and self.get_filename(doc) == doc_filename):
|
|
found_doc = doc
|
|
break
|
|
if found_doc:
|
|
break
|
|
if found_doc:
|
|
# Process document for agent based on the specification
|
|
processed_doc = await self.process_document_for_agent(found_doc, doc_spec)
|
|
|
|
prepared_inputs.append(processed_doc)
|
|
else:
|
|
logger.warning(f"Document with label '{doc_filename}', file_id '{doc_file_id}' not found in workflow")
|
|
|
|
return prepared_inputs
|
|
|
|
async def process_document_for_agent(self, document: Dict[str, Any], doc_spec: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""
|
|
Processes a document for an agent based on the document specification.
|
|
Uses AI to extract relevant content from the document based on the specification.
|
|
|
|
Args:
|
|
document: The document to process
|
|
doc_spec: The document specification from the project manager
|
|
|
|
Returns:
|
|
Processed document with AI-extracted content
|
|
"""
|
|
processed_doc = document.copy()
|
|
part_spec = doc_spec.get("content_part","")
|
|
|
|
# Process each content item in the document
|
|
if "contents" in processed_doc:
|
|
processed_contents = []
|
|
|
|
for content in processed_doc["contents"]:
|
|
|
|
# Check if part required
|
|
if part_spec != "" and part_spec != content.get("name"):
|
|
continue
|
|
|
|
# Get the data from the content
|
|
data = content.get("data", "")
|
|
processed_content = content.copy()
|
|
|
|
# Check if content data is base64 encoded
|
|
is_base64 = content.get("metadata", {}).get("base64_encoded", False)
|
|
|
|
try:
|
|
# Use the AI service to process the document content according to the prompt from the project manager for the document specification
|
|
summary = doc_spec.get("prompt", "Extract the relevant information from this document")
|
|
ai_prompt = f"""
|
|
# Please process the following document content according to this instruction:
|
|
<instruction>
|
|
{summary}
|
|
</instruction>
|
|
|
|
# Document content:
|
|
<data>
|
|
{data}
|
|
</data>
|
|
|
|
# Extract and provide only the relevant information as requested.
|
|
"""
|
|
|
|
# Call the AI service through lucy_interface for language support
|
|
processed_data = await self.lucy_interface.call_ai([
|
|
{"role": "system", "content": "You are a document processing assistant. Extract only the relevant information as requested."},
|
|
{"role": "user", "content": ai_prompt}
|
|
])
|
|
|
|
# DO NOT change the original data field
|
|
# processed_content["data"] unchanged
|
|
processed_content["data_extracted"] = processed_data
|
|
processed_content["metadata"]["ai_processed"] = True
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error processing document content with AI: {str(e)}")
|
|
# Fall back to original content if AI processing fails
|
|
processed_content["data_extracted"] = "(no information)"
|
|
|
|
processed_contents.append(processed_content)
|
|
|
|
processed_doc["contents"] = processed_contents
|
|
|
|
return processed_doc
|
|
|
|
async def agent_processing(self, task: Dict[str, Any], workflow: Dict[str, Any]) -> List[Dict[str, Any]]:
|
|
"""
|
|
Process a single agent task from the workflow.
|
|
Optimized for the task-based approach where all agents implement process_task.
|
|
|
|
Args:
|
|
task: The task definition containing agent name, prompt, and document specifications
|
|
workflow: The current workflow object
|
|
|
|
Returns:
|
|
List of document objects created by the agent
|
|
"""
|
|
# 1. Extract task information
|
|
agent_name = task.get("agent")
|
|
agent_prompt = task.get("prompt", "")
|
|
|
|
# Log the current step
|
|
output_labels = [d.get("label", "unknown") for d in task.get("output_documents", [])]
|
|
step_info = f"Agent '{agent_name}' to create {', '.join(output_labels)}."
|
|
self.log_add(workflow, step_info, level="info")
|
|
|
|
# Check if prompt is empty
|
|
if agent_prompt == "":
|
|
logger.warning("Empty prompt, no task to do")
|
|
return []
|
|
|
|
# Get agent from registry
|
|
agent = self.agent_registry.get_agent(agent_name)
|
|
if not agent:
|
|
logger.error(f"Agent '{agent_name}' not found")
|
|
return []
|
|
|
|
# Prepare output document specifications
|
|
output_specs = []
|
|
for doc in task.get("output_documents", []):
|
|
output_spec = {
|
|
"label": doc.get("label"),
|
|
"description": doc.get("prompt", "")
|
|
}
|
|
output_specs.append(output_spec)
|
|
|
|
# Prepare input documents for the agent
|
|
input_documents = await self.agent_input_documents(task.get('input_documents', []), workflow)
|
|
|
|
# Create a standardized task object for the agent
|
|
agent_task = {
|
|
"task_id": str(uuid.uuid4()),
|
|
"workflow_id": workflow.get("id"),
|
|
"prompt": agent_prompt,
|
|
"input_documents": input_documents,
|
|
"output_specifications": output_specs,
|
|
"context": {
|
|
"workflow_round": workflow.get("current_round", 1),
|
|
"agent_type": agent_name,
|
|
"timestamp": datetime.now().isoformat(),
|
|
"language": self.lucy_interface.user_language # Pass language to agent
|
|
}
|
|
}
|
|
|
|
# Execute the agent with the standardized task
|
|
try:
|
|
# Process the task using the agent's standardized interface
|
|
logger.debug("TASK: "+self.parse_json2text(agent_task))
|
|
logger.debug(f"Agent '{agent_name}' AI service available: {agent.ai_service is not None}")
|
|
|
|
agent_results = await agent.process_task(agent_task)
|
|
|
|
logger.debug(f"Agent '{agent_name}' completed task. RESULT: {self.parse_json2text(agent_results)}")
|
|
|
|
# Log the agent response
|
|
self.log_add(
|
|
workflow,
|
|
f"Agent '{agent_name}' completed task. Feedback: {agent_results.get('feedback', 'No feedback provided')}",
|
|
level="info"
|
|
)
|
|
|
|
# Store produced files and prepare input object for message
|
|
agent_inputs = {
|
|
"prompt": agent_results.get("feedback", ""),
|
|
"list_file_id": self.agent_save_documents(agent_results)
|
|
}
|
|
|
|
# Create a message in the workflow with the agent's response
|
|
agent_message = await self.chat_message_to_workflow("assistant", agent_name, agent_inputs, workflow)
|
|
logger.debug(f"Agent result = {self.parse_json2text(agent_message)}.")
|
|
|
|
return agent_message.get("documents", [])
|
|
|
|
except Exception as e:
|
|
error_msg = f"Error executing agent '{agent_name}': {str(e)}"
|
|
logger.error(error_msg, exc_info=True) # Add exc_info=True to get full traceback
|
|
self.log_add(workflow, error_msg, level="error")
|
|
return []
|
|
|
|
def agent_save_documents(self, agent_results: Dict[str, Any]) -> List[int]:
|
|
"""
|
|
Saves all documents from agent results as files and returns a list of file IDs.
|
|
Enhanced to handle the standardized document format from agents.
|
|
|
|
Args:
|
|
agent_results: Dictionary containing agent feedback and documents
|
|
|
|
Returns:
|
|
List of file IDs for the saved documents
|
|
"""
|
|
file_ids = []
|
|
|
|
# Extract documents from agent results
|
|
documents = agent_results.get("documents", [])
|
|
|
|
for doc in documents:
|
|
try:
|
|
# Extract label (filename) and content
|
|
label = doc.get("label", "unnamed_file.txt")
|
|
content = doc.get("content", "")
|
|
|
|
# Split label into name and extension
|
|
name, ext = os.path.splitext(label)
|
|
if ext.startswith('.'):
|
|
ext = ext[1:] # Remove leading dot
|
|
elif not ext:
|
|
# If no extension is provided, default to .txt for text content
|
|
ext = "txt"
|
|
label = f"{label}.{ext}"
|
|
|
|
# Determine if content is base64 encoded
|
|
is_base64 = False
|
|
if isinstance(content, dict) and content.get("metadata", {}).get("base64_encoded", False):
|
|
is_base64 = True
|
|
content = content.get("data", "")
|
|
|
|
# Convert content to bytes
|
|
if isinstance(content, str):
|
|
if is_base64:
|
|
# Decode base64 to bytes
|
|
try:
|
|
file_content = base64.b64decode(content)
|
|
except Exception as e:
|
|
logger.warning(f"Failed to decode base64 content: {str(e)}")
|
|
file_content = content.encode('utf-8')
|
|
else:
|
|
# Convert text to bytes
|
|
file_content = content.encode('utf-8')
|
|
else:
|
|
# Already bytes
|
|
file_content = content
|
|
|
|
# Save file to database
|
|
file_meta = self.lucy_interface.save_uploaded_file(file_content, label)
|
|
|
|
if file_meta and "id" in file_meta:
|
|
file_id = file_meta["id"]
|
|
file_ids.append(file_id)
|
|
logger.info(f"Saved document '{label}' with file ID: {file_id}")
|
|
else:
|
|
logger.warning(f"Failed to save document '{label}'")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error saving document from agent results: {str(e)}")
|
|
# Continue with other documents instead of failing
|
|
continue
|
|
|
|
return file_ids
|
|
|
|
|
|
### Messages
|
|
|
|
def message_add(self, workflow: Dict[str, Any], message: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""
|
|
Adds a message to the workflow and updates last_activity.
|
|
Saves the message in the database and updates the workflow with references.
|
|
|
|
Args:
|
|
workflow: Workflow object
|
|
message: Message to be saved
|
|
|
|
Returns:
|
|
Added message
|
|
"""
|
|
current_time = datetime.now().isoformat()
|
|
|
|
# Ensure messages list exists
|
|
if "messages" not in workflow:
|
|
workflow["messages"] = []
|
|
|
|
# Generate new message ID if not present
|
|
if "id" not in message:
|
|
message["id"] = f"msg_{str(uuid.uuid4())}"
|
|
|
|
# Add workflow ID and timestamps
|
|
message["workflow_id"] = workflow["id"]
|
|
message["started_at"] = current_time
|
|
message["finished_at"] = current_time
|
|
|
|
# Set sequence number
|
|
message["sequence_no"] = len(workflow["messages"]) + 1
|
|
|
|
# Ensure required fields are present
|
|
if "role" not in message:
|
|
# Set a default role based on agent_name
|
|
message["role"] = "assistant" if message.get("agent_name") else "user"
|
|
|
|
if "agent_name" not in message:
|
|
message["agent_name"] = ""
|
|
|
|
# Set status
|
|
message["status"] = "completed"
|
|
|
|
# Add message to workflow
|
|
workflow["messages"].append(message)
|
|
|
|
# Ensure message_ids list exists
|
|
if "message_ids" not in workflow:
|
|
workflow["message_ids"] = []
|
|
|
|
# Add message ID to the message_ids list
|
|
workflow["message_ids"].append(message["id"])
|
|
|
|
# Update workflow status
|
|
workflow["last_activity"] = current_time
|
|
|
|
# Save to database - first the message itself
|
|
self.lucy_interface.create_workflow_message(message)
|
|
|
|
# Then save the workflow with updated references
|
|
workflow_update = {
|
|
"last_activity": current_time,
|
|
"message_ids": workflow["message_ids"] # Update the message_ids field
|
|
}
|
|
self.lucy_interface.update_workflow(workflow["id"], workflow_update)
|
|
|
|
return message
|
|
|
|
async def message_summarize(self, message: Dict[str, Any]) -> str:
|
|
"""
|
|
Creates a summary of a message including its documents.
|
|
|
|
Args:
|
|
message: Message to summarize
|
|
|
|
Returns:
|
|
Summary of the message
|
|
"""
|
|
role = message.get("role", "undefined")
|
|
agent_name = message.get("agent_name", "")
|
|
content = message.get("content", "")
|
|
|
|
try:
|
|
# Use the lucy_interface for language-aware AI calls
|
|
content_summary = await self.lucy_interface.call_ai([
|
|
{"role": "system", "content": f"You are a chat message summarizer. Create a very concise summary (2-3 sentences, maximum 300 characters)"},
|
|
{"role": "user", "content": content}
|
|
])
|
|
except Exception as e:
|
|
logger.error(f"Error creating summary: {str(e)}")
|
|
content_summary = content[:200] + "..."
|
|
|
|
# Summarize documents
|
|
docs_summary = ""
|
|
if "documents" in message and message["documents"]:
|
|
docs_list = []
|
|
for i, doc in enumerate(message["documents"]):
|
|
doc_name = self.get_filename(doc)
|
|
docs_list.append(doc_name)
|
|
if docs_list:
|
|
docs_summary = "\nDocuments:" + "\n- ".join(docs_list)
|
|
|
|
return f"[{role} {agent_name}]: {content_summary}{docs_summary}"
|
|
|
|
async def message_summarize_content(self, content: Dict[str, Any]) -> str:
|
|
"""
|
|
Generates a summary for a content item using AI.
|
|
|
|
Args:
|
|
content: Content item to summarize (already processed by get_document_contents)
|
|
|
|
Returns:
|
|
Brief summary of the content
|
|
"""
|
|
# Extract relevant information
|
|
data = content.get("data", "")
|
|
content_type = content.get("content_type", "text/plain")
|
|
is_text = content.get("metadata", {}).get("is_text", False)
|
|
|
|
try:
|
|
# Use the lucy_interface for language-aware AI calls
|
|
summary = await self.lucy_interface.call_ai([
|
|
{"role": "system", "content": "You are a content summarizer. Create very concise summary (1-2 sentences, maximum 200 characters) about this file."},
|
|
{"role": "user", "content": f"Summarize this {content_type} content briefly:\n\n{data}"}
|
|
])
|
|
return summary
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error generating content summary: {str(e)}")
|
|
return f"Text content ({content_type})"
|
|
|
|
|
|
### Documents
|
|
|
|
async def process_file_ids(self, file_ids: List[int]) -> List[Dict[str, Any]]:
|
|
"""
|
|
Processes a list of File-IDs and returns the corresponding file objects as a list of Document objects.
|
|
Loads all contents directly and adds summaries to each content item.
|
|
|
|
Args:
|
|
file_ids: List of file IDs
|
|
|
|
Returns:
|
|
List of Document objects with contents and summaries
|
|
"""
|
|
documents = []
|
|
logger.info(f"Processing {len(file_ids)} files")
|
|
|
|
for file_id in file_ids:
|
|
try:
|
|
# Check if the file exists
|
|
file = self.lucy_interface.get_file(file_id)
|
|
if not file:
|
|
logger.warning(f"File with ID {file_id} not found")
|
|
continue
|
|
|
|
# Check if file belongs to the current mandate
|
|
if file.get("mandate_id") != self.mandate_id:
|
|
logger.warning(f"File {file_id} does not belong to mandate {self.mandate_id}")
|
|
continue
|
|
|
|
# Load file content
|
|
file_content = self.lucy_interface.get_file_data(file_id)
|
|
if file_content is None:
|
|
logger.warning(f"No content found for file with ID {file_id}")
|
|
continue
|
|
|
|
# Create document
|
|
file_name_ext = file.get("name")
|
|
document = {
|
|
"id": f"doc_{str(uuid.uuid4())}",
|
|
"file_id": file_id,
|
|
"name": os.path.splitext(file_name_ext)[0] if os.path.splitext(file_name_ext)[0] else "noname",
|
|
"ext": os.path.splitext(file_name_ext)[1][1:] if os.path.splitext(file_name_ext)[1] else "bin",
|
|
"data": base64.b64encode(file_content).decode('utf-8'), # Add file data as base64
|
|
"contents": []
|
|
}
|
|
|
|
# Extract contents
|
|
contents = get_document_contents(file, file_content)
|
|
|
|
# Add summaries to each content item
|
|
for content in contents:
|
|
content["summary"] = await self.message_summarize_content(content)
|
|
|
|
document["contents"] = contents
|
|
|
|
logger.info(f"File {file.get('name', 'unnamed')} (ID: {file_id}) loaded with {len(contents)} contents and summaries")
|
|
documents.append(document)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error processing file {file_id}: {str(e)}")
|
|
# Continue with remaining files instead of failing
|
|
continue
|
|
|
|
return documents
|
|
|
|
def available_documents_get(self, workflow: Dict[str, Any], message_user: Dict[str, Any]) -> List[Dict[str, Any]]:
|
|
"""
|
|
Determines all currently available documents from user input and already generated documents.
|
|
|
|
Args:
|
|
message_user: Current message from the user
|
|
workflow: Current workflow object
|
|
|
|
Returns:
|
|
List with information about all available documents, sorted by message sequence_nr in descending order
|
|
"""
|
|
available_docs = []
|
|
|
|
if "messages" in workflow and workflow["messages"]:
|
|
for message in workflow["messages"]:
|
|
message_id = message.get("id", "unknown")
|
|
sequence_nr = message.get("sequence_no", 0)
|
|
|
|
# Determine source
|
|
source = "user" if message_id == message_user.get("id") else "workflow"
|
|
|
|
# Process documents in this message
|
|
if "documents" in message and message["documents"]:
|
|
for doc in message["documents"]:
|
|
# Get filename using our helper method
|
|
filename = self.get_filename(doc)
|
|
file_id = doc.get("file_id")
|
|
|
|
# Extract summaries from all contents
|
|
content_summaries = []
|
|
for content in doc.get("contents", []):
|
|
content_summaries.append({
|
|
"content_part": content.get("name","noname"),
|
|
"metadata": content.get("metadata",""),
|
|
"summary": content.get("summary","No summary"),
|
|
})
|
|
|
|
# Create document info
|
|
doc_info = {
|
|
"sequence_nr": sequence_nr,
|
|
"file_source": source,
|
|
"file_id": file_id,
|
|
"message_id": message_id,
|
|
"label": filename,
|
|
"content_summary_list": content_summaries,
|
|
}
|
|
available_docs.append(doc_info)
|
|
|
|
# Sort by message sequence_nr in descending order (newest first)
|
|
available_docs.sort(key=lambda x: x["sequence_nr"], reverse=True)
|
|
|
|
logger.info(f"Available documents: {len(available_docs)}")
|
|
return available_docs
|
|
|
|
def save_document_to_file(self, document: Dict[str, Any]) -> Optional[int]:
|
|
"""
|
|
Saves a Document as a file in the database and returns the File-ID.
|
|
|
|
Args:
|
|
document: Document object with contents
|
|
|
|
Returns:
|
|
File-ID or None on error
|
|
"""
|
|
try:
|
|
if not document or "contents" not in document or not document["contents"]:
|
|
logger.warning("Document has no contents to save")
|
|
return None
|
|
|
|
# Take the first content as main content
|
|
main_content = document["contents"][0]
|
|
name = main_content.get("name", "document")
|
|
content_type = main_content.get("content_type", "text/plain")
|
|
data = main_content.get("data", b"")
|
|
|
|
# Ensure binary data
|
|
if isinstance(data, str):
|
|
data = data.encode('utf-8')
|
|
|
|
# Save file in the database
|
|
file_meta = self.lucy_interface.save_uploaded_file(data, name)
|
|
if file_meta and "id" in file_meta:
|
|
# Update the Document with the File-ID
|
|
document["file_id"] = file_meta["id"]
|
|
return file_meta["id"]
|
|
|
|
return None
|
|
except Exception as e:
|
|
logger.error(f"Error saving document as file: {str(e)}")
|
|
return None
|
|
|
|
def add_document_to_message(self, message: Dict[str, Any], document: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""
|
|
Adds a Document to a message.
|
|
|
|
Args:
|
|
message: Message to which the document should be added
|
|
document: Document to add
|
|
|
|
Returns:
|
|
Updated message
|
|
"""
|
|
# Ensure the documents list exists
|
|
if "documents" not in message:
|
|
message["documents"] = []
|
|
|
|
# Add Document
|
|
message["documents"].append(document)
|
|
|
|
return message
|
|
|
|
|
|
### Tools
|
|
|
|
def get_filename(self, document: Dict[str, Any]) -> str:
|
|
"""
|
|
Gets the filename from a document by combining name and extension.
|
|
|
|
Args:
|
|
document: Document object
|
|
|
|
Returns:
|
|
Filename with extension
|
|
"""
|
|
name = document.get("name", "unnamed")
|
|
ext = document.get("ext", "")
|
|
if ext:
|
|
return f"{name}.{ext}"
|
|
return name
|
|
|
|
def log_add(self, workflow: Dict[str, Any], message: str, level: str = "info",
|
|
progress: Optional[int] = None) -> str:
|
|
"""
|
|
Adds a log entry to the workflow and also logs it in the logger.
|
|
Enhanced with standardized formatting and workflow status tracking.
|
|
|
|
Args:
|
|
workflow: Workflow object
|
|
message: Log message
|
|
level: Log level (info, warning, error)
|
|
progress: Optional - Progress value (0-100)
|
|
|
|
Returns:
|
|
ID of the created log entry
|
|
"""
|
|
# Ensure logs list exists
|
|
if "logs" not in workflow:
|
|
workflow["logs"] = []
|
|
|
|
# Generate log ID
|
|
log_id = f"log_{str(uuid.uuid4())}"
|
|
|
|
# Get workflow status
|
|
workflow_status = workflow.get("status", "running")
|
|
|
|
# Set agent_name from global settings
|
|
agent_name = GLOBAL_SETTINGS.get("system_name", "AI Assistant")
|
|
|
|
# Create log entry
|
|
log_entry = {
|
|
"id": log_id,
|
|
"workflow_id": workflow["id"],
|
|
"message": message,
|
|
"type": level,
|
|
"timestamp": datetime.now().isoformat(),
|
|
"agent_name": agent_name,
|
|
"status": workflow_status
|
|
}
|
|
|
|
# Add progress if provided
|
|
if progress is not None:
|
|
log_entry["progress"] = progress
|
|
|
|
# Add log to workflow
|
|
workflow["logs"].append(log_entry)
|
|
|
|
# Save in database
|
|
self.lucy_interface.create_workflow_log(log_entry)
|
|
|
|
# Also log in logger
|
|
if level == "info":
|
|
logger.info(f"Workflow {workflow['id']}: {message}")
|
|
elif level == "warning":
|
|
logger.warning(f"Workflow {workflow['id']}: {message}")
|
|
elif level == "error":
|
|
logger.error(f"Workflow {workflow['id']}: {message}")
|
|
|
|
return log_id
|
|
|
|
def parse_json2text(self, json_obj: Any) -> str:
|
|
"""
|
|
Converts a JSON object to a readable text representation.
|
|
|
|
Args:
|
|
json_obj: JSON object to convert
|
|
|
|
Returns:
|
|
Formatted text representation
|
|
"""
|
|
if not json_obj:
|
|
return "No data available"
|
|
|
|
try:
|
|
# Format with indentation for better readability
|
|
return json.dumps(json_obj, indent=2, ensure_ascii=False)
|
|
except Exception as e:
|
|
logger.error(f"Error in JSON conversion: {str(e)}")
|
|
return str(json_obj)
|
|
|
|
def parse_json_response(self, response_text: str) -> Dict[str, Any]:
|
|
"""
|
|
Parses the JSON response from a text.
|
|
|
|
Args:
|
|
response_text: Text with JSON content
|
|
|
|
Returns:
|
|
Parsed JSON data
|
|
"""
|
|
try:
|
|
# Extract JSON from the text (if mixed with other content)
|
|
json_start = response_text.find('{')
|
|
json_end = response_text.rfind('}') + 1
|
|
|
|
if json_start >= 0 and json_end > json_start:
|
|
json_str = response_text[json_start:json_end]
|
|
return json.loads(json_str)
|
|
else:
|
|
# Try to parse the entire text
|
|
return json.loads(response_text)
|
|
except json.JSONDecodeError as e:
|
|
logger.error(f"JSON parsing error: {str(e)}")
|
|
# Fallback: Return empty structure
|
|
return {
|
|
"obj_final_documents": [],
|
|
"obj_workplan": [],
|
|
"obj_user_response": "Sorry, I could not parse your data.",
|
|
"user_language": "en"
|
|
}
|
|
|
|
|
|
# Singleton factory for the ChatManager
|
|
_chat_managers = {}
|
|
|
|
def get_chat_manager(mandate_id: int = 0, user_id: int = 0) -> ChatManager:
|
|
"""
|
|
Returns a ChatManager for the specified context.
|
|
Reuses existing instances.
|
|
|
|
Args:
|
|
mandate_id: ID of the mandate
|
|
user_id: ID of the user
|
|
|
|
Returns:
|
|
ChatManager instance
|
|
"""
|
|
context_key = f"{mandate_id}_{user_id}"
|
|
if context_key not in _chat_managers:
|
|
_chat_managers[context_key] = ChatManager(mandate_id, user_id)
|
|
return _chat_managers[context_key] |