"""
ChatManager Module for managing AI-Chat workflows.
Implements a compact and modular architecture for processing
user requests, agent execution, and result formatting.
"""
import os
import logging
import json
import re
import uuid
import base64
from datetime import datetime
from typing import Dict, Any, List, Optional, Union
# Required imports
from connectors.connector_aichat_openai import ChatService
from modules.chat_registry import get_agent_registry
from modules.lucydom_interface import get_lucydom_interface, GLOBAL_SETTINGS
from modules.chat_content_extraction import get_document_contents
# Configure logger
logger = logging.getLogger(__name__)
class ChatManager:
"""
Manages the processing of chat requests, agent execution, and
the integration of results into the workflow.
"""
def __init__(self, mandate_id: int, user_id: int):
"""
Initializes the ChatManager with mandate and user context.
Args:
mandate_id: ID of the current mandate
user_id: ID of the current user
"""
self.mandate_id = mandate_id
self.user_id = user_id
self.ai_service = ChatService()
self.lucy_interface = get_lucydom_interface(mandate_id, user_id)
self.agent_registry = get_agent_registry()
self.agent_registry.set_ai_service(self.ai_service)
# Set AI service in lucy interface for language support
self.lucy_interface.set_ai_service(self.ai_service)
### Chat Management
async def chat_run(self, user_input: Dict[str, Any], workflow_id: Optional[str] = None) -> Dict[str, Any]:
"""
Main function for integrating user requests into the workflow.
Args:
user_input: Dictionary with user request and file IDs
workflow_id: Optional - ID of the workflow (None for new workflows)
Returns:
Workflow object with updated state
"""
# 1. Initialize workflow or load existing one
workflow = self.workflow_init(workflow_id)
self.log_add(workflow, "Starting workflow processing", level="info", progress=0)
# 2. Transform user input into a message object and save in workflow
message_user = await self.chat_message_to_workflow("user", "", user_input, workflow)
# 3. Create project manager prompt and analyze response
self.log_add(workflow, "Analyzing request and planning work", level="info", progress=10)
project_manager_response = await self.chat_prompt(message_user, workflow)
obj_final_documents = project_manager_response.get("obj_final_documents", [])
obj_workplan = project_manager_response.get("obj_workplan", [])
obj_user_response = project_manager_response.get("obj_user_response", "")
# Get detected language and set it in the lucy interface
user_language = project_manager_response.get("user_language", "en")
self.lucy_interface.set_user_language(user_language)
# 4. Save the response as a message in the workflow and add log entries
response_message = {
"role": "assistant",
"agent_name": "project_manager",
"content": obj_user_response
}
self.message_add(workflow, response_message)
self.log_add(workflow, f"Planned outputs: {len(obj_final_documents)} documents", level="info", progress=20)
self.log_add(workflow, f"Work plan created with {len(obj_workplan)} steps", level="info", progress=25)
# 5. Execute agents according to work plan
obj_results = []
if obj_workplan:
total_tasks = len(obj_workplan)
for task_index, task in enumerate(obj_workplan):
agent_name = task.get("agent", "unknown")
progress_value = 30 + int((task_index / total_tasks) * 60) # Progress from 30% to 90%
progress_msg = f"Running task {task_index+1}/{total_tasks}: {agent_name}"
self.log_add(workflow, progress_msg, level="info", progress=progress_value)
task_results = await self.agent_processing(task, workflow)
obj_results.extend(task_results)
# Log completion of this task
self.log_add(
workflow,
f"Completed task {task_index+1}/{total_tasks}: {agent_name}",
level="info",
progress=progress_value + (60/total_tasks)/2
)
# 6. Create the final response with relevant documents from obj_final_documents
self.log_add(workflow, "Creating final response", level="info", progress=90)
final_message = await self.chat_final_message(obj_user_response, obj_final_documents, obj_results)
self.message_add(workflow, final_message)
# 7. Finalize the workflow
self.workflow_finish(workflow)
self.log_add(workflow, "Workflow completed successfully", level="info", progress=100)
return workflow
async def chat_prompt(self, message_user: Dict[str, Any], workflow: Dict[str, Any]) -> Dict[str, Any]:
"""
Creates the prompt for the project manager and processes the response.
Args:
message_user: Message object with user request
workflow: Current workflow object
Returns:
Project manager's response with obj_final_documents, obj_workplan and obj_user_response
"""
# Get available agents with their capabilities
available_agents = self.agent_profiles()
# Create a workflow summary
workflow_summary = await self.workflow_summarize(workflow, message_user)
# Create a list of currently available documents from user input or previously generated documents
available_documents = self.available_documents_get(workflow, message_user)
available_docs_str = json.dumps(available_documents, indent=2)
# Create the prompt for the project manager with language detection requirement
prompt = f"""
Based on the user request and the provided documents, please analyze the requirements and create a processing plan.
Also, identify the language of the user's request and include it in your response.
{message_user.get('content')}
# Previous conversation history:
{workflow_summary}
# Available documents (currently in workflow):
{available_docs_str}
# Available agents and their capabilities:
{self.parse_json2text(available_agents)}
Please analyze the request and create:
1. A list of required result documents (obj_final_documents)
2. A plan for executing agents (obj_workplan)
3. A clear response to the user explaining what you're doing (obj_user_response)
4. Identified language of the user's request (user_language)
## IMPORTANT RULES FOR THE WORKPLAN:
1. Each input document must either already exist (provided by the user or previously created by an agent) or be created by an agent before it's used.
2. If necessary, convert input documents to a suitable format using agents when the type doesn't match.
3. Do not define document inputs that don't exist or haven't been generated beforehand.
4. Create a logical sequence - earlier agents can create documents that are later used as inputs.
5. If the user has provided documents but hasn't clearly stated what they want, try to act according to the context.
Your answer must be strictly in the JSON_OUTPUT format, with no additions before or after the JSON object.
JSON_OUTPUT = {{
"obj_final_documents": [
FILEREF
],
"obj_workplan": [
{{
"agent": "agent_name", # Name of an available agent
"prompt": "Specific instructions to the agent, that he knows what to do with which documents and which output to provide."
"output_documents": [
"label":"document label in the format 'filename.ext'",
"prompt":"AI prompt to describe the content of the file"
],
"input_documents": [
"label":"document label in the format 'filename.ext'",
"file_id":id, # if refering to an existing document, provide file_id to select the correct file
"content_part":"", # provide empty string, if all document contents to consider, otherwise the content_part of the document to focus on
"prompt":"AI prompt to describe what data to extract from the file."
], # If no input documents are needed, include "input_documents" as an empty list
}}
# Multiple agent tasks can be added here and should build logically on each other
],
"obj_user_response": "Information to the user about how his request will be solved.",
"user_language": "en" # Language code (e.g., en, de, fr, es) based on the user's request
}}
## RULES for input_documents:
1. The user request refers to documents where "file_source" in available documents is "user". Those documents are in the focus for input
2. In case of redundant label in available documents, use document with highest sequence_nr if not specified differently
## STRICT RULES FOR document "label":
1. Every document label MUST include a proper file extension that matches the content type.
2. Use standard extensions like:
- ".txt" for text files
- ".md" for markdown files
- ".csv" for comma-separated values
- ".json" for JSON data
- ".html" for HTML content
- ".jpg" or ".png" for images
- ".docx" for Word documents
- ".xlsx" for Excel files
- ".pdf" for PDF documents
3. Use descriptive filenames that indicate the document's purpose (e.g., "analysis_report.txt" rather than just "report.txt")
4. If you use label for an existing file
"""
# Call the AI service through lucy_interface for language support
logger.debug(f"Planning prompt: {prompt}")
project_manager_output = await self.lucy_interface.call_ai([
{
"role": "system",
"content": "You are an experienced project manager who analyzes user requests and creates work plans. You pay very careful attention to ensure that all document dependencies are correct and that no non-existent documents are defined as inputs. The output follows strictly the specified format."
},
{
"role": "user",
"content": prompt
}
])
# Parse the JSON response
return self.parse_json_response(project_manager_output)
async def chat_message_to_workflow(self, role: str, agent_name: str, chat_message: Dict[str, Any], workflow: Dict[str, Any]) -> Dict[str, Any]:
"""
Integrates user inputs into a Message object including files with complete contents.
Args:
role: Role of the message sender ('user' or 'assistant')
agent_name: Name of the agent, if message is from an agent
chat_message: Input data with "prompt"=str, "list_file_id"=[]
workflow: Current workflow object
Returns:
Message object with content and documents including contents
"""
logger.info(f"Message from {role} {agent_name} sent with {len(chat_message.get('list_file_id', []))} documents")
logger.debug(f"message = {self.parse_json2text(chat_message)}.")
# Check message content
message_content = chat_message.get("prompt", "")
if isinstance(message_content, dict) and "content" in message_content:
message_content = message_content["content"]
# If message content is empty, no chat
if role=="user" and (message_content is None or message_content.strip() == ""):
logger.warning(f"Empty message, no chat")
message_content = "(No user input received)"
# Process additional files with complete contents
additional_fileids = chat_message.get("list_file_id", [])
additional_files = await self.process_file_ids(additional_fileids)
# Create message object
message_object = {
"role": role,
"agent_name": agent_name,
"content": message_content,
"documents": additional_files
}
message_object = self.message_add(workflow, message_object)
logger.debug(f"message_user = {self.parse_json2text(message_object)}.")
return message_object
async def chat_final_message(self, obj_user_response: str, obj_final_documents: List[Dict[str, Any]], obj_results: List[Dict[str, Any]]) -> Dict[str, Any]:
"""
Creates the final response message with review of proposed and delivered.
Args:
obj_user_response: Initial text response to the user
obj_final_documents: List of expected response documents
obj_results: List of generated result documents
Returns:
Complete message object with content and relevant documents
"""
# Find documents that match the obj_final_documents requirements
matching_documents = []
for answer_spec in obj_final_documents:
answer_label = answer_spec.get("label")
# Find matching document in results
for doc in obj_results:
doc_name = self.get_filename(doc)
# Check if this document matches the answer specification
if doc_name == answer_label:
content_ref = []
for c in doc.get("contents"):
content_ref.append(c.get("summary"))
doc_ref = {
"label": doc_name,
"content_summary": content_ref
}
matching_documents.append(doc_ref)
break
# Use the lucy_interface for language-aware AI calls
final_prompt = await self.lucy_interface.call_ai([
{"role": "system", "content": "You are a project manager, who delivers results to a user."},
{"role": "user", "content": f"""
Give the final short feedback to the user with reference to the initial statement (obj_user_response). Provide a list of delivered files (files_delivered). If in the list of delivered files (files_delivered) some files from the original list (files_promised) are not available, then just give a comment on this, otherwise task is completed.
Here the data:
obj_user_response = {self.parse_json2text(obj_user_response)}
files_promised = {self.parse_json2text(matching_documents)}
files_delivered = {self.parse_json2text(obj_user_response)}
"""
}
], produce_user_answer=True)
# Create basic message structure with proper fields
logger.debug(f"FINAL PROMPT = {self.parse_json2text(final_prompt)}.")
final_message = {
"role": "assistant",
"agent_name": "project_manager",
"content": final_prompt,
"documents": [] # DO NOT include the results documents, already with agents
}
logger.debug(f"FINAL MESSAGE = {self.parse_json2text(final_message)}.")
return final_message
### Workflow
def workflow_init(self, workflow_id: Optional[str] = None) -> Dict[str, Any]:
"""
Initializes a workflow or loads an existing one with round counting.
Args:
workflow_id: Optional - ID of the workflow to load
Returns:
Initialized workflow object
"""
current_time = datetime.now().isoformat()
if workflow_id is None or not self.lucy_interface.get_workflow(workflow_id):
# Create new workflow
new_workflow_id = str(uuid.uuid4()) if workflow_id is None else workflow_id
workflow = {
"id": new_workflow_id,
"mandate_id": self.mandate_id,
"user_id": self.user_id,
"name": f"Workflow {new_workflow_id[:8]}",
"started_at": current_time,
"messages": [], # Empty list - will be filled with references
"message_ids": [], # Initialize empty message_ids list
"logs": [],
"data_stats": {},
"current_round": 1,
"status": "running",
"last_activity": current_time,
}
# Save to database - only the workflow metadata
workflow_db = {
"id": workflow["id"],
"mandate_id": workflow["mandate_id"],
"user_id": workflow["user_id"],
"name": workflow["name"],
"started_at": workflow["started_at"],
"status": workflow["status"],
"data_stats": workflow["data_stats"],
"current_round": workflow["current_round"],
"last_activity": workflow["last_activity"],
"message_ids": workflow["message_ids"] # Include message_ids
}
self.lucy_interface.create_workflow(workflow_db)
self.log_add(workflow, GLOBAL_SETTINGS["workflow_status_messages"]["init"], level="info", progress=0)
return workflow
else:
# Load existing workflow
workflow = self.lucy_interface.load_workflow_state(workflow_id)
# Ensure message_ids exists
if "message_ids" not in workflow:
# Initialize from existing messages
workflow["message_ids"] = [msg["id"] for msg in workflow.get("messages", [])]
# Update in database
self.lucy_interface.update_workflow(workflow_id, {"message_ids": workflow["message_ids"]})
# Update status and increment round counter
workflow["status"] = "running"
workflow["last_activity"] = current_time
# Increment current_round if it exists, otherwise set it to 1
if "current_round" in workflow:
workflow["current_round"] += 1
else:
workflow["current_round"] = 1
# Update in database - only the relevant workflow fields
workflow_update = {
"status": workflow["status"],
"last_activity": workflow["last_activity"],
"current_round": workflow["current_round"]
}
self.lucy_interface.update_workflow(workflow_id, workflow_update)
self.log_add(workflow, GLOBAL_SETTINGS["workflow_status_messages"]["running"], level="info", progress=0)
return workflow
def workflow_finish(self, workflow: Dict[str, Any]) -> Dict[str, Any]:
"""
Finalizes a workflow and sets the status to 'completed'.
Args:
workflow: Workflow object
Returns:
Updated workflow object
"""
# Prepare workflow update data
workflow_update = {
"status": "completed",
"last_activity": datetime.now().isoformat(),
}
# Update the workflow object in memory
workflow["status"] = workflow_update["status"]
workflow["last_activity"] = workflow_update["last_activity"]
# Save workflow state to database - only relevant fields, not the messages list
self.lucy_interface.update_workflow(workflow["id"], workflow_update)
self.log_add(workflow, GLOBAL_SETTINGS["workflow_status_messages"]["completed"], level="info", progress=100)
return workflow
async def workflow_summarize(self, workflow: Dict[str, Any], message_user: Dict[str, Any]) -> str:
"""
Creates a summary of the workflow without the current user message.
Args:
workflow: Workflow object
message_user: Current user message
Returns:
Summary of the workflow
"""
if not workflow or "messages" not in workflow or not workflow["messages"]:
return "" # first message
# Go through messages in reverse order (newest first)
messages = sorted(workflow["messages"], key=lambda m: m.get("sequence_no", 0), reverse=False)
summary_parts = []
for message in messages:
if True: # including user message, excluding would be: message["id"] != message_user["id"]:
message_summary = await self.message_summarize(message)
summary_parts.append(message_summary)
return "\n\n".join(summary_parts)
### Agents
def agent_profiles(self) -> List[Dict[str, Any]]:
"""
Gets information about all available agents.
Returns:
List with information about all available agents
"""
return self.agent_registry.get_agent_infos()
async def agent_input_documents(self, doc_input_list: List[Dict[str, Any]], workflow: Dict[str, Any]) -> List[Dict[str, Any]]:
"""
Prepares input documents for an agent, sorted with newest first.
Args:
doc_input_list: List of required input documents as specified by the project manager
workflow: Workflow object
Returns:
Prepared input documents for the agent, sorted with newest first
"""
prepared_inputs = []
# Sort workflow messages by sequence number (descending)
sorted_messages = sorted(
workflow.get("messages", []),
key=lambda m: m.get("sequence_no", 0),
reverse=True
)
for doc_spec in doc_input_list:
doc_filename = doc_spec.get("label","")
doc_file_id = doc_spec.get("file_id","")
found_doc = None
# Search for the document in sorted workflow messages (newest first)
for message in sorted_messages:
for doc in message.get("documents", []):
if (doc_file_id!="" and doc_file_id==doc.get("file_id")) or (doc_filename!="" and self.get_filename(doc) == doc_filename):
found_doc = doc
break
if found_doc:
break
if found_doc:
# Process document for agent based on the specification
processed_doc = await self.process_document_for_agent(found_doc, doc_spec)
prepared_inputs.append(processed_doc)
else:
logger.warning(f"Document with label '{doc_filename}', file_id '{doc_file_id}' not found in workflow")
return prepared_inputs
async def process_document_for_agent(self, document: Dict[str, Any], doc_spec: Dict[str, Any]) -> Dict[str, Any]:
"""
Processes a document for an agent based on the document specification.
Uses AI to extract relevant content from the document based on the specification.
Args:
document: The document to process
doc_spec: The document specification from the project manager
Returns:
Processed document with AI-extracted content
"""
processed_doc = document.copy()
part_spec = doc_spec.get("content_part","")
# Process each content item in the document
if "contents" in processed_doc:
processed_contents = []
for content in processed_doc["contents"]:
# Check if part required
if part_spec != "" and part_spec != content.get("name"):
continue
# Get the data from the content
data = content.get("data", "")
processed_content = content.copy()
# Check if content data is base64 encoded
is_base64 = content.get("metadata", {}).get("base64_encoded", False)
try:
# Use the AI service to process the document content according to the prompt from the project manager for the document specification
summary = doc_spec.get("prompt", "Extract the relevant information from this document")
ai_prompt = f"""
# Please process the following document content according to this instruction:
{summary}
# Document content:
{data}
# Extract and provide only the relevant information as requested.
"""
# Call the AI service through lucy_interface for language support
processed_data = await self.lucy_interface.call_ai([
{"role": "system", "content": "You are a document processing assistant. Extract only the relevant information as requested."},
{"role": "user", "content": ai_prompt}
])
# DO NOT change the original data field
# processed_content["data"] unchanged
processed_content["data_extracted"] = processed_data
processed_content["metadata"]["ai_processed"] = True
except Exception as e:
logger.error(f"Error processing document content with AI: {str(e)}")
# Fall back to original content if AI processing fails
processed_content["data_extracted"] = "(no information)"
processed_contents.append(processed_content)
processed_doc["contents"] = processed_contents
return processed_doc
async def agent_processing(self, task: Dict[str, Any], workflow: Dict[str, Any]) -> List[Dict[str, Any]]:
"""
Process a single agent task from the workflow.
Optimized for the task-based approach where all agents implement process_task.
Args:
task: The task definition containing agent name, prompt, and document specifications
workflow: The current workflow object
Returns:
List of document objects created by the agent
"""
# 1. Extract task information
agent_name = task.get("agent")
agent_prompt = task.get("prompt", "")
# Log the current step
output_labels = [d.get("label", "unknown") for d in task.get("output_documents", [])]
step_info = f"Agent '{agent_name}' to create {', '.join(output_labels)}."
self.log_add(workflow, step_info, level="info")
# Check if prompt is empty
if agent_prompt == "":
logger.warning("Empty prompt, no task to do")
return []
# Get agent from registry
agent = self.agent_registry.get_agent(agent_name)
if not agent:
logger.error(f"Agent '{agent_name}' not found")
return []
# Prepare output document specifications
output_specs = []
for doc in task.get("output_documents", []):
output_spec = {
"label": doc.get("label"),
"description": doc.get("prompt", "")
}
output_specs.append(output_spec)
# Prepare input documents for the agent
input_documents = await self.agent_input_documents(task.get('input_documents', []), workflow)
# Create a standardized task object for the agent
agent_task = {
"task_id": str(uuid.uuid4()),
"workflow_id": workflow.get("id"),
"prompt": agent_prompt,
"input_documents": input_documents,
"output_specifications": output_specs,
"context": {
"workflow_round": workflow.get("current_round", 1),
"agent_type": agent_name,
"timestamp": datetime.now().isoformat(),
"language": self.lucy_interface.user_language # Pass language to agent
}
}
# Execute the agent with the standardized task
try:
# Process the task using the agent's standardized interface
logger.debug("TASK: "+self.parse_json2text(agent_task))
logger.debug(f"Agent '{agent_name}' AI service available: {agent.ai_service is not None}")
agent_results = await agent.process_task(agent_task)
logger.debug(f"Agent '{agent_name}' completed task. RESULT: {self.parse_json2text(agent_results)}")
# Log the agent response
self.log_add(
workflow,
f"Agent '{agent_name}' completed task. Feedback: {agent_results.get('feedback', 'No feedback provided')}",
level="info"
)
# Store produced files and prepare input object for message
agent_inputs = {
"prompt": agent_results.get("feedback", ""),
"list_file_id": self.agent_save_documents(agent_results)
}
# Create a message in the workflow with the agent's response
agent_message = await self.chat_message_to_workflow("assistant", agent_name, agent_inputs, workflow)
logger.debug(f"Agent result = {self.parse_json2text(agent_message)}.")
return agent_message.get("documents", [])
except Exception as e:
error_msg = f"Error executing agent '{agent_name}': {str(e)}"
logger.error(error_msg, exc_info=True) # Add exc_info=True to get full traceback
self.log_add(workflow, error_msg, level="error")
return []
def agent_save_documents(self, agent_results: Dict[str, Any]) -> List[int]:
"""
Saves all documents from agent results as files and returns a list of file IDs.
Enhanced to handle the standardized document format from agents.
Args:
agent_results: Dictionary containing agent feedback and documents
Returns:
List of file IDs for the saved documents
"""
file_ids = []
# Extract documents from agent results
documents = agent_results.get("documents", [])
for doc in documents:
try:
# Extract label (filename) and content
label = doc.get("label", "unnamed_file.txt")
content = doc.get("content", "")
# Split label into name and extension
name, ext = os.path.splitext(label)
if ext.startswith('.'):
ext = ext[1:] # Remove leading dot
elif not ext:
# If no extension is provided, default to .txt for text content
ext = "txt"
label = f"{label}.{ext}"
# Determine if content is base64 encoded
is_base64 = False
if isinstance(content, dict) and content.get("metadata", {}).get("base64_encoded", False):
is_base64 = True
content = content.get("data", "")
# Convert content to bytes
if isinstance(content, str):
if is_base64:
# Decode base64 to bytes
try:
file_content = base64.b64decode(content)
except Exception as e:
logger.warning(f"Failed to decode base64 content: {str(e)}")
file_content = content.encode('utf-8')
else:
# Convert text to bytes
file_content = content.encode('utf-8')
else:
# Already bytes
file_content = content
# Save file to database
file_meta = self.lucy_interface.save_uploaded_file(file_content, label)
if file_meta and "id" in file_meta:
file_id = file_meta["id"]
file_ids.append(file_id)
logger.info(f"Saved document '{label}' with file ID: {file_id}")
else:
logger.warning(f"Failed to save document '{label}'")
except Exception as e:
logger.error(f"Error saving document from agent results: {str(e)}")
# Continue with other documents instead of failing
continue
return file_ids
### Messages
def message_add(self, workflow: Dict[str, Any], message: Dict[str, Any]) -> Dict[str, Any]:
"""
Adds a message to the workflow and updates last_activity.
Saves the message in the database and updates the workflow with references.
Args:
workflow: Workflow object
message: Message to be saved
Returns:
Added message
"""
current_time = datetime.now().isoformat()
# Ensure messages list exists
if "messages" not in workflow:
workflow["messages"] = []
# Generate new message ID if not present
if "id" not in message:
message["id"] = f"msg_{str(uuid.uuid4())}"
# Add workflow ID and timestamps
message["workflow_id"] = workflow["id"]
message["started_at"] = current_time
message["finished_at"] = current_time
# Set sequence number
message["sequence_no"] = len(workflow["messages"]) + 1
# Ensure required fields are present
if "role" not in message:
# Set a default role based on agent_name
message["role"] = "assistant" if message.get("agent_name") else "user"
if "agent_name" not in message:
message["agent_name"] = ""
# Set status
message["status"] = "completed"
# Add message to workflow
workflow["messages"].append(message)
# Ensure message_ids list exists
if "message_ids" not in workflow:
workflow["message_ids"] = []
# Add message ID to the message_ids list
workflow["message_ids"].append(message["id"])
# Update workflow status
workflow["last_activity"] = current_time
# Save to database - first the message itself
self.lucy_interface.create_workflow_message(message)
# Then save the workflow with updated references
workflow_update = {
"last_activity": current_time,
"message_ids": workflow["message_ids"] # Update the message_ids field
}
self.lucy_interface.update_workflow(workflow["id"], workflow_update)
return message
async def message_summarize(self, message: Dict[str, Any]) -> str:
"""
Creates a summary of a message including its documents.
Args:
message: Message to summarize
Returns:
Summary of the message
"""
role = message.get("role", "undefined")
agent_name = message.get("agent_name", "")
content = message.get("content", "")
try:
# Use the lucy_interface for language-aware AI calls
content_summary = await self.lucy_interface.call_ai([
{"role": "system", "content": f"You are a chat message summarizer. Create a very concise summary (2-3 sentences, maximum 300 characters)"},
{"role": "user", "content": content}
])
except Exception as e:
logger.error(f"Error creating summary: {str(e)}")
content_summary = content[:200] + "..."
# Summarize documents
docs_summary = ""
if "documents" in message and message["documents"]:
docs_list = []
for i, doc in enumerate(message["documents"]):
doc_name = self.get_filename(doc)
docs_list.append(doc_name)
if docs_list:
docs_summary = "\nDocuments:" + "\n- ".join(docs_list)
return f"[{role} {agent_name}]: {content_summary}{docs_summary}"
async def message_summarize_content(self, content: Dict[str, Any]) -> str:
"""
Generates a summary for a content item using AI.
Args:
content: Content item to summarize (already processed by get_document_contents)
Returns:
Brief summary of the content
"""
# Extract relevant information
data = content.get("data", "")
content_type = content.get("content_type", "text/plain")
is_text = content.get("metadata", {}).get("is_text", False)
try:
# Use the lucy_interface for language-aware AI calls
summary = await self.lucy_interface.call_ai([
{"role": "system", "content": "You are a content summarizer. Create very concise summary (1-2 sentences, maximum 200 characters) about this file."},
{"role": "user", "content": f"Summarize this {content_type} content briefly:\n\n{data}"}
])
return summary
except Exception as e:
logger.error(f"Error generating content summary: {str(e)}")
return f"Text content ({content_type})"
### Documents
async def process_file_ids(self, file_ids: List[int]) -> List[Dict[str, Any]]:
"""
Processes a list of File-IDs and returns the corresponding file objects as a list of Document objects.
Loads all contents directly and adds summaries to each content item.
Args:
file_ids: List of file IDs
Returns:
List of Document objects with contents and summaries
"""
documents = []
logger.info(f"Processing {len(file_ids)} files")
for file_id in file_ids:
try:
# Check if the file exists
file = self.lucy_interface.get_file(file_id)
if not file:
logger.warning(f"File with ID {file_id} not found")
continue
# Check if file belongs to the current mandate
if file.get("mandate_id") != self.mandate_id:
logger.warning(f"File {file_id} does not belong to mandate {self.mandate_id}")
continue
# Load file content
file_content = self.lucy_interface.get_file_data(file_id)
if file_content is None:
logger.warning(f"No content found for file with ID {file_id}")
continue
# Create document
file_name_ext = file.get("name")
document = {
"id": f"doc_{str(uuid.uuid4())}",
"file_id": file_id,
"name": os.path.splitext(file_name_ext)[0] if os.path.splitext(file_name_ext)[0] else "noname",
"ext": os.path.splitext(file_name_ext)[1][1:] if os.path.splitext(file_name_ext)[1] else "bin",
"data": base64.b64encode(file_content).decode('utf-8'), # Add file data as base64
"contents": []
}
# Extract contents
contents = get_document_contents(file, file_content)
# Add summaries to each content item
for content in contents:
content["summary"] = await self.message_summarize_content(content)
document["contents"] = contents
logger.info(f"File {file.get('name', 'unnamed')} (ID: {file_id}) loaded with {len(contents)} contents and summaries")
documents.append(document)
except Exception as e:
logger.error(f"Error processing file {file_id}: {str(e)}")
# Continue with remaining files instead of failing
continue
return documents
def available_documents_get(self, workflow: Dict[str, Any], message_user: Dict[str, Any]) -> List[Dict[str, Any]]:
"""
Determines all currently available documents from user input and already generated documents.
Args:
message_user: Current message from the user
workflow: Current workflow object
Returns:
List with information about all available documents, sorted by message sequence_nr in descending order
"""
available_docs = []
if "messages" in workflow and workflow["messages"]:
for message in workflow["messages"]:
message_id = message.get("id", "unknown")
sequence_nr = message.get("sequence_no", 0)
# Determine source
source = "user" if message_id == message_user.get("id") else "workflow"
# Process documents in this message
if "documents" in message and message["documents"]:
for doc in message["documents"]:
# Get filename using our helper method
filename = self.get_filename(doc)
file_id = doc.get("file_id")
# Extract summaries from all contents
content_summaries = []
for content in doc.get("contents", []):
content_summaries.append({
"content_part": content.get("name","noname"),
"metadata": content.get("metadata",""),
"summary": content.get("summary","No summary"),
})
# Create document info
doc_info = {
"sequence_nr": sequence_nr,
"file_source": source,
"file_id": file_id,
"message_id": message_id,
"label": filename,
"content_summary_list": content_summaries,
}
available_docs.append(doc_info)
# Sort by message sequence_nr in descending order (newest first)
available_docs.sort(key=lambda x: x["sequence_nr"], reverse=True)
logger.info(f"Available documents: {len(available_docs)}")
return available_docs
def save_document_to_file(self, document: Dict[str, Any]) -> Optional[int]:
"""
Saves a Document as a file in the database and returns the File-ID.
Args:
document: Document object with contents
Returns:
File-ID or None on error
"""
try:
if not document or "contents" not in document or not document["contents"]:
logger.warning("Document has no contents to save")
return None
# Take the first content as main content
main_content = document["contents"][0]
name = main_content.get("name", "document")
content_type = main_content.get("content_type", "text/plain")
data = main_content.get("data", b"")
# Ensure binary data
if isinstance(data, str):
data = data.encode('utf-8')
# Save file in the database
file_meta = self.lucy_interface.save_uploaded_file(data, name)
if file_meta and "id" in file_meta:
# Update the Document with the File-ID
document["file_id"] = file_meta["id"]
return file_meta["id"]
return None
except Exception as e:
logger.error(f"Error saving document as file: {str(e)}")
return None
def add_document_to_message(self, message: Dict[str, Any], document: Dict[str, Any]) -> Dict[str, Any]:
"""
Adds a Document to a message.
Args:
message: Message to which the document should be added
document: Document to add
Returns:
Updated message
"""
# Ensure the documents list exists
if "documents" not in message:
message["documents"] = []
# Add Document
message["documents"].append(document)
return message
### Tools
def get_filename(self, document: Dict[str, Any]) -> str:
"""
Gets the filename from a document by combining name and extension.
Args:
document: Document object
Returns:
Filename with extension
"""
name = document.get("name", "unnamed")
ext = document.get("ext", "")
if ext:
return f"{name}.{ext}"
return name
def log_add(self, workflow: Dict[str, Any], message: str, level: str = "info",
progress: Optional[int] = None) -> str:
"""
Adds a log entry to the workflow and also logs it in the logger.
Enhanced with standardized formatting and workflow status tracking.
Args:
workflow: Workflow object
message: Log message
level: Log level (info, warning, error)
progress: Optional - Progress value (0-100)
Returns:
ID of the created log entry
"""
# Ensure logs list exists
if "logs" not in workflow:
workflow["logs"] = []
# Generate log ID
log_id = f"log_{str(uuid.uuid4())}"
# Get workflow status
workflow_status = workflow.get("status", "running")
# Set agent_name from global settings
agent_name = GLOBAL_SETTINGS.get("system_name", "AI Assistant")
# Create log entry
log_entry = {
"id": log_id,
"workflow_id": workflow["id"],
"message": message,
"type": level,
"timestamp": datetime.now().isoformat(),
"agent_name": agent_name,
"status": workflow_status
}
# Add progress if provided
if progress is not None:
log_entry["progress"] = progress
# Add log to workflow
workflow["logs"].append(log_entry)
# Save in database
self.lucy_interface.create_workflow_log(log_entry)
# Also log in logger
if level == "info":
logger.info(f"Workflow {workflow['id']}: {message}")
elif level == "warning":
logger.warning(f"Workflow {workflow['id']}: {message}")
elif level == "error":
logger.error(f"Workflow {workflow['id']}: {message}")
return log_id
def parse_json2text(self, json_obj: Any) -> str:
"""
Converts a JSON object to a readable text representation.
Args:
json_obj: JSON object to convert
Returns:
Formatted text representation
"""
if not json_obj:
return "No data available"
try:
# Format with indentation for better readability
return json.dumps(json_obj, indent=2, ensure_ascii=False)
except Exception as e:
logger.error(f"Error in JSON conversion: {str(e)}")
return str(json_obj)
def parse_json_response(self, response_text: str) -> Dict[str, Any]:
"""
Parses the JSON response from a text.
Args:
response_text: Text with JSON content
Returns:
Parsed JSON data
"""
try:
# Extract JSON from the text (if mixed with other content)
json_start = response_text.find('{')
json_end = response_text.rfind('}') + 1
if json_start >= 0 and json_end > json_start:
json_str = response_text[json_start:json_end]
return json.loads(json_str)
else:
# Try to parse the entire text
return json.loads(response_text)
except json.JSONDecodeError as e:
logger.error(f"JSON parsing error: {str(e)}")
# Fallback: Return empty structure
return {
"obj_final_documents": [],
"obj_workplan": [],
"obj_user_response": "Sorry, I could not parse your data.",
"user_language": "en"
}
# Singleton factory for the ChatManager
_chat_managers = {}
def get_chat_manager(mandate_id: int = 0, user_id: int = 0) -> ChatManager:
"""
Returns a ChatManager for the specified context.
Reuses existing instances.
Args:
mandate_id: ID of the mandate
user_id: ID of the user
Returns:
ChatManager instance
"""
context_key = f"{mandate_id}_{user_id}"
if context_key not in _chat_managers:
_chat_managers[context_key] = ChatManager(mandate_id, user_id)
return _chat_managers[context_key]