1078 lines
46 KiB
Python
1078 lines
46 KiB
Python
# Copyright (c) 2025 Patrick Motsch
|
|
# All rights reserved.
|
|
"""
|
|
Simple chatbot feature - direct AI center implementation.
|
|
Bypasses complex workflow engine for fast, simple chatbot responses.
|
|
"""
|
|
|
|
import logging
|
|
import json
|
|
import uuid
|
|
import asyncio
|
|
import re
|
|
import datetime
|
|
import base64
|
|
from typing import Optional, Dict, Any, List
|
|
|
|
from modules.datamodels.datamodelChat import ChatWorkflow, UserInputRequest, ChatDocument, WorkflowModeEnum
|
|
from modules.datamodels.datamodelUam import User
|
|
from modules.datamodels.datamodelAi import AiCallRequest, AiCallOptions, OperationTypeEnum, ProcessingModeEnum
|
|
from modules.shared.timeUtils import getUtcTimestamp, parseTimestamp
|
|
from modules.services import getInterface as getServices
|
|
from modules.connectors.connectorPreprocessor import PreprocessorConnector
|
|
from modules.features.chatbot.eventManager import get_event_manager
|
|
from modules.datamodels.datamodelDocref import DocumentReferenceList, DocumentItemReference
|
|
from modules.workflows.methods.methodAi.methodAi import MethodAi
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _extractJsonFromResponse(content: str) -> Optional[dict]:
|
|
"""Extract JSON from AI response, handling markdown code blocks."""
|
|
# Try direct JSON parse first
|
|
try:
|
|
return json.loads(content.strip())
|
|
except json.JSONDecodeError:
|
|
pass
|
|
|
|
# Try to extract JSON from markdown code blocks
|
|
json_match = re.search(r'```(?:json)?\s*(\{.*?\})\s*```', content, re.DOTALL)
|
|
if json_match:
|
|
try:
|
|
return json.loads(json_match.group(1))
|
|
except json.JSONDecodeError:
|
|
pass
|
|
|
|
# Try to find JSON object in the text
|
|
json_match = re.search(r'\{.*\}', content, re.DOTALL)
|
|
if json_match:
|
|
try:
|
|
return json.loads(json_match.group(0))
|
|
except json.JSONDecodeError:
|
|
pass
|
|
|
|
return None
|
|
|
|
|
|
async def _generate_conversation_name(
|
|
services,
|
|
userPrompt: str,
|
|
userLanguage: str = "en"
|
|
) -> str:
|
|
"""
|
|
Generate a short, descriptive conversation name based on user's prompt.
|
|
|
|
Args:
|
|
services: Services instance with AI access
|
|
userPrompt: The user's input prompt
|
|
userLanguage: User's preferred language (for prompt localization)
|
|
|
|
Returns:
|
|
Short conversation name (max 60 characters)
|
|
"""
|
|
try:
|
|
truncated_prompt = userPrompt[:200] if len(userPrompt) > 200 else userPrompt
|
|
|
|
name_prompt = f"""Create a professional conversation title in THE SAME LANGUAGE as the user's question.
|
|
|
|
Question: "{truncated_prompt}"
|
|
|
|
Rules:
|
|
- Title MUST be in the same language as the question (German→German, French→French, English→English)
|
|
- Max 60 characters, no punctuation (?, !, .)
|
|
- Professional and concise
|
|
- Respond ONLY with the title, nothing else"""
|
|
|
|
await services.ai.ensureAiObjectsInitialized()
|
|
|
|
nameRequest = AiCallRequest(
|
|
prompt=name_prompt,
|
|
options=AiCallOptions(
|
|
resultFormat="txt",
|
|
operationType=OperationTypeEnum.DATA_GENERATE,
|
|
processingMode=ProcessingModeEnum.DETAILED,
|
|
temperature=0.7
|
|
)
|
|
)
|
|
|
|
nameResponse = await services.ai.callAi(nameRequest)
|
|
generated_name = nameResponse.content.strip()
|
|
|
|
# Extract first line and clean up
|
|
generated_name = generated_name.split('\n')[0].strip()
|
|
generated_name = re.sub(r'^(Title|Titel|Titre|Name|Name:):\s*', '', generated_name, flags=re.IGNORECASE)
|
|
generated_name = re.sub(r'^["\']|["\']$', '', generated_name)
|
|
generated_name = re.sub(r'[?!.]+$', '', generated_name) # Remove trailing punctuation
|
|
|
|
# Apply title case
|
|
if generated_name:
|
|
words = generated_name.split()
|
|
capitalized_words = []
|
|
for word in words:
|
|
if word.isupper() and len(word) > 1:
|
|
capitalized_words.append(word) # Keep acronyms
|
|
else:
|
|
capitalized_words.append(word.capitalize())
|
|
generated_name = " ".join(capitalized_words).strip()
|
|
|
|
# Validate and truncate if needed
|
|
if not generated_name or len(generated_name) < 3:
|
|
if userLanguage == "de":
|
|
generated_name = "Chatbot Konversation"
|
|
elif userLanguage == "fr":
|
|
generated_name = "Conversation Chatbot"
|
|
else:
|
|
generated_name = "Chatbot Conversation"
|
|
|
|
if len(generated_name) > 60:
|
|
truncated = generated_name[:57]
|
|
last_space = truncated.rfind(' ')
|
|
generated_name = truncated[:last_space] + "..." if last_space > 30 else truncated + "..."
|
|
|
|
logger.info(f"Generated conversation name: '{generated_name}'")
|
|
return generated_name
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error generating conversation name: {e}", exc_info=True)
|
|
if userLanguage == "de":
|
|
return "Chatbot Konversation"
|
|
elif userLanguage == "fr":
|
|
return "Conversation Chatbot"
|
|
else:
|
|
return "Chatbot Conversation"
|
|
|
|
|
|
async def chatProcess(
|
|
currentUser: User,
|
|
userInput: UserInputRequest,
|
|
workflowId: Optional[str] = None
|
|
) -> ChatWorkflow:
|
|
"""
|
|
Simple chatbot processing - direct AI center implementation.
|
|
|
|
Flow:
|
|
1. Create or load workflow
|
|
2. Store user message
|
|
3. AI analyzes: determine if DB query/web research needed
|
|
4. Execute database query if needed
|
|
5. Execute web research if needed
|
|
6. Generate final answer
|
|
7. Store assistant message
|
|
8. Return workflow
|
|
|
|
Args:
|
|
currentUser: Current user
|
|
userInput: User input request
|
|
workflowId: Optional workflow ID to continue existing conversation
|
|
|
|
Returns:
|
|
ChatWorkflow instance
|
|
"""
|
|
try:
|
|
# Get services
|
|
services = getServices(currentUser, None)
|
|
interfaceDbChat = services.interfaceDbChat
|
|
|
|
# Get event manager and create queue if needed
|
|
event_manager = get_event_manager()
|
|
|
|
# Create or load workflow
|
|
if workflowId:
|
|
workflow = interfaceDbChat.getWorkflow(workflowId)
|
|
if not workflow:
|
|
raise ValueError(f"Workflow {workflowId} not found")
|
|
|
|
# Resume workflow: increment round number
|
|
new_round = workflow.currentRound + 1
|
|
interfaceDbChat.updateWorkflow(workflowId, {
|
|
"status": "running",
|
|
"currentRound": new_round,
|
|
"lastActivity": getUtcTimestamp()
|
|
})
|
|
workflow = interfaceDbChat.getWorkflow(workflowId)
|
|
logger.info(f"Resumed workflow {workflowId}, round incremented to {new_round}")
|
|
|
|
# Create event queue if it doesn't exist (for streaming)
|
|
if not event_manager.has_queue(workflowId):
|
|
event_manager.create_queue(workflowId)
|
|
else:
|
|
# Generate conversation name based on user's prompt
|
|
conversation_name = await _generate_conversation_name(
|
|
services,
|
|
userInput.prompt,
|
|
userInput.userLanguage
|
|
)
|
|
|
|
# Create new workflow
|
|
workflowData = {
|
|
"id": str(uuid.uuid4()),
|
|
"mandateId": currentUser.mandateId,
|
|
"status": "running",
|
|
"name": conversation_name,
|
|
"currentRound": 1,
|
|
"currentTask": 0,
|
|
"currentAction": 0,
|
|
"totalTasks": 0,
|
|
"totalActions": 0,
|
|
"workflowMode": WorkflowModeEnum.WORKFLOW_CHATBOT.value, # Use Chatbot mode for chatbot conversations
|
|
"startedAt": getUtcTimestamp(),
|
|
"lastActivity": getUtcTimestamp()
|
|
}
|
|
workflow = interfaceDbChat.createWorkflow(workflowData)
|
|
logger.info(f"Created new chatbot workflow: {workflow.id} with name: {conversation_name}")
|
|
|
|
# Create event queue for new workflow (for streaming)
|
|
event_manager.create_queue(workflow.id)
|
|
|
|
# Reload workflow to get current message count
|
|
workflow = interfaceDbChat.getWorkflow(workflow.id)
|
|
|
|
# Process uploaded files and create ChatDocuments
|
|
user_documents = []
|
|
if userInput.listFileId and len(userInput.listFileId) > 0:
|
|
logger.info(f"Processing {len(userInput.listFileId)} uploaded file(s) for user message")
|
|
for fileId in userInput.listFileId:
|
|
try:
|
|
# Get file info from chat service
|
|
fileInfo = services.chat.getFileInfo(fileId)
|
|
if not fileInfo:
|
|
logger.warning(f"No file info found for file ID {fileId}")
|
|
continue
|
|
|
|
originalFileName = fileInfo.get("fileName", "unknown")
|
|
originalMimeType = fileInfo.get("mimeType", "application/octet-stream")
|
|
fileSizeToUse = fileInfo.get("size", 0)
|
|
|
|
# Create ChatDocument for the file
|
|
document = ChatDocument(
|
|
id=str(uuid.uuid4()),
|
|
messageId="", # Will be set when message is created
|
|
fileId=fileId,
|
|
fileName=originalFileName,
|
|
fileSize=fileSizeToUse,
|
|
mimeType=originalMimeType,
|
|
roundNumber=workflow.currentRound,
|
|
taskNumber=0,
|
|
actionNumber=0
|
|
)
|
|
user_documents.append(document)
|
|
logger.info(f"Created ChatDocument for file {fileId} -> {originalFileName}")
|
|
except Exception as e:
|
|
logger.error(f"Error processing file ID {fileId}: {e}", exc_info=True)
|
|
|
|
# Store user message
|
|
userMessageData = {
|
|
"id": f"msg_{uuid.uuid4()}",
|
|
"workflowId": workflow.id,
|
|
"message": userInput.prompt,
|
|
"role": "user",
|
|
"status": "first" if workflowId is None else "step",
|
|
"sequenceNr": len(workflow.messages) + 1,
|
|
"publishedAt": getUtcTimestamp(),
|
|
"roundNumber": workflow.currentRound,
|
|
"taskNumber": 0,
|
|
"actionNumber": 0
|
|
}
|
|
|
|
# Add documents to message if any
|
|
if user_documents:
|
|
# Update messageId in all documents
|
|
for doc in user_documents:
|
|
doc.messageId = userMessageData["id"]
|
|
userMessageData["documents"] = [doc.dict() for doc in user_documents]
|
|
userMessageData["documentsLabel"] = "Uploaded Files"
|
|
logger.info(f"Attaching {len(user_documents)} document(s) to user message")
|
|
|
|
userMessage = interfaceDbChat.createMessage(userMessageData)
|
|
logger.info(f"Stored user message: {userMessage.id} with {len(user_documents)} document(s)")
|
|
|
|
# Emit message event for streaming (exact chatData format)
|
|
event_manager = get_event_manager()
|
|
message_timestamp = parseTimestamp(userMessage.publishedAt, default=getUtcTimestamp())
|
|
await event_manager.emit_event(
|
|
workflow.id,
|
|
"chatdata",
|
|
"New message",
|
|
"message",
|
|
{
|
|
"type": "message",
|
|
"createdAt": message_timestamp,
|
|
"item": userMessage.dict()
|
|
}
|
|
)
|
|
|
|
# Update workflow status
|
|
interfaceDbChat.updateWorkflow(workflow.id, {
|
|
"status": "running",
|
|
"lastActivity": getUtcTimestamp()
|
|
})
|
|
|
|
# Process in background (async)
|
|
asyncio.create_task(_processChatbotMessage(
|
|
services,
|
|
workflow.id,
|
|
userInput,
|
|
userMessage.id
|
|
))
|
|
|
|
# Reload workflow to include new message
|
|
workflow = interfaceDbChat.getWorkflow(workflow.id)
|
|
return workflow
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error in chatProcess: {str(e)}", exc_info=True)
|
|
raise
|
|
|
|
|
|
async def _check_workflow_status(interfaceDbChat, workflowId: str, event_manager) -> bool:
|
|
"""
|
|
Check if workflow is stopped. If stopped, emit stopped event and return True.
|
|
|
|
Returns:
|
|
True if workflow is stopped, False otherwise
|
|
"""
|
|
workflow = interfaceDbChat.getWorkflow(workflowId)
|
|
if workflow and workflow.status == "stopped":
|
|
await event_manager.emit_event(
|
|
workflowId,
|
|
"stopped",
|
|
"Workflow stopped",
|
|
"stopped"
|
|
)
|
|
logger.info(f"Workflow {workflowId} was stopped, exiting processing")
|
|
return True
|
|
return False
|
|
|
|
|
|
async def _convert_file_ids_to_document_references(
|
|
services,
|
|
file_ids: List[str]
|
|
) -> DocumentReferenceList:
|
|
"""
|
|
Convert file IDs to DocumentReferenceList for use with ai.process.
|
|
|
|
Args:
|
|
services: Services instance
|
|
file_ids: List of file IDs to convert
|
|
|
|
Returns:
|
|
DocumentReferenceList with docItem references
|
|
"""
|
|
references = []
|
|
|
|
# Get workflow to search for ChatDocuments
|
|
workflow = services.workflow
|
|
if not workflow:
|
|
logger.error("Cannot convert file IDs to document references: workflow not set in services")
|
|
return DocumentReferenceList(references=[])
|
|
|
|
for file_id in file_ids:
|
|
try:
|
|
# Get file info to verify it exists
|
|
file_info = services.chat.getFileInfo(file_id)
|
|
if not file_info:
|
|
logger.warning(f"File {file_id} not found, skipping")
|
|
continue
|
|
|
|
# Find ChatDocument that has this fileId
|
|
document_id = None
|
|
if workflow.messages:
|
|
for message in workflow.messages:
|
|
if hasattr(message, 'documents') and message.documents:
|
|
for doc in message.documents:
|
|
if getattr(doc, 'fileId', None) == file_id:
|
|
document_id = getattr(doc, 'id', None)
|
|
break
|
|
if document_id:
|
|
break
|
|
|
|
# Search database if not found in messages
|
|
if not document_id:
|
|
try:
|
|
from modules.datamodels.datamodelChat import ChatDocument
|
|
from modules.shared.databaseUtils import getRecordsetWithRBAC
|
|
documents = getRecordsetWithRBAC(
|
|
services.interfaceDbChat.db,
|
|
ChatDocument,
|
|
services.currentUser,
|
|
recordFilter={"fileId": file_id}
|
|
)
|
|
if documents:
|
|
workflow_message_ids = {msg.id for msg in workflow.messages} if workflow.messages else set()
|
|
for doc in documents:
|
|
if doc.get("messageId") in workflow_message_ids:
|
|
document_id = doc.get("id")
|
|
break
|
|
except Exception:
|
|
pass # Fallback to fileId
|
|
|
|
# Use ChatDocument ID if found, otherwise use fileId as fallback
|
|
ref = DocumentItemReference(documentId=document_id if document_id else file_id)
|
|
references.append(ref)
|
|
except Exception as e:
|
|
logger.error(f"Error converting fileId {file_id}: {e}", exc_info=True)
|
|
|
|
logger.info(f"Converted {len(references)} file IDs to document references")
|
|
return DocumentReferenceList(references=references)
|
|
|
|
|
|
def _format_query_results_as_lookup(query_data: Dict[str, List[Dict]]) -> str:
|
|
"""
|
|
Format database query results as JSON lookup table for Excel matching.
|
|
Converts query result data into structured JSON format: {Artikelnummer: {columns...}}
|
|
|
|
Args:
|
|
query_data: Dict with query_key -> list of row dicts (from connector with return_json=True)
|
|
|
|
Returns:
|
|
JSON string formatted as lookup table
|
|
"""
|
|
lookup_table = {}
|
|
|
|
for query_key, rows in query_data.items():
|
|
if query_key == "error" or not rows:
|
|
logger.warning(f"Skipping query key '{query_key}' - no rows or error")
|
|
continue
|
|
|
|
logger.info(f"Processing {len(rows)} rows from query '{query_key}'")
|
|
|
|
for row in rows:
|
|
if not isinstance(row, dict):
|
|
logger.warning(f"Skipping non-dict row: {type(row)}")
|
|
continue
|
|
|
|
# Find Artikelnummer field (case-insensitive)
|
|
artikelnummer = None
|
|
for key in row.keys():
|
|
if key.lower() in ['artikelnummer', 'artikel_nummer', 'art_nr', 'part_number']:
|
|
artikelnummer = str(row[key])
|
|
break
|
|
|
|
if artikelnummer:
|
|
lookup_table[artikelnummer] = row
|
|
else:
|
|
logger.warning(f"No Artikelnummer found in row with keys: {list(row.keys())}")
|
|
|
|
logger.info(f"Generated lookup table with {len(lookup_table)} entries")
|
|
if lookup_table:
|
|
sample_keys = list(lookup_table.keys())[:3]
|
|
logger.info(f"Sample Artikelnummern: {sample_keys}")
|
|
if sample_keys:
|
|
sample_entry = lookup_table[sample_keys[0]]
|
|
logger.info(f"Sample entry keys: {list(sample_entry.keys())}")
|
|
|
|
return json.dumps(lookup_table, ensure_ascii=False, indent=2)
|
|
|
|
|
|
async def _create_chat_document_from_action_document(
|
|
services,
|
|
action_document,
|
|
message_id: str,
|
|
workflow_id: str,
|
|
round_number: int
|
|
) -> ChatDocument:
|
|
"""
|
|
Create a ChatDocument from an ActionDocument by storing the file data.
|
|
|
|
Args:
|
|
services: Services instance
|
|
action_document: ActionDocument from ai.process result
|
|
message_id: ID of the message to attach to
|
|
workflow_id: Workflow ID
|
|
round_number: Round number
|
|
|
|
Returns:
|
|
ChatDocument instance
|
|
"""
|
|
try:
|
|
# Get file data (could be bytes or string)
|
|
document_data = action_document.documentData
|
|
|
|
# Convert to bytes if needed
|
|
if isinstance(document_data, str):
|
|
# Check if it's base64 encoded
|
|
try:
|
|
# Try to decode as base64 first
|
|
file_bytes = base64.b64decode(document_data)
|
|
except Exception:
|
|
# Not base64, encode as UTF-8
|
|
file_bytes = document_data.encode('utf-8')
|
|
elif isinstance(document_data, bytes):
|
|
file_bytes = document_data
|
|
else:
|
|
# Try to convert to bytes
|
|
try:
|
|
file_bytes = bytes(document_data)
|
|
except Exception:
|
|
# Last resort: convert to string then encode
|
|
file_bytes = str(document_data).encode('utf-8')
|
|
|
|
# Get MIME type (default to Excel)
|
|
mime_type = action_document.mimeType or "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
|
|
|
# Get file name
|
|
file_name = action_document.documentName or "data_export.xlsx"
|
|
# Ensure it has .xlsx extension
|
|
if not file_name.lower().endswith('.xlsx'):
|
|
# Remove any existing extension and add .xlsx
|
|
file_name = file_name.rsplit('.', 1)[0] + '.xlsx'
|
|
|
|
# Store file using component interface
|
|
file_item = services.interfaceDbComponent.createFile(
|
|
name=file_name,
|
|
mimeType=mime_type,
|
|
content=file_bytes
|
|
)
|
|
|
|
# Store file data
|
|
success = services.interfaceDbComponent.createFileData(file_item.id, file_bytes)
|
|
if not success:
|
|
logger.warning(f"Failed to store file data for {file_item.id}, but continuing...")
|
|
|
|
# Create ChatDocument
|
|
chat_document = ChatDocument(
|
|
id=str(uuid.uuid4()),
|
|
messageId=message_id,
|
|
fileId=file_item.id,
|
|
fileName=file_name,
|
|
fileSize=len(file_bytes),
|
|
mimeType=mime_type,
|
|
roundNumber=round_number,
|
|
taskNumber=0,
|
|
actionNumber=0
|
|
)
|
|
|
|
logger.info(f"Created ChatDocument {chat_document.id} from ActionDocument {file_name} (size: {len(file_bytes)} bytes)")
|
|
return chat_document
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error creating ChatDocument from ActionDocument: {e}", exc_info=True)
|
|
raise
|
|
|
|
|
|
async def _processChatbotMessage(
|
|
services,
|
|
workflowId: str,
|
|
userInput: UserInputRequest,
|
|
userMessageId: str
|
|
):
|
|
"""
|
|
Process chatbot message in background.
|
|
Simplified 4-step flow: Analysis → Query → Excel → Answer
|
|
"""
|
|
event_manager = get_event_manager()
|
|
|
|
try:
|
|
interfaceDbChat = services.interfaceDbChat
|
|
|
|
# Reload workflow to get current messages
|
|
workflow = interfaceDbChat.getWorkflow(workflowId)
|
|
if not workflow:
|
|
logger.error(f"Workflow {workflowId} not found during processing")
|
|
await event_manager.emit_event(workflowId, "error", f"Workflow {workflowId} nicht gefunden", "error")
|
|
return
|
|
|
|
if await _check_workflow_status(interfaceDbChat, workflowId, event_manager):
|
|
return
|
|
|
|
# Build conversation context from history
|
|
context = ""
|
|
if workflow.messages:
|
|
recent_messages = workflow.messages[-5:]
|
|
context = "\n\nPrevious conversation:\n"
|
|
for msg in recent_messages:
|
|
if msg.role == "user":
|
|
context += f"User: {msg.message}\n"
|
|
elif msg.role == "assistant":
|
|
context += f"Assistant: {msg.message}\n"
|
|
|
|
await services.ai.ensureAiObjectsInitialized()
|
|
current_date = datetime.datetime.now().strftime("%d.%m.%Y")
|
|
|
|
if await _check_workflow_status(interfaceDbChat, workflowId, event_manager):
|
|
return
|
|
|
|
# Step 1: Unified Analysis
|
|
logger.info("Step 1: Analyzing user input and attached files...")
|
|
await event_manager.emit_event(workflowId, "status", "Analysiere Benutzeranfrage und angehängte Dateien...", "analysis")
|
|
|
|
# Prepare document references if files are attached
|
|
has_uploaded_files = bool(userInput.listFileId and len(userInput.listFileId) > 0)
|
|
document_references = None
|
|
|
|
if has_uploaded_files:
|
|
workflow = interfaceDbChat.getWorkflow(workflowId)
|
|
if workflow:
|
|
services.workflow = workflow
|
|
document_references = await _convert_file_ids_to_document_references(services, userInput.listFileId)
|
|
logger.info(f"Prepared {len(document_references.references)} document references for analysis")
|
|
|
|
# Build analysis prompt
|
|
file_context = "\n\nIMPORTANT - ATTACHED FILES:\nIf files are attached, read them completely. For Excel files, extract ALL article identifiers (Artikelnummer) from the file and include them directly in the SQL query using WHERE Artikelnummer IN ('ART1', 'ART2', ...). DO NOT reference ExcelFile table - it doesn't exist!\n" if has_uploaded_files else ""
|
|
|
|
analysisPrompt = f"""Heute ist der {current_date}.
|
|
|
|
Du bist ein Chatbot der Althaus AG mit Zugriff auf eine SQL-Datenbank.
|
|
|
|
DATENBANK-SCHEMA:
|
|
- Artikel: I_ID, Artikelnummer, Artikelbezeichnung, Lieferant, etc.
|
|
- Einkaufspreis: m_Artikel, EP_CHF
|
|
- Lagerplatz_Artikel: R_ARTIKEL, S_IST_BESTAND, etc.
|
|
- Beziehungen: Artikel.I_ID = Einkaufspreis.m_Artikel = Lagerplatz_Artikel.R_ARTIKEL
|
|
|
|
SQL-HINWEISE:
|
|
- IMMER doppelte Anführungszeichen für Spaltennamen: "Artikelnummer", "S_IST_BESTAND"
|
|
- S_IST_BESTAND kann "Unbekannt" sein (TEXT), prüfe mit WHERE l."S_IST_BESTAND" != 'Unbekannt'
|
|
- Verwende Tabellenaliase: a für Artikel, e für Einkaufspreis, l für Lagerplatz_Artikel
|
|
- WICHTIG: Wenn Excel-Dateien angehängt sind, extrahiere Artikelnummern aus der Datei und verwende sie direkt im SQL: WHERE a."Artikelnummer" IN ('ART1', 'ART2', 'ART3')
|
|
- VERBOTEN: Verwende NICHT "SELECT ... FROM ExcelFile" - diese Tabelle existiert nicht!
|
|
|
|
{file_context}User question: {userInput.prompt}{context}
|
|
|
|
Return ONLY valid JSON:
|
|
{{
|
|
"needsDatabaseQuery": boolean,
|
|
"needsWebResearch": boolean,
|
|
"needsExcelFile": boolean,
|
|
"excelAction": "create" | "update" | null,
|
|
"excelFileName": string | null,
|
|
"sqlQuery": string (ready-to-execute SQL with double quotes for column names. If Excel file attached, extract Artikelnummern and use WHERE a."Artikelnummer" IN ('ART1', 'ART2', ...)),
|
|
"requestedColumns": array of strings (columns to add for Excel updates, e.g. ["Lieferant", "Lagerbestand"]),
|
|
"reasoning": string
|
|
}}"""
|
|
|
|
# Single AI call for analysis
|
|
method_ai = MethodAi(services)
|
|
analysis_result = await method_ai.process({
|
|
"aiPrompt": analysisPrompt,
|
|
"documentList": document_references if has_uploaded_files else None,
|
|
"resultType": "json",
|
|
"simpleMode": True
|
|
})
|
|
|
|
# Extract content from ActionResult (in simple mode, content is in first document's documentData)
|
|
analysis_content = None
|
|
if analysis_result.success and analysis_result.documents:
|
|
analysis_content = analysis_result.documents[0].documentData
|
|
if isinstance(analysis_content, bytes):
|
|
analysis_content = analysis_content.decode('utf-8')
|
|
|
|
if not analysis_content:
|
|
logger.warning("Analysis failed, using fallback")
|
|
analysis = {}
|
|
else:
|
|
analysis = _extractJsonFromResponse(analysis_content)
|
|
|
|
# Extract analysis results
|
|
needsDatabaseQuery = analysis.get("needsDatabaseQuery", False) if analysis else False
|
|
needsWebResearch = analysis.get("needsWebResearch", False) if analysis else False
|
|
needsExcelFile = analysis.get("needsExcelFile", False) if analysis else False
|
|
excelAction = analysis.get("excelAction")
|
|
excelFileName = analysis.get("excelFileName")
|
|
sql_query = analysis.get("sqlQuery", "")
|
|
requested_columns = analysis.get("requestedColumns", [])
|
|
|
|
# Ensure database query if Excel update is needed
|
|
if needsExcelFile and excelAction == "update" and not needsDatabaseQuery:
|
|
needsDatabaseQuery = True
|
|
|
|
logger.info(f"Analysis: DB={needsDatabaseQuery}, Excel={needsExcelFile}, SQL={'present' if sql_query else 'missing'}")
|
|
|
|
if await _check_workflow_status(interfaceDbChat, workflowId, event_manager):
|
|
return
|
|
|
|
# Step 2: Execute SQL Query
|
|
queryResults = {}
|
|
queryData = {}
|
|
excel_documents = []
|
|
|
|
if needsDatabaseQuery and sql_query:
|
|
logger.info("Step 2: Executing SQL query...")
|
|
await event_manager.emit_event(workflowId, "status", "Führe Datenbankabfrage aus...", "query_execution")
|
|
|
|
try:
|
|
connector = PreprocessorConnector()
|
|
result_dict = await connector.executeQuery(sql_query, return_json=True)
|
|
await connector.close()
|
|
|
|
if result_dict and not result_dict.get("text", "").startswith(("Error:", "Query failed:")):
|
|
queryResults["main"] = result_dict.get("text", "")
|
|
queryData["main"] = result_dict.get("data", [])
|
|
logger.info(f"Query executed successfully, returned {len(queryData.get('main', []))} rows")
|
|
else:
|
|
error_text = result_dict.get("text", "Query failed") if result_dict else "Query failed: No response"
|
|
queryResults["error"] = error_text
|
|
logger.error(f"Query failed: {error_text}")
|
|
|
|
# If query failed and we need Excel update, try to extract article numbers from Excel and retry
|
|
if needsExcelFile and excelAction == "update" and has_uploaded_files and "ExcelFile" in error_text:
|
|
logger.warning("Query failed due to ExcelFile table reference. Attempting to extract article numbers from Excel file and retry...")
|
|
# The AI should have extracted article numbers, but if query failed, we can't proceed
|
|
# Log the issue for debugging
|
|
logger.error("Cannot proceed with Excel update: SQL query references non-existent ExcelFile table. AI should extract article numbers directly.")
|
|
except Exception as e:
|
|
logger.error(f"Error executing query: {e}")
|
|
queryResults["error"] = f"Error executing query: {str(e)}"
|
|
|
|
# Step 2.5: Web Research (simplified)
|
|
webResearchResults = ""
|
|
if needsWebResearch:
|
|
logger.info("Performing web research...")
|
|
try:
|
|
researchResult = await services.web.performWebResearch(
|
|
prompt=userInput.prompt,
|
|
urls=[],
|
|
country=None,
|
|
language=userInput.userLanguage or "de",
|
|
researchDepth="general",
|
|
operationId=None
|
|
)
|
|
webResearchResults = json.dumps(researchResult, ensure_ascii=False, indent=2) if isinstance(researchResult, dict) else str(researchResult)
|
|
except Exception as e:
|
|
logger.error(f"Web research failed: {e}")
|
|
webResearchResults = f"Web research error: {str(e)}"
|
|
|
|
if await _check_workflow_status(interfaceDbChat, workflowId, event_manager):
|
|
return
|
|
|
|
# Build answer context
|
|
answerContext = f"User question: {userInput.prompt}{context}\n\n"
|
|
if queryResults:
|
|
for key, result in queryResults.items():
|
|
if key != "error":
|
|
answerContext += f"Database results:\n{result}\n\n"
|
|
if webResearchResults:
|
|
answerContext += f"Web research:\n{webResearchResults}\n\n"
|
|
|
|
# Step 3: Process Excel File
|
|
excel_error = None
|
|
if needsExcelFile:
|
|
logger.info(f"Step 3: Processing Excel file (action: {excelAction})...")
|
|
await event_manager.emit_event(
|
|
workflowId,
|
|
"status",
|
|
f"Erstelle Excel-Datei..." if excelAction == "create" else "Aktualisiere Excel-Datei...",
|
|
"excel_processing"
|
|
)
|
|
|
|
try:
|
|
# Check if we have query data for updates
|
|
if excelAction == "update":
|
|
if not queryData or "main" not in queryData or not queryData["main"]:
|
|
error_msg = f"No query data available for Excel update. queryData keys: {list(queryData.keys()) if queryData else 'None'}"
|
|
if queryResults.get("error"):
|
|
error_msg += f" Query error: {queryResults['error']}"
|
|
logger.error(error_msg)
|
|
excel_error = f"Die Datenbankabfrage ist fehlgeschlagen: {queryResults.get('error', 'Unbekannter Fehler')}. Die Excel-Datei konnte nicht aktualisiert werden."
|
|
raise ValueError(excel_error)
|
|
|
|
workflow = interfaceDbChat.getWorkflow(workflowId)
|
|
if workflow:
|
|
services.workflow = workflow
|
|
|
|
method_ai = MethodAi(services)
|
|
|
|
# Prepare document references for updates
|
|
if excelAction == "update" and userInput.listFileId:
|
|
if not document_references or not document_references.references:
|
|
document_references = await _convert_file_ids_to_document_references(services, userInput.listFileId)
|
|
else:
|
|
from modules.datamodels.datamodelDocref import DocumentReferenceList
|
|
document_references = DocumentReferenceList(references=[])
|
|
|
|
# Build Excel prompt
|
|
if excelAction == "update":
|
|
# Format query data as JSON lookup table
|
|
lookup_table_json = _format_query_results_as_lookup(queryData)
|
|
lookup_table_dict = json.loads(lookup_table_json)
|
|
logger.info(f"Generated lookup table with {len(lookup_table_dict)} entries")
|
|
|
|
# Log sample entries for debugging
|
|
if lookup_table_dict:
|
|
sample_keys = list(lookup_table_dict.keys())[:3]
|
|
logger.info(f"Sample Artikelnummern in lookup table: {sample_keys}")
|
|
if sample_keys:
|
|
sample_entry = lookup_table_dict[sample_keys[0]]
|
|
logger.info(f"Sample entry for '{sample_keys[0]}': {list(sample_entry.keys())}")
|
|
logger.info(f"Sample entry values: {sample_entry}")
|
|
|
|
# Determine columns to add
|
|
column_names = requested_columns if requested_columns else []
|
|
if not column_names:
|
|
prompt_lower = userInput.prompt.lower()
|
|
if "einkaufspreis" in prompt_lower or "purchase price" in prompt_lower or "kaufpreis" in prompt_lower:
|
|
column_names = ["Einkaufspreis"]
|
|
elif "lieferant" in prompt_lower or "supplier" in prompt_lower:
|
|
column_names = ["Lieferant"]
|
|
elif "lagerbestand" in prompt_lower or "stock" in prompt_lower:
|
|
column_names = ["Lagerbestand"]
|
|
else:
|
|
# Default: try to infer from available columns in lookup table
|
|
if lookup_table_dict:
|
|
sample_entry = next(iter(lookup_table_dict.values()))
|
|
available_columns = list(sample_entry.keys())
|
|
logger.info(f"Available columns in lookup table: {available_columns}")
|
|
# Prefer common columns
|
|
if "Einkaufspreis" in available_columns:
|
|
column_names = ["Einkaufspreis"]
|
|
elif "Lieferant" in available_columns:
|
|
column_names = ["Lieferant"]
|
|
else:
|
|
column_names = available_columns[:1] if available_columns else ["Lieferant"]
|
|
else:
|
|
column_names = ["Lieferant"]
|
|
|
|
logger.info(f"Adding columns to Excel: {column_names}")
|
|
column_name_str = ", ".join(column_names)
|
|
|
|
# Build a more explicit prompt with clear instructions and the lookup table
|
|
# Include the lookup table data directly in the prompt so it's always available
|
|
excel_prompt = f"""You are updating an Excel file. CRITICAL TASK:
|
|
|
|
Add new column(s) "{column_name_str}" to the attached Excel file by matching Artikelnummer values.
|
|
|
|
STEP-BY-STEP PROCESS (MUST FOLLOW EXACTLY):
|
|
1. Read the attached Excel file completely - all sheets, all rows, all columns.
|
|
2. Find the column containing "Artikelnummer" (may be named "Artikelnummer", "Artikel Nummer", "Art-Nr", "Artikelnummer", etc.)
|
|
3. For EACH ROW in the Excel file:
|
|
a. Read the Artikelnummer value from that row (e.g., "LED001", "12345", etc.)
|
|
b. Look up that EXACT Artikelnummer in the DATABASE LOOKUP TABLE below (case-sensitive match)
|
|
c. If the Artikelnummer is found in the lookup table:
|
|
- Extract the value(s) for column(s) "{column_name_str}" from the matching entry
|
|
- Write the value(s) to the new column(s) in that row
|
|
d. If the Artikelnummer is NOT found in the lookup table:
|
|
- Leave the new column(s) empty for that row
|
|
4. Preserve ALL existing data, formatting, formulas, and structure exactly as-is.
|
|
5. Return ONLY the complete modified Excel file (.xlsx format). Do NOT include any text or explanations.
|
|
|
|
=== DATABASE LOOKUP TABLE (JSON) ===
|
|
Use this table to match Artikelnummer and extract values. Format: {{"Artikelnummer": {{"column": "value", ...}}}}
|
|
|
|
{lookup_table_json}
|
|
|
|
=== EXAMPLE MATCHING ===
|
|
- Excel row has Artikelnummer "LED001" in column "Artikelnummer"
|
|
- Search for "LED001" in the lookup table above
|
|
- Find entry: {{"LED001": {{"Lieferant": "Phoenix Contact", "Einkaufspreis": 12.50, ...}}}}
|
|
- Extract "{column_name_str}" value from that entry
|
|
- Write the extracted value to the new "{column_name_str}" column in that Excel row
|
|
|
|
CRITICAL REQUIREMENTS:
|
|
- Match EXACTLY by Artikelnummer (case-sensitive, exact string match)
|
|
- Preserve ALL existing columns, rows, formatting, and formulas
|
|
- Only add the new column(s), do NOT modify existing data
|
|
- Return ONLY the Excel file binary data, no explanations"""
|
|
else:
|
|
excel_prompt = f"""Create Excel file (.xlsx) from database results:
|
|
|
|
{answerContext if queryResults else "No database results available."}
|
|
|
|
User request: {userInput.prompt}
|
|
|
|
Create well-structured spreadsheet with appropriate headers and formatting."""
|
|
|
|
# Use simpleMode=True for Excel updates to ensure lookup table is directly accessible
|
|
# simpleMode=False goes through complex document generation pipeline which may lose context
|
|
excel_result = await method_ai.process({
|
|
"aiPrompt": excel_prompt,
|
|
"documentList": document_references if excelAction == "update" else None,
|
|
"resultType": "xlsx",
|
|
"simpleMode": True # Use simple mode for direct Excel updates with lookup table
|
|
})
|
|
|
|
if excel_result.success and excel_result.documents:
|
|
for action_doc in excel_result.documents:
|
|
try:
|
|
chat_doc = await _create_chat_document_from_action_document(
|
|
services, action_doc, "", workflowId, workflow.currentRound
|
|
)
|
|
if chat_doc:
|
|
excel_documents.append(chat_doc)
|
|
except Exception as e:
|
|
logger.error(f"Error creating ChatDocument: {e}", exc_info=True)
|
|
logger.info(f"Excel processing complete: {len(excel_documents)} documents")
|
|
else:
|
|
excel_error = f"Excel-Verarbeitung fehlgeschlagen: {excel_result.error if excel_result else 'Unbekannter Fehler'}"
|
|
logger.warning(excel_error)
|
|
except Exception as e:
|
|
excel_error = str(e) if not excel_error else excel_error
|
|
logger.error(f"Error processing Excel: {e}", exc_info=True)
|
|
|
|
if await _check_workflow_status(interfaceDbChat, workflowId, event_manager):
|
|
return
|
|
|
|
# Step 4: Generate Final Answer
|
|
logger.info("Step 4: Generating final answer...")
|
|
await event_manager.emit_event(workflowId, "status", "Formuliere finale Antwort...", "answer_generation")
|
|
|
|
excel_context = ""
|
|
if needsExcelFile:
|
|
if excel_documents:
|
|
file_names = [doc.fileName for doc in excel_documents]
|
|
action_text = "erstellt" if excelAction == "create" else "aktualisiert"
|
|
file_name_str = file_names[0] if len(file_names) == 1 else f"{len(file_names)} Dateien"
|
|
excel_context = f"\n\nWICHTIG - EXCEL-DATEI {action_text.upper()}:\nDie Excel-Datei '{file_name_str}' wurde erfolgreich {action_text} und ist als Anhang verfügbar. Erwähne dies am Ende deiner Antwort."
|
|
elif excel_error:
|
|
excel_context = f"\n\nFEHLER - EXCEL-VERARBEITUNG:\n{excel_error}\nErkläre dem Nutzer, warum die Excel-Datei nicht aktualisiert werden konnte."
|
|
|
|
# Build answer context parts (avoid backslashes in f-string expressions)
|
|
db_results_part = f"\nDATENBANK-ERGEBNISSE:\n{answerContext}" if queryResults and not queryResults.get("error") else ""
|
|
if queryResults.get("error"):
|
|
db_results_part = f"\nDATENBANK-FEHLER:\n{queryResults['error']}"
|
|
web_results_part = f"\nINTERNET-RECHERCHE:\n{webResearchResults}" if webResearchResults else ""
|
|
|
|
answerPrompt = f"""Heute ist der {current_date}.
|
|
|
|
Du bist ein Chatbot der Althaus AG. Antworte auf Deutsch (kein ß, immer ss).
|
|
|
|
Antworte auf: {userInput.prompt}{context}
|
|
{db_results_part}{web_results_part}{excel_context}
|
|
|
|
WICHTIG:
|
|
- Klare, strukturierte Antwort
|
|
- Zahlen mit Tausender-Trennzeichen (7'411)
|
|
- Markdown-Tabellen für Daten (max 20 Zeilen)
|
|
- Artikelnummern als Link: [ARTIKELNUMMER](/details/ARTIKELNUMMER)
|
|
- Wenn Excel-Datei erstellt wurde, bestätige dies explizit am Ende
|
|
- Wenn Excel-Verarbeitung fehlgeschlagen ist, erkläre dem Nutzer den Fehler klar"""
|
|
|
|
answerRequest = AiCallRequest(
|
|
prompt=answerPrompt,
|
|
context=answerContext if (queryResults or webResearchResults) else None,
|
|
options=AiCallOptions(
|
|
resultFormat="txt",
|
|
operationType=OperationTypeEnum.DATA_ANALYSE,
|
|
processingMode=ProcessingModeEnum.BASIC
|
|
)
|
|
)
|
|
|
|
answerResponse = await services.ai.callAi(answerRequest)
|
|
finalAnswer = answerResponse.content
|
|
|
|
logger.info("Final answer generated")
|
|
|
|
if await _check_workflow_status(interfaceDbChat, workflowId, event_manager):
|
|
return
|
|
|
|
# Step 5: Store assistant message
|
|
# Reload workflow to get updated message count
|
|
workflow = interfaceDbChat.getWorkflow(workflowId)
|
|
|
|
# Prepare message data with Excel documents if any
|
|
message_id = f"msg_{uuid.uuid4()}"
|
|
assistantMessageData = {
|
|
"id": message_id,
|
|
"workflowId": workflowId,
|
|
"parentMessageId": userMessageId,
|
|
"message": finalAnswer,
|
|
"role": "assistant",
|
|
"status": "last",
|
|
"sequenceNr": len(workflow.messages) + 1,
|
|
"publishedAt": getUtcTimestamp(),
|
|
"success": True,
|
|
"roundNumber": workflow.currentRound,
|
|
"taskNumber": 0,
|
|
"actionNumber": 0
|
|
}
|
|
|
|
# Add Excel documents if any were created
|
|
if excel_documents:
|
|
# Update messageId in all Excel documents
|
|
for excel_doc in excel_documents:
|
|
excel_doc.messageId = message_id
|
|
|
|
# Convert ChatDocuments to dict format
|
|
try:
|
|
documents_dict = [doc.model_dump() for doc in excel_documents]
|
|
except Exception:
|
|
documents_dict = [doc.dict() for doc in excel_documents]
|
|
|
|
assistantMessageData["documents"] = documents_dict
|
|
assistantMessageData["documentsLabel"] = "Excel Files"
|
|
|
|
assistantMessage = interfaceDbChat.createMessage(assistantMessageData)
|
|
logger.info(f"Stored assistant message: {assistantMessage.id}")
|
|
|
|
# Emit message event for streaming (exact chatData format)
|
|
message_timestamp = parseTimestamp(assistantMessage.publishedAt, default=getUtcTimestamp())
|
|
await event_manager.emit_event(
|
|
workflowId,
|
|
"chatdata",
|
|
"New message",
|
|
"message",
|
|
{
|
|
"type": "message",
|
|
"createdAt": message_timestamp,
|
|
"item": assistantMessage.dict()
|
|
}
|
|
)
|
|
|
|
# Update workflow status to completed
|
|
interfaceDbChat.updateWorkflow(workflowId, {
|
|
"status": "completed",
|
|
"lastActivity": getUtcTimestamp()
|
|
})
|
|
|
|
logger.info(f"Chatbot processing completed for workflow {workflowId}")
|
|
|
|
# Emit completion event
|
|
await event_manager.emit_event(
|
|
workflowId,
|
|
"complete",
|
|
"Chatbot-Verarbeitung abgeschlossen",
|
|
"complete",
|
|
{"workflowId": workflowId}
|
|
)
|
|
|
|
# Schedule cleanup
|
|
await event_manager.cleanup(workflowId)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error processing chatbot message: {str(e)}", exc_info=True)
|
|
|
|
# Store error message
|
|
try:
|
|
# Reload workflow to get current message count
|
|
workflow = interfaceDbChat.getWorkflow(workflowId)
|
|
|
|
errorMessageData = {
|
|
"id": f"msg_{uuid.uuid4()}",
|
|
"workflowId": workflowId,
|
|
"parentMessageId": userMessageId,
|
|
"message": f"Sorry, I encountered an error: {str(e)}",
|
|
"role": "assistant",
|
|
"status": "last",
|
|
"sequenceNr": len(workflow.messages) + 1,
|
|
"publishedAt": getUtcTimestamp(),
|
|
"success": False,
|
|
"roundNumber": workflow.currentRound if workflow else 1,
|
|
"taskNumber": 0,
|
|
"actionNumber": 0
|
|
}
|
|
errorMessage = interfaceDbChat.createMessage(errorMessageData)
|
|
|
|
# Emit message event for streaming (exact chatData format)
|
|
message_timestamp = parseTimestamp(errorMessage.publishedAt, default=getUtcTimestamp())
|
|
await event_manager.emit_event(
|
|
workflowId,
|
|
"chatdata",
|
|
"New message",
|
|
"message",
|
|
{
|
|
"type": "message",
|
|
"createdAt": message_timestamp,
|
|
"item": errorMessage.dict()
|
|
}
|
|
)
|
|
|
|
# Update workflow status to error
|
|
interfaceDbChat.updateWorkflow(workflowId, {
|
|
"status": "error",
|
|
"lastActivity": getUtcTimestamp()
|
|
})
|
|
|
|
# Schedule cleanup
|
|
await event_manager.cleanup(workflowId)
|
|
except Exception as storeError:
|
|
logger.error(f"Error storing error message: {storeError}")
|
|
|