Integrated neutralizer to MVP

This commit is contained in:
ValueOn AG 2025-11-04 15:50:16 +01:00
parent 2255c9009d
commit e11ab4ebc5
4 changed files with 314 additions and 122 deletions

View file

@ -1230,9 +1230,12 @@ class ChatObjects:
allAutomations = self.db.getRecordset(AutomationDefinition)
filteredAutomations = self._uam(AutomationDefinition, allAutomations)
# Compute status for each automation
# Compute status for each automation and normalize executionLogs
for automation in filteredAutomations:
automation["status"] = self._computeAutomationStatus(automation)
# Ensure executionLogs is always a list, not None
if automation.get("executionLogs") is None:
automation["executionLogs"] = []
# If no pagination requested, return all items
if pagination is None:
@ -1272,6 +1275,9 @@ class ChatObjects:
automation = filtered[0]
automation["status"] = self._computeAutomationStatus(automation)
# Ensure executionLogs is always a list, not None
if automation.get("executionLogs") is None:
automation["executionLogs"] = []
return automation
except Exception as e:
logger.error(f"Error getting automation definition: {str(e)}")
@ -1306,6 +1312,9 @@ class ChatObjects:
# Compute status
createdAutomation["status"] = self._computeAutomationStatus(createdAutomation)
# Ensure executionLogs is always a list, not None
if createdAutomation.get("executionLogs") is None:
createdAutomation["executionLogs"] = []
# Trigger sync (async, don't wait)
asyncio.create_task(self.syncAutomationEvents())
@ -1334,6 +1343,9 @@ class ChatObjects:
# Compute status
updatedAutomation["status"] = self._computeAutomationStatus(updatedAutomation)
# Ensure executionLogs is always a list, not None
if updatedAutomation.get("executionLogs") is None:
updatedAutomation["executionLogs"] = []
# Trigger sync (async, don't wait)
asyncio.create_task(self.syncAutomationEvents())

View file

@ -34,6 +34,7 @@ class NeutralizationService:
"""
self.services = serviceCenter
self.interfaceDbApp = serviceCenter.interfaceDbApp
self.interfaceDbComponent = serviceCenter.interfaceDbComponent
# Initialize anonymization processors
self.NamesToParse = NamesToParse or []
@ -61,19 +62,19 @@ class NeutralizationService:
return self._neutralizeText(text, 'text')
def processFile(self, fileId: str) -> Dict[str, Any]:
"""Neutralize a file referenced by its fileId using app interface."""
if not self.interfaceDbApp:
raise ValueError("User context is required to process a file by fileId")
"""Neutralize a file referenced by its fileId using component interface."""
if not self.interfaceDbComponent:
raise ValueError("Component interface is required to process a file by fileId")
# Fetch file data and metadata
fileInfo = None
try:
# getFile returns an object; fallback to dict-like
fileInfo = self.interfaceDbApp.getFile(fileId)
fileInfo = self.interfaceDbComponent.getFile(fileId)
except Exception:
fileInfo = None
fileName = getattr(fileInfo, 'fileName', None) if fileInfo else None
mimeType = getattr(fileInfo, 'mimeType', None) if fileInfo else None
fileData = self.interfaceDbApp.getFileData(fileId)
fileData = self.interfaceDbComponent.getFileData(fileId)
if not fileData:
raise ValueError(f"No file data found for fileId: {fileId}")

View file

@ -1337,7 +1337,7 @@ Return JSON:
draft_id = draft_data.get("id", "Unknown")
# Create draft result data with full draft information
draft_result_data = {
draftResultData = {
"status": "draft",
"message": "Email draft created successfully with AI-generated content",
"draftId": draft_id,
@ -1361,7 +1361,7 @@ Return JSON:
success=True,
documents=[ActionDocument(
documentName=f"ai_generated_email_draft_{self._format_timestamp_for_filename()}.json",
documentData=json.dumps(draft_result_data, indent=2),
documentData=json.dumps(draftResultData, indent=2),
mimeType="application/json"
)]
)
@ -1381,47 +1381,27 @@ Return JSON:
async def sendDraftEmail(self, parameters: Dict[str, Any]) -> ActionResult:
"""
GENERAL:
- Purpose: Send a draft email using the draft email JSON data from action outlook.composeAndDraftEmailWithContext. This action is used to send the email after the email has been composed and drafted.
- Input requirements: connectionReference (required); draftEmailJson (required).
- Output format: JSON confirmation with sent mail metadata.
- Purpose: Send draft email(s) using draft email JSON document(s) from action outlook.composeAndDraftEmailWithContext.
- Input requirements: connectionReference (required); documentList with draft email JSON documents (required).
- Output format: JSON confirmation with sent mail metadata for all emails.
Parameters:
- connectionReference (str, required): Microsoft connection label.
- draftEmailJson (str or dict, required): Draft email JSON data containing draftId or draftData with id field.
- documentList (list, required): Document reference(s) to draft emails in JSON format (outputs from outlook.composeAndDraftEmailWithContext function).
"""
try:
connectionReference = parameters.get("connectionReference")
draftEmailJson = parameters.get("draftEmailJson")
documentList = parameters.get("documentList", [])
if not connectionReference:
return ActionResult.isFailure(error="Connection reference is required")
if not draftEmailJson:
return ActionResult.isFailure(error="Draft email JSON is required")
if not documentList:
return ActionResult.isFailure(error="documentList is required and cannot be empty")
# Parse draft email JSON if it's a string
if isinstance(draftEmailJson, str):
try:
draftEmailJson = json.loads(draftEmailJson)
except json.JSONDecodeError:
return ActionResult.isFailure(error="Invalid JSON format in draftEmailJson parameter")
# Extract draft ID from the JSON
draft_id = None
if isinstance(draftEmailJson, dict):
# Try to get draftId directly
draft_id = draftEmailJson.get("draftId")
# If not found, try to get it from draftData
if not draft_id and "draftData" in draftEmailJson:
draft_data = draftEmailJson.get("draftData")
if isinstance(draft_data, dict):
draft_id = draft_data.get("id")
# If still not found, try id field directly
if not draft_id:
draft_id = draftEmailJson.get("id")
if not draft_id:
return ActionResult.isFailure(error="Could not extract draft ID from draftEmailJson. Ensure it contains 'draftId' or 'draftData.id' field")
# Convert single value to list if needed
if isinstance(documentList, str):
documentList = [documentList]
# Get Microsoft connection
connection = self._getMicrosoftConnection(connectionReference)
@ -1433,81 +1413,199 @@ Return JSON:
if not permissions_ok:
return ActionResult.isFailure(error="Connection lacks necessary permissions for Outlook operations")
# Send the draft email
try:
graph_url = "https://graph.microsoft.com/v1.0"
headers = {
"Authorization": f"Bearer {connection['accessToken']}",
"Content-Type": "application/json"
}
send_url = f"{graph_url}/me/messages/{draft_id}/send"
send_response = requests.post(send_url, headers=headers)
# Extract email details from draft JSON for confirmation
subject = draftEmailJson.get("subject", "Unknown")
recipients = draftEmailJson.get("recipients", [])
cc = draftEmailJson.get("cc", [])
bcc = draftEmailJson.get("bcc", [])
attachments_count = draftEmailJson.get("attachments", 0)
if send_response.status_code in [200, 202, 204]:
sent_confirmation_data = {
"status": "sent",
"message": "Email sent successfully",
"draftId": draft_id,
"subject": subject,
"recipients": recipients,
"cc": cc,
"bcc": bcc,
"attachments": attachments_count,
"sentTimestamp": self.services.utils.timestampGetUtc(),
"confirmation": "Email has been successfully sent to recipients"
}
# Read draft email JSON documents from documentList
draftEmails = []
for docRef in documentList:
try:
# Get documents from document reference
chatDocuments = self.services.chat.getChatDocumentsFromDocumentList([docRef])
if not chatDocuments:
logger.warning(f"No documents found for reference: {docRef}")
continue
logger.info(f"Email sent successfully. Draft ID: {draft_id}")
return ActionResult(
success=True,
documents=[ActionDocument(
documentName=f"sent_mail_confirmation_{self._format_timestamp_for_filename()}.json",
documentData=json.dumps(sent_confirmation_data, indent=2),
mimeType="application/json"
)]
)
else:
logger.error(f"Failed to send email. Status: {send_response.status_code}, Response: {send_response.text}")
sent_confirmation_data = {
"status": "error",
"message": "Failed to send draft email",
"draftId": draft_id,
"subject": subject,
"recipients": recipients,
"sendError": {
"statusCode": send_response.status_code,
"response": send_response.text
},
"sentTimestamp": self.services.utils.timestampGetUtc(),
"confirmation": "Email draft sending failed"
}
return ActionResult.isFailure(
error=f"Failed to send email: {send_response.status_code} - {send_response.text}",
documents=[ActionDocument(
documentName=f"sent_mail_confirmation_{self._format_timestamp_for_filename()}.json",
documentData=json.dumps(sent_confirmation_data, indent=2),
mimeType="application/json"
)]
)
except ImportError:
logger.error("requests module not available")
return ActionResult.isFailure(error="requests module not available")
except Exception as e:
logger.error(f"Error sending draft email via Microsoft Graph API: {str(e)}")
return ActionResult.isFailure(error=f"Failed to send draft email: {str(e)}")
# Process each document in the reference
for doc in chatDocuments:
try:
# Read file data
fileId = getattr(doc, 'fileId', None)
if not fileId:
logger.warning(f"Document {doc.fileName} has no fileId")
continue
fileData = self.services.chat.getFileData(fileId)
if not fileData:
logger.warning(f"No file data found for document: {doc.fileName}")
continue
# Parse JSON content
if isinstance(fileData, bytes):
jsonContent = fileData.decode('utf-8')
else:
jsonContent = str(fileData)
# Parse JSON - handle both direct JSON and JSON wrapped in documentData
try:
draftEmailData = json.loads(jsonContent)
# If the JSON contains a 'documentData' field, extract it
if isinstance(draftEmailData, dict) and 'documentData' in draftEmailData:
documentDataStr = draftEmailData['documentData']
if isinstance(documentDataStr, str):
draftEmailData = json.loads(documentDataStr)
# Validate draft email structure
if not isinstance(draftEmailData, dict):
logger.warning(f"Document {doc.fileName} does not contain a valid draft email JSON object")
continue
draftId = draftEmailData.get("draftId")
if not draftId:
logger.warning(f"Document {doc.fileName} does not contain 'draftId' field")
continue
draftEmails.append({
"draftEmailJson": draftEmailData,
"draftId": draftId,
"sourceDocument": doc.fileName,
"sourceReference": docRef
})
except json.JSONDecodeError as e:
logger.error(f"Failed to parse JSON from document {doc.fileName}: {str(e)}")
continue
except Exception as e:
logger.error(f"Error processing document {doc.fileName}: {str(e)}")
continue
except Exception as e:
logger.error(f"Error reading documents from reference {docRef}: {str(e)}")
continue
if not draftEmails:
return ActionResult.isFailure(error="No valid draft email JSON documents found in documentList")
# Send all draft emails
graph_url = "https://graph.microsoft.com/v1.0"
headers = {
"Authorization": f"Bearer {connection['accessToken']}",
"Content-Type": "application/json"
}
sentResults = []
failedResults = []
for draftEmail in draftEmails:
draftEmailJson = draftEmail["draftEmailJson"]
draftId = draftEmail["draftId"]
sourceDocument = draftEmail["sourceDocument"]
try:
send_url = f"{graph_url}/me/messages/{draftId}/send"
sendResponse = requests.post(send_url, headers=headers)
# Extract email details from draft JSON for confirmation
subject = draftEmailJson.get("subject", "Unknown")
recipients = draftEmailJson.get("recipients", [])
cc = draftEmailJson.get("cc", [])
bcc = draftEmailJson.get("bcc", [])
attachmentsCount = draftEmailJson.get("attachments", 0)
if sendResponse.status_code in [200, 202, 204]:
sentResults.append({
"status": "sent",
"message": "Email sent successfully",
"draftId": draftId,
"subject": subject,
"recipients": recipients,
"cc": cc,
"bcc": bcc,
"attachments": attachmentsCount,
"sentTimestamp": self.services.utils.timestampGetUtc(),
"sourceDocument": sourceDocument
})
logger.info(f"Email sent successfully. Draft ID: {draftId}, Subject: {subject}")
else:
errorResult = {
"status": "error",
"message": "Failed to send draft email",
"draftId": draftId,
"subject": subject,
"recipients": recipients,
"sendError": {
"statusCode": sendResponse.status_code,
"response": sendResponse.text
},
"sentTimestamp": self.services.utils.timestampGetUtc(),
"sourceDocument": sourceDocument
}
failedResults.append(errorResult)
logger.error(f"Failed to send email. Draft ID: {draftId}, Status: {sendResponse.status_code}, Response: {sendResponse.text}")
except Exception as e:
errorResult = {
"status": "error",
"message": f"Exception while sending draft email: {str(e)}",
"draftId": draftId,
"subject": draftEmailJson.get("subject", "Unknown"),
"recipients": draftEmailJson.get("recipients", []),
"exception": str(e),
"sentTimestamp": self.services.utils.timestampGetUtc(),
"sourceDocument": sourceDocument
}
failedResults.append(errorResult)
logger.error(f"Error sending draft email {draftId}: {str(e)}")
# Build result summary
totalEmails = len(draftEmails)
successfulEmails = len(sentResults)
failedEmails = len(failedResults)
resultData = {
"totalEmails": totalEmails,
"successfulEmails": successfulEmails,
"failedEmails": failedEmails,
"sentResults": sentResults,
"failedResults": failedResults,
"timestamp": self.services.utils.timestampGetUtc()
}
# Determine overall success status
if successfulEmails == 0:
return ActionResult.isFailure(
error=f"Failed to send all {totalEmails} email(s)",
documents=[ActionDocument(
documentName=f"sent_mail_confirmation_{self._format_timestamp_for_filename()}.json",
documentData=json.dumps(resultData, indent=2),
mimeType="application/json"
)]
)
elif failedEmails > 0:
# Partial success
logger.warning(f"Sent {successfulEmails} out of {totalEmails} emails. {failedEmails} failed.")
return ActionResult(
success=True,
documents=[ActionDocument(
documentName=f"sent_mail_confirmation_{self._format_timestamp_for_filename()}.json",
documentData=json.dumps(resultData, indent=2),
mimeType="application/json"
)]
)
else:
# All successful
logger.info(f"Successfully sent all {totalEmails} email(s)")
return ActionResult(
success=True,
documents=[ActionDocument(
documentName=f"sent_mail_confirmation_{self._format_timestamp_for_filename()}.json",
documentData=json.dumps(resultData, indent=2),
mimeType="application/json"
)]
)
except ImportError:
logger.error("requests module not available")
return ActionResult.isFailure(error="requests module not available")
except Exception as e:
logger.error(f"Error in sendDraftEmail: {str(e)}")
return ActionResult.isFailure(error=str(e))

View file

@ -721,28 +721,109 @@ class WorkflowManager:
raise
async def _processFileIds(self, fileIds: List[str], messageId: str = None) -> List[ChatDocument]:
"""Process file IDs from existing files and return ChatDocument objects"""
"""Process file IDs from existing files and return ChatDocument objects.
If neutralization is enabled, files are neutralized and new files are created with neutralized content.
If neutralization fails, the document is not included and an error is logged to ChatLog."""
documents = []
# Check if neutralization is enabled
neutralizationEnabled = False
try:
config = self.services.neutralization.getConfig()
neutralizationEnabled = config and config.enabled
except Exception as e:
logger.debug(f"Could not check neutralization config: {str(e)}")
workflow = self.services.workflow
for fileId in fileIds:
try:
# Get file info from chat service
fileInfo = self.services.chat.getFileInfo(fileId)
if fileInfo:
# Create document directly with all file attributes
if not fileInfo:
logger.warning(f"No file info found for file ID {fileId}")
continue
originalFileName = fileInfo.get("fileName", "unknown")
originalMimeType = fileInfo.get("mimeType", "application/octet-stream")
fileIdToUse = fileId
fileNameToUse = originalFileName
fileSizeToUse = fileInfo.get("size", 0)
neutralizationFailed = False
# Neutralize file if enabled
if neutralizationEnabled:
try:
# Neutralize the file using the neutralization service
neutralizationResult = self.services.neutralization.processFile(fileId)
if neutralizationResult and 'neutralized_text' in neutralizationResult:
neutralizedText = neutralizationResult['neutralized_text']
# Create new file with neutralized content
neutralizedFileName = neutralizationResult.get('neutralized_file_name', f"neutralized_{originalFileName}")
neutralizedContentBytes = neutralizedText.encode('utf-8')
# Create file in component storage
neutralizedFileItem = self.services.interfaceDbComponent.createFile(
name=neutralizedFileName,
mimeType=originalMimeType,
content=neutralizedContentBytes
)
# Persist file data
self.services.interfaceDbComponent.createFileData(neutralizedFileItem.id, neutralizedContentBytes)
# Use the neutralized file ID and actual size
fileIdToUse = neutralizedFileItem.id
fileNameToUse = neutralizedFileName
fileSizeToUse = len(neutralizedContentBytes)
logger.info(f"Neutralized file {fileId} -> {fileIdToUse} ({fileNameToUse})")
else:
neutralizationFailed = True
errorMsg = f"Neutralization did not return neutralized_text for file '{originalFileName}' (ID: {fileId})"
logger.warning(errorMsg)
self.services.chat.storeLog(workflow, {
"message": errorMsg,
"type": "error",
"status": "error",
"progress": -1
})
except Exception as e:
neutralizationFailed = True
errorMsg = f"Failed to neutralize file '{originalFileName}' (ID: {fileId}): {str(e)}"
logger.error(errorMsg)
self.services.chat.storeLog(workflow, {
"message": errorMsg,
"type": "error",
"status": "error",
"progress": -1
})
# Only add document if neutralization didn't fail (or if neutralization is disabled)
if not neutralizationFailed:
# Create document with file ID (neutralized or original)
document = ChatDocument(
id=str(uuid.uuid4()),
messageId=messageId or "", # Use provided messageId or empty string as fallback
fileId=fileId,
fileName=fileInfo.get("fileName", "unknown"),
fileSize=fileInfo.get("size", 0),
mimeType=fileInfo.get("mimeType", "application/octet-stream")
messageId=messageId or "",
fileId=fileIdToUse,
fileName=fileNameToUse,
fileSize=fileSizeToUse,
mimeType=originalMimeType
)
documents.append(document)
logger.info(f"Processed file ID {fileId} -> {document.fileName}")
logger.info(f"Processed file ID {fileId} -> {document.fileName} (using fileId: {fileIdToUse})")
else:
logger.warning(f"No file info found for file ID {fileId}")
logger.warning(f"Skipping document for file ID {fileId} due to neutralization failure")
except Exception as e:
logger.error(f"Error processing file ID {fileId}: {str(e)}")
errorMsg = f"Error processing file ID {fileId}: {str(e)}"
logger.error(errorMsg)
self.services.chat.storeLog(workflow, {
"message": errorMsg,
"type": "error",
"status": "error",
"progress": -1
})
return documents