Integrated neutralizer to MVP

modules/interfaces/interfaceDbChatObjects.py

@@ -1230,9 +1230,12 @@ class ChatObjects:
         allAutomations = self.db.getRecordset(AutomationDefinition)
         filteredAutomations = self._uam(AutomationDefinition, allAutomations)
         
-        # Compute status for each automation
+        # Compute status for each automation and normalize executionLogs
         for automation in filteredAutomations:
             automation["status"] = self._computeAutomationStatus(automation)
+            # Ensure executionLogs is always a list, not None
+            if automation.get("executionLogs") is None:
+                automation["executionLogs"] = []
         
         # If no pagination requested, return all items
         if pagination is None:
@@ -1272,6 +1275,9 @@ class ChatObjects:
             
             automation = filtered[0]
             automation["status"] = self._computeAutomationStatus(automation)
+            # Ensure executionLogs is always a list, not None
+            if automation.get("executionLogs") is None:
+                automation["executionLogs"] = []
             return automation
         except Exception as e:
             logger.error(f"Error getting automation definition: {str(e)}")
@@ -1306,6 +1312,9 @@ class ChatObjects:
             
             # Compute status
             createdAutomation["status"] = self._computeAutomationStatus(createdAutomation)
+            # Ensure executionLogs is always a list, not None
+            if createdAutomation.get("executionLogs") is None:
+                createdAutomation["executionLogs"] = []
             
             # Trigger sync (async, don't wait)
             asyncio.create_task(self.syncAutomationEvents())
@@ -1334,6 +1343,9 @@ class ChatObjects:
             
             # Compute status
             updatedAutomation["status"] = self._computeAutomationStatus(updatedAutomation)
+            # Ensure executionLogs is always a list, not None
+            if updatedAutomation.get("executionLogs") is None:
+                updatedAutomation["executionLogs"] = []
             
             # Trigger sync (async, don't wait)
             asyncio.create_task(self.syncAutomationEvents())

modules/services/serviceNeutralization/mainServiceNeutralization.py

@@ -34,6 +34,7 @@ class NeutralizationService:
         """
         self.services = serviceCenter
         self.interfaceDbApp = serviceCenter.interfaceDbApp
+        self.interfaceDbComponent = serviceCenter.interfaceDbComponent
         
         # Initialize anonymization processors
         self.NamesToParse = NamesToParse or []
@@ -61,19 +62,19 @@ class NeutralizationService:
         return self._neutralizeText(text, 'text')
 
     def processFile(self, fileId: str) -> Dict[str, Any]:
-        """Neutralize a file referenced by its fileId using app interface."""
+        """Neutralize a file referenced by its fileId using component interface."""
-        if not self.interfaceDbApp:
+        if not self.interfaceDbComponent:
-            raise ValueError("User context is required to process a file by fileId")
+            raise ValueError("Component interface is required to process a file by fileId")
         # Fetch file data and metadata
         fileInfo = None
         try:
             # getFile returns an object; fallback to dict-like
-            fileInfo = self.interfaceDbApp.getFile(fileId)
+            fileInfo = self.interfaceDbComponent.getFile(fileId)
         except Exception:
             fileInfo = None
         fileName = getattr(fileInfo, 'fileName', None) if fileInfo else None
         mimeType = getattr(fileInfo, 'mimeType', None) if fileInfo else None
-        fileData = self.interfaceDbApp.getFileData(fileId)
+        fileData = self.interfaceDbComponent.getFileData(fileId)
         if not fileData:
             raise ValueError(f"No file data found for fileId: {fileId}")
 

modules/workflows/methods/methodOutlook.py

@@ -1337,7 +1337,7 @@ Return JSON:
                     draft_id = draft_data.get("id", "Unknown")
                     
                     # Create draft result data with full draft information
-                    draft_result_data = {
+                    draftResultData = {
                         "status": "draft",
                         "message": "Email draft created successfully with AI-generated content",
                         "draftId": draft_id,
@@ -1361,7 +1361,7 @@ Return JSON:
                         success=True,
                         documents=[ActionDocument(
                             documentName=f"ai_generated_email_draft_{self._format_timestamp_for_filename()}.json",
-                            documentData=json.dumps(draft_result_data, indent=2),
+                            documentData=json.dumps(draftResultData, indent=2),
                             mimeType="application/json"
                         )]
                     )
@@ -1381,47 +1381,27 @@ Return JSON:
     async def sendDraftEmail(self, parameters: Dict[str, Any]) -> ActionResult:
         """
         GENERAL:
-        - Purpose: Send a draft email using the draft email JSON data from action outlook.composeAndDraftEmailWithContext. This action is used to send the email after the email has been composed and drafted.
+        - Purpose: Send draft email(s) using draft email JSON document(s) from action outlook.composeAndDraftEmailWithContext.
-        - Input requirements: connectionReference (required); draftEmailJson (required).
+        - Input requirements: connectionReference (required); documentList with draft email JSON documents (required).
-        - Output format: JSON confirmation with sent mail metadata.
+        - Output format: JSON confirmation with sent mail metadata for all emails.
 
         Parameters:
         - connectionReference (str, required): Microsoft connection label.
-        - draftEmailJson (str or dict, required): Draft email JSON data containing draftId or draftData with id field.
+        - documentList (list, required): Document reference(s) to draft emails in JSON format (outputs from outlook.composeAndDraftEmailWithContext function).
         """
         try:
             connectionReference = parameters.get("connectionReference")
-            draftEmailJson = parameters.get("draftEmailJson")
+            documentList = parameters.get("documentList", [])
             
             if not connectionReference:
                 return ActionResult.isFailure(error="Connection reference is required")
             
-            if not draftEmailJson:
+            if not documentList:
-                return ActionResult.isFailure(error="Draft email JSON is required")
+                return ActionResult.isFailure(error="documentList is required and cannot be empty")
             
-            # Parse draft email JSON if it's a string
+            # Convert single value to list if needed
-            if isinstance(draftEmailJson, str):
+            if isinstance(documentList, str):
-                try:
+                documentList = [documentList]
-                    draftEmailJson = json.loads(draftEmailJson)
-                except json.JSONDecodeError:
-                    return ActionResult.isFailure(error="Invalid JSON format in draftEmailJson parameter")
-            
-            # Extract draft ID from the JSON
-            draft_id = None
-            if isinstance(draftEmailJson, dict):
-                # Try to get draftId directly
-                draft_id = draftEmailJson.get("draftId")
-                # If not found, try to get it from draftData
-                if not draft_id and "draftData" in draftEmailJson:
-                    draft_data = draftEmailJson.get("draftData")
-                    if isinstance(draft_data, dict):
-                        draft_id = draft_data.get("id")
-                # If still not found, try id field directly
-                if not draft_id:
-                    draft_id = draftEmailJson.get("id")
-            
-            if not draft_id:
-                return ActionResult.isFailure(error="Could not extract draft ID from draftEmailJson. Ensure it contains 'draftId' or 'draftData.id' field")
             
             # Get Microsoft connection
             connection = self._getMicrosoftConnection(connectionReference)
@@ -1433,81 +1413,199 @@ Return JSON:
             if not permissions_ok:
                 return ActionResult.isFailure(error="Connection lacks necessary permissions for Outlook operations")
             
-            # Send the draft email
+            # Read draft email JSON documents from documentList
-            try:
+            draftEmails = []
-                graph_url = "https://graph.microsoft.com/v1.0"
+            for docRef in documentList:
-                headers = {
+                try:
-                    "Authorization": f"Bearer {connection['accessToken']}",
+                    # Get documents from document reference
-                    "Content-Type": "application/json"
+                    chatDocuments = self.services.chat.getChatDocumentsFromDocumentList([docRef])
-                }
+                    if not chatDocuments:
-                
+                        logger.warning(f"No documents found for reference: {docRef}")
-                send_url = f"{graph_url}/me/messages/{draft_id}/send"
+                        continue
-                send_response = requests.post(send_url, headers=headers)
-                
-                # Extract email details from draft JSON for confirmation
-                subject = draftEmailJson.get("subject", "Unknown")
-                recipients = draftEmailJson.get("recipients", [])
-                cc = draftEmailJson.get("cc", [])
-                bcc = draftEmailJson.get("bcc", [])
-                attachments_count = draftEmailJson.get("attachments", 0)
-                
-                if send_response.status_code in [200, 202, 204]:
-                    sent_confirmation_data = {
-                        "status": "sent",
-                        "message": "Email sent successfully",
-                        "draftId": draft_id,
-                        "subject": subject,
-                        "recipients": recipients,
-                        "cc": cc,
-                        "bcc": bcc,
-                        "attachments": attachments_count,
-                        "sentTimestamp": self.services.utils.timestampGetUtc(),
-                        "confirmation": "Email has been successfully sent to recipients"
-                    }
                     
-                    logger.info(f"Email sent successfully. Draft ID: {draft_id}")
+                    # Process each document in the reference
-                    
+                    for doc in chatDocuments:
-                    return ActionResult(
+                        try:
-                        success=True,
+                            # Read file data
-                        documents=[ActionDocument(
+                            fileId = getattr(doc, 'fileId', None)
-                            documentName=f"sent_mail_confirmation_{self._format_timestamp_for_filename()}.json",
+                            if not fileId:
-                            documentData=json.dumps(sent_confirmation_data, indent=2),
+                                logger.warning(f"Document {doc.fileName} has no fileId")
-                            mimeType="application/json"
+                                continue
-                        )]
+                            
-                    )
+                            fileData = self.services.chat.getFileData(fileId)
-                else:
+                            if not fileData:
-                    logger.error(f"Failed to send email. Status: {send_response.status_code}, Response: {send_response.text}")
+                                logger.warning(f"No file data found for document: {doc.fileName}")
-                    
+                                continue
-                    sent_confirmation_data = {
+                            
-                        "status": "error",
+                            # Parse JSON content
-                        "message": "Failed to send draft email",
+                            if isinstance(fileData, bytes):
-                        "draftId": draft_id,
+                                jsonContent = fileData.decode('utf-8')
-                        "subject": subject,
+                            else:
-                        "recipients": recipients,
+                                jsonContent = str(fileData)
-                        "sendError": {
+                            
-                            "statusCode": send_response.status_code,
+                            # Parse JSON - handle both direct JSON and JSON wrapped in documentData
-                            "response": send_response.text
+                            try:
-                        },
+                                draftEmailData = json.loads(jsonContent)
-                        "sentTimestamp": self.services.utils.timestampGetUtc(),
+                                
-                        "confirmation": "Email draft sending failed"
+                                # If the JSON contains a 'documentData' field, extract it
-                    }
+                                if isinstance(draftEmailData, dict) and 'documentData' in draftEmailData:
-                    
+                                    documentDataStr = draftEmailData['documentData']
-                    return ActionResult.isFailure(
+                                    if isinstance(documentDataStr, str):
-                        error=f"Failed to send email: {send_response.status_code} - {send_response.text}",
+                                        draftEmailData = json.loads(documentDataStr)
-                        documents=[ActionDocument(
+                                
-                            documentName=f"sent_mail_confirmation_{self._format_timestamp_for_filename()}.json",
+                                # Validate draft email structure
-                            documentData=json.dumps(sent_confirmation_data, indent=2),
+                                if not isinstance(draftEmailData, dict):
-                            mimeType="application/json"
+                                    logger.warning(f"Document {doc.fileName} does not contain a valid draft email JSON object")
-                        )]
+                                    continue
-                    )
+                                
-                    
+                                draftId = draftEmailData.get("draftId")
-            except ImportError:
+                                if not draftId:
-                logger.error("requests module not available")
+                                    logger.warning(f"Document {doc.fileName} does not contain 'draftId' field")
-                return ActionResult.isFailure(error="requests module not available")
+                                    continue
-            except Exception as e:
+                                
-                logger.error(f"Error sending draft email via Microsoft Graph API: {str(e)}")
+                                draftEmails.append({
-                return ActionResult.isFailure(error=f"Failed to send draft email: {str(e)}")
+                                    "draftEmailJson": draftEmailData,
+                                    "draftId": draftId,
+                                    "sourceDocument": doc.fileName,
+                                    "sourceReference": docRef
+                                })
+                                
+                            except json.JSONDecodeError as e:
+                                logger.error(f"Failed to parse JSON from document {doc.fileName}: {str(e)}")
+                                continue
+                                
+                        except Exception as e:
+                            logger.error(f"Error processing document {doc.fileName}: {str(e)}")
+                            continue
+                            
+                except Exception as e:
+                    logger.error(f"Error reading documents from reference {docRef}: {str(e)}")
+                    continue
             
+            if not draftEmails:
+                return ActionResult.isFailure(error="No valid draft email JSON documents found in documentList")
+            
+            # Send all draft emails
+            graph_url = "https://graph.microsoft.com/v1.0"
+            headers = {
+                "Authorization": f"Bearer {connection['accessToken']}",
+                "Content-Type": "application/json"
+            }
+            
+            sentResults = []
+            failedResults = []
+            
+            for draftEmail in draftEmails:
+                draftEmailJson = draftEmail["draftEmailJson"]
+                draftId = draftEmail["draftId"]
+                sourceDocument = draftEmail["sourceDocument"]
+                
+                try:
+                    send_url = f"{graph_url}/me/messages/{draftId}/send"
+                    sendResponse = requests.post(send_url, headers=headers)
+                    
+                    # Extract email details from draft JSON for confirmation
+                    subject = draftEmailJson.get("subject", "Unknown")
+                    recipients = draftEmailJson.get("recipients", [])
+                    cc = draftEmailJson.get("cc", [])
+                    bcc = draftEmailJson.get("bcc", [])
+                    attachmentsCount = draftEmailJson.get("attachments", 0)
+                    
+                    if sendResponse.status_code in [200, 202, 204]:
+                        sentResults.append({
+                            "status": "sent",
+                            "message": "Email sent successfully",
+                            "draftId": draftId,
+                            "subject": subject,
+                            "recipients": recipients,
+                            "cc": cc,
+                            "bcc": bcc,
+                            "attachments": attachmentsCount,
+                            "sentTimestamp": self.services.utils.timestampGetUtc(),
+                            "sourceDocument": sourceDocument
+                        })
+                        logger.info(f"Email sent successfully. Draft ID: {draftId}, Subject: {subject}")
+                    else:
+                        errorResult = {
+                            "status": "error",
+                            "message": "Failed to send draft email",
+                            "draftId": draftId,
+                            "subject": subject,
+                            "recipients": recipients,
+                            "sendError": {
+                                "statusCode": sendResponse.status_code,
+                                "response": sendResponse.text
+                            },
+                            "sentTimestamp": self.services.utils.timestampGetUtc(),
+                            "sourceDocument": sourceDocument
+                        }
+                        failedResults.append(errorResult)
+                        logger.error(f"Failed to send email. Draft ID: {draftId}, Status: {sendResponse.status_code}, Response: {sendResponse.text}")
+                        
+                except Exception as e:
+                    errorResult = {
+                        "status": "error",
+                        "message": f"Exception while sending draft email: {str(e)}",
+                        "draftId": draftId,
+                        "subject": draftEmailJson.get("subject", "Unknown"),
+                        "recipients": draftEmailJson.get("recipients", []),
+                        "exception": str(e),
+                        "sentTimestamp": self.services.utils.timestampGetUtc(),
+                        "sourceDocument": sourceDocument
+                    }
+                    failedResults.append(errorResult)
+                    logger.error(f"Error sending draft email {draftId}: {str(e)}")
+            
+            # Build result summary
+            totalEmails = len(draftEmails)
+            successfulEmails = len(sentResults)
+            failedEmails = len(failedResults)
+            
+            resultData = {
+                "totalEmails": totalEmails,
+                "successfulEmails": successfulEmails,
+                "failedEmails": failedEmails,
+                "sentResults": sentResults,
+                "failedResults": failedResults,
+                "timestamp": self.services.utils.timestampGetUtc()
+            }
+            
+            # Determine overall success status
+            if successfulEmails == 0:
+                return ActionResult.isFailure(
+                    error=f"Failed to send all {totalEmails} email(s)",
+                    documents=[ActionDocument(
+                        documentName=f"sent_mail_confirmation_{self._format_timestamp_for_filename()}.json",
+                        documentData=json.dumps(resultData, indent=2),
+                        mimeType="application/json"
+                    )]
+                )
+            elif failedEmails > 0:
+                # Partial success
+                logger.warning(f"Sent {successfulEmails} out of {totalEmails} emails. {failedEmails} failed.")
+                return ActionResult(
+                    success=True,
+                    documents=[ActionDocument(
+                        documentName=f"sent_mail_confirmation_{self._format_timestamp_for_filename()}.json",
+                        documentData=json.dumps(resultData, indent=2),
+                        mimeType="application/json"
+                    )]
+                )
+            else:
+                # All successful
+                logger.info(f"Successfully sent all {totalEmails} email(s)")
+                return ActionResult(
+                    success=True,
+                    documents=[ActionDocument(
+                        documentName=f"sent_mail_confirmation_{self._format_timestamp_for_filename()}.json",
+                        documentData=json.dumps(resultData, indent=2),
+                        mimeType="application/json"
+                    )]
+                )
+            
+        except ImportError:
+            logger.error("requests module not available")
+            return ActionResult.isFailure(error="requests module not available")
         except Exception as e:
             logger.error(f"Error in sendDraftEmail: {str(e)}")
             return ActionResult.isFailure(error=str(e))

modules/workflows/workflowManager.py

@@ -721,28 +721,109 @@ class WorkflowManager:
         raise
 
     async def _processFileIds(self, fileIds: List[str], messageId: str = None) -> List[ChatDocument]:
-        """Process file IDs from existing files and return ChatDocument objects"""
+        """Process file IDs from existing files and return ChatDocument objects.
+        If neutralization is enabled, files are neutralized and new files are created with neutralized content.
+        If neutralization fails, the document is not included and an error is logged to ChatLog."""
         documents = []
+        
+        # Check if neutralization is enabled
+        neutralizationEnabled = False
+        try:
+            config = self.services.neutralization.getConfig()
+            neutralizationEnabled = config and config.enabled
+        except Exception as e:
+            logger.debug(f"Could not check neutralization config: {str(e)}")
+        
+        workflow = self.services.workflow
+        
         for fileId in fileIds:
             try:
                 # Get file info from chat service
                 fileInfo = self.services.chat.getFileInfo(fileId)
-                if fileInfo:
+                if not fileInfo:
-                    # Create document directly with all file attributes
+                    logger.warning(f"No file info found for file ID {fileId}")
+                    continue
+                
+                originalFileName = fileInfo.get("fileName", "unknown")
+                originalMimeType = fileInfo.get("mimeType", "application/octet-stream")
+                fileIdToUse = fileId
+                fileNameToUse = originalFileName
+                fileSizeToUse = fileInfo.get("size", 0)
+                neutralizationFailed = False
+                
+                # Neutralize file if enabled
+                if neutralizationEnabled:
+                    try:
+                        # Neutralize the file using the neutralization service
+                        neutralizationResult = self.services.neutralization.processFile(fileId)
+                        
+                        if neutralizationResult and 'neutralized_text' in neutralizationResult:
+                            neutralizedText = neutralizationResult['neutralized_text']
+                            
+                            # Create new file with neutralized content
+                            neutralizedFileName = neutralizationResult.get('neutralized_file_name', f"neutralized_{originalFileName}")
+                            neutralizedContentBytes = neutralizedText.encode('utf-8')
+                            
+                            # Create file in component storage
+                            neutralizedFileItem = self.services.interfaceDbComponent.createFile(
+                                name=neutralizedFileName,
+                                mimeType=originalMimeType,
+                                content=neutralizedContentBytes
+                            )
+                            # Persist file data
+                            self.services.interfaceDbComponent.createFileData(neutralizedFileItem.id, neutralizedContentBytes)
+                            
+                            # Use the neutralized file ID and actual size
+                            fileIdToUse = neutralizedFileItem.id
+                            fileNameToUse = neutralizedFileName
+                            fileSizeToUse = len(neutralizedContentBytes)
+                            
+                            logger.info(f"Neutralized file {fileId} -> {fileIdToUse} ({fileNameToUse})")
+                        else:
+                            neutralizationFailed = True
+                            errorMsg = f"Neutralization did not return neutralized_text for file '{originalFileName}' (ID: {fileId})"
+                            logger.warning(errorMsg)
+                            self.services.chat.storeLog(workflow, {
+                                "message": errorMsg,
+                                "type": "error",
+                                "status": "error",
+                                "progress": -1
+                            })
+                    except Exception as e:
+                        neutralizationFailed = True
+                        errorMsg = f"Failed to neutralize file '{originalFileName}' (ID: {fileId}): {str(e)}"
+                        logger.error(errorMsg)
+                        self.services.chat.storeLog(workflow, {
+                            "message": errorMsg,
+                            "type": "error",
+                            "status": "error",
+                            "progress": -1
+                        })
+                
+                # Only add document if neutralization didn't fail (or if neutralization is disabled)
+                if not neutralizationFailed:
+                    # Create document with file ID (neutralized or original)
                     document = ChatDocument(
                         id=str(uuid.uuid4()),
-                        messageId=messageId or "",  # Use provided messageId or empty string as fallback
+                        messageId=messageId or "",
-                        fileId=fileId,
+                        fileId=fileIdToUse,
-                        fileName=fileInfo.get("fileName", "unknown"),
+                        fileName=fileNameToUse,
-                        fileSize=fileInfo.get("size", 0),
+                        fileSize=fileSizeToUse,
-                        mimeType=fileInfo.get("mimeType", "application/octet-stream")
+                        mimeType=originalMimeType
                     )
                     documents.append(document)
-                    logger.info(f"Processed file ID {fileId} -> {document.fileName}")
+                    logger.info(f"Processed file ID {fileId} -> {document.fileName} (using fileId: {fileIdToUse})")
                 else:
-                    logger.warning(f"No file info found for file ID {fileId}")
+                    logger.warning(f"Skipping document for file ID {fileId} due to neutralization failure")
             except Exception as e:
-                logger.error(f"Error processing file ID {fileId}: {str(e)}")
+                errorMsg = f"Error processing file ID {fileId}: {str(e)}"
+                logger.error(errorMsg)
+                self.services.chat.storeLog(workflow, {
+                    "message": errorMsg,
+                    "type": "error",
+                    "status": "error",
+                    "progress": -1
+                })
         return documents