integration test automated workflow execution with ui completed

modules/interfaces/interfaceDbChatObjects.py

@@ -1057,13 +1057,13 @@ class ChatObjects:
         
         # Add progress information if not present
         if "progress" not in logData:
-            # Default progress values based on log type
+            # Default progress values based on log type (0.0 to 1.0 format)
             if logData.get("type") == "info":
-                logData["progress"] = 50  # Default middle progress
+                logData["progress"] = 0.5  # Default middle progress
             elif logData.get("type") == "error":
-                logData["progress"] = -1  # Error state
+                logData["progress"] = 1.0  # Error state - completed (failed)
             elif logData.get("type") == "warning":
-                logData["progress"] = 50  # Default middle progress
+                logData["progress"] = 0.5  # Default middle progress
         
         # Validate log data against ChatLog model
         try:

modules/services/serviceChat/mainServiceChat.py

@@ -145,52 +145,41 @@ class ChatService:
                         if messageFound and messageFound.documents:
                             allDocuments.extend(messageFound.documents)
                 else:
-                    # Direct label reference (round1_task2_action3_contextinfo)
+                    # Direct label reference - can be round1_task2_action3_contextinfo format or simple label
                     # Search for messages with matching documentsLabel to find the actual documents
-                    if docRef.startswith("round"):
+                    matchingMessages = []
-                        # Parse round/task/action to find the corresponding document list
+                    for message in workflow.messages:
-                        labelParts = docRef.split('_', 3)
+                        # Validate message belongs to this workflow
-                        if len(labelParts) >= 4:
+                        msgWorkflowId = getattr(message, 'workflowId', None)
-                            roundNum = int(labelParts[0].replace('round', ''))
+                        if not msgWorkflowId or msgWorkflowId != workflowId:
-                            taskNum = int(labelParts[1].replace('task', ''))
+                            if msgWorkflowId:
-                            actionNum = int(labelParts[2].replace('action', ''))
+                                logger.debug(f"Skipping message {message.id} with workflowId {msgWorkflowId} (expected {workflowId})")
-                            contextInfo = labelParts[3]
-                            
-                            # Find messages with matching documentsLabel (this is the correct way!)
-                            # In case of retries, we want the NEWEST message (most recent publishedAt)
-                            matchingMessages = []
-                            for message in workflow.messages:
-                                # Validate message belongs to this workflow
-                                msgWorkflowId = getattr(message, 'workflowId', None)
-                                if not msgWorkflowId or msgWorkflowId != workflowId:
-                                    if msgWorkflowId:
-                                        logger.debug(f"Skipping message {message.id} with workflowId {msgWorkflowId} (expected {workflowId})")
-                                    else:
-                                        logger.debug(f"Skipping message {message.id} with no workflowId (expected {workflowId})")
-                                    continue
-                                
-                                msgDocumentsLabel = getattr(message, 'documentsLabel', '')
-                                
-                                # Check if this message's documentsLabel matches our reference
-                                if msgDocumentsLabel == docRef:
-                                    # Found a matching message, collect it for comparison
-                                    matchingMessages.append(message)
-                            
-                            # If we found matching messages, take the newest one (highest publishedAt)
-                            if matchingMessages:
-                                # Sort by publishedAt descending (newest first)
-                                matchingMessages.sort(key=lambda msg: getattr(msg, 'publishedAt', 0), reverse=True)
-                                newestMessage = matchingMessages[0]
-                                
-                                if newestMessage.documents:
-                                    docNames = [doc.fileName for doc in newestMessage.documents if hasattr(doc, 'fileName')]
-                                    logger.debug(f"Added {len(newestMessage.documents)} documents from newest message {newestMessage.id}: {docNames}")
-                                    allDocuments.extend(newestMessage.documents)
-                                else:
-                                    logger.debug(f"No documents found in newest message {newestMessage.id}")
                             else:
-                                logger.error(f"No messages found with documentsLabel: {docRef}")
+                                logger.debug(f"Skipping message {message.id} with no workflowId (expected {workflowId})")
-                                raise ValueError(f"Document reference not found: {docRef}")
+                            continue
+                        
+                        msgDocumentsLabel = getattr(message, 'documentsLabel', '')
+                        
+                        # Check if this message's documentsLabel matches our reference
+                        if msgDocumentsLabel == docRef:
+                            # Found a matching message, collect it for comparison
+                            matchingMessages.append(message)
+                    
+                    # If we found matching messages, take the newest one (highest publishedAt)
+                    if matchingMessages:
+                        # Sort by publishedAt descending (newest first)
+                        matchingMessages.sort(key=lambda msg: getattr(msg, 'publishedAt', 0), reverse=True)
+                        newestMessage = matchingMessages[0]
+                        
+                        if newestMessage.documents:
+                            docNames = [doc.fileName for doc in newestMessage.documents if hasattr(doc, 'fileName')]
+                            logger.debug(f"Added {len(newestMessage.documents)} documents from newest message {newestMessage.id}: {docNames}")
+                            allDocuments.extend(newestMessage.documents)
+                        else:
+                            logger.debug(f"No documents found in newest message {newestMessage.id}")
+                    else:
+                        logger.error(f"No messages found with documentsLabel: {docRef}")
+                        raise ValueError(f"Document reference not found: {docRef}")
                         
             logger.debug(f"Resolved {len(allDocuments)} documents from document list: {documentList}")
             return allDocuments

modules/workflows/methods/methodSharepoint.py

@@ -5,9 +5,10 @@ Handles SharePoint document operations using the SharePoint service.
 
 import logging
 import re
+import json
 from typing import Dict, Any, List, Optional
 from datetime import datetime, UTC
-from urllib.parse import urlparse
+import urllib
 import aiohttp
 import asyncio
 
@@ -346,7 +347,7 @@ class MethodSharepoint(MethodBase):
     def _parseSiteUrl(self, siteUrl: str) -> Dict[str, str]:
         """Parse SharePoint site URL to extract hostname and site path"""
         try:
-            parsed = urlparse(siteUrl)
+            parsed = urllib.parse.urlparse(siteUrl)
             hostname = parsed.hostname
             path = parsed.path.strip('/')
             
@@ -497,7 +498,6 @@ class MethodSharepoint(MethodBase):
                 if searchType == "folders" and fileQuery and fileQuery.strip() != "" and fileQuery.strip() != "*":
                     # Use unified search for folders - this is global and searches all sites
                     try:
-                        import json
                         
                         # Use Microsoft Graph Search API syntax (simple term search only)
                         terms = [t for t in fileQuery.split() if t.strip()]
@@ -644,7 +644,6 @@ class MethodSharepoint(MethodBase):
                                     if '/sites/' in web_url:
                                         path_part = web_url.split('/sites/')[1]
                                         # Decode URL encoding and convert to backslash format
-                                        import urllib.parse
                                         decoded_path = urllib.parse.unquote(path_part)
                                         full_path = "\\" + decoded_path.replace('/', '\\')
                                 elif parent_path:
@@ -745,7 +744,6 @@ class MethodSharepoint(MethodBase):
                                 if '/sites/' in web_url:
                                     path_part = web_url.split('/sites/')[1]
                                     # Decode URL encoding and convert to backslash format
-                                    import urllib.parse
                                     decoded_path = urllib.parse.unquote(path_part)
                                     full_path = "\\" + decoded_path.replace('/', '\\')
                             elif parent_path:
@@ -845,7 +843,6 @@ class MethodSharepoint(MethodBase):
                 if pathQuery and pathQuery != "*":
                     logger.debug(f"Both pathObject and pathQuery provided - using pathObject (pathQuery '{pathQuery}' will be ignored)")
                 try:
-                    import json
                     # Resolve the reference label to get the actual document list
                     document_list = self.services.chat.getChatDocumentsFromDocumentList([pathObject])
                     if not document_list or len(document_list) == 0:
@@ -1125,7 +1122,6 @@ class MethodSharepoint(MethodBase):
             # If pathObject is provided, extract folder IDs from it
             if pathObject:
                 try:
-                    import json
                     # Resolve the reference label to get the actual document list
                     document_list = self.services.chat.getChatDocumentsFromDocumentList([pathObject])
                     if not document_list or len(document_list) == 0:
@@ -1479,7 +1475,6 @@ class MethodSharepoint(MethodBase):
                 if pathQuery and pathQuery != "*":
                     logger.debug(f"Both pathObject and pathQuery provided - using pathObject (pathQuery '{pathQuery}' will be ignored)")
                 try:
-                    import json
                     # Resolve the reference label to get the actual document list
                     document_list = self.services.chat.getChatDocumentsFromDocumentList([pathObject])
                     if not document_list or len(document_list) == 0:
@@ -1528,7 +1523,7 @@ class MethodSharepoint(MethodBase):
                                 }
                                 found_documents = [doc]
                                 logger.info(f"Extracted 1 document from validation report")
-                        except json.JSONDecodeError as e:
+                        except ValueError as e:
                             logger.error(f"Failed to parse nested JSON in result field: {e}")
                             return ActionResult.isFailure(error=f"Invalid nested JSON in pathObject: {str(e)}")
                     
@@ -1571,7 +1566,7 @@ class MethodSharepoint(MethodBase):
                     else:
                         return ActionResult.isFailure(error="No folders found in pathObject")
                         
-                except json.JSONDecodeError as e:
+                except ValueError as e:
                     return ActionResult.isFailure(error=f"Invalid JSON in pathObject: {str(e)}")
                 except Exception as e:
                     return ActionResult.isFailure(error=f"Error resolving pathObject reference: {str(e)}")
@@ -1584,8 +1579,16 @@ class MethodSharepoint(MethodBase):
             logger.info(f"Starting SharePoint listDocuments for list_query: {list_query}")
             logger.debug(f"Connection ID: {connection['id']}")
             
-            # Parse list_query to extract path, search terms, search type, and options
+            # For listDocuments, if pathQuery starts with /site:, use it directly without parsing
-            pathQuery, fileQuery, searchType, searchOptions = self._parseSearchQuery(list_query)
+            # (parsing would split on the colon and break the site name)
+            if list_query and list_query.strip().startswith('/site:'):
+                pathQuery = list_query.strip()
+                fileQuery = "*"
+                searchType = "all"
+                searchOptions = {}
+            else:
+                # Parse list_query to extract path, search terms, search type, and options
+                pathQuery, fileQuery, searchType, searchOptions = self._parseSearchQuery(list_query)
             
             # Determine sites to use - strict validation: pathObject → pathQuery → ERROR
             sites = None
@@ -1631,9 +1634,26 @@ class MethodSharepoint(MethodBase):
                     return ActionResult.isFailure(error=f"Invalid pathQuery '{pathQuery}'. This appears to be search terms, not a valid SharePoint path. Use findDocumentPath action first to search for folders, then use the returned folder path as pathQuery.")
                 
                 # For pathQuery, we need to discover sites to find the specific one
-                sites = await self._discoverSharePointSites()
+                all_sites = await self._discoverSharePointSites()
-                if not sites:
+                if not all_sites:
                     return ActionResult.isFailure(error="No SharePoint sites found or accessible")
+                
+                # If pathQuery starts with /site:, extract site name and filter
+                if pathQuery.startswith('/site:'):
+                    # Extract site name from /site:Company Share/... format
+                    site_path_part = pathQuery[6:]  # Remove '/site:'
+                    if '/' in site_path_part:
+                        site_name = site_path_part.split('/', 1)[0]
+                    else:
+                        site_name = site_path_part
+                    
+                    # Filter sites by name (case-insensitive substring match)
+                    sites = self._filter_sites_by_hint(all_sites, site_name)
+                    if not sites:
+                        return ActionResult.isFailure(error=f"No SharePoint site found matching '{site_name}'")
+                    logger.info(f"Filtered to site(s) matching '{site_name}': {[s['displayName'] for s in sites]}")
+                else:
+                    sites = all_sites
             else:
                 # Step 3: Both pathObject and pathQuery failed - ERROR, NO FALLBACK
                 return ActionResult.isFailure(error="No valid list path provided. Either provide pathObject (from findDocumentPath) or a valid pathQuery with specific site information.")
@@ -1647,8 +1667,40 @@ class MethodSharepoint(MethodBase):
                 folder_paths = [list_query]
                 logger.info(f"Using direct folder ID: {list_query}")
             else:
+                # Remove /site:SiteName prefix from pathQuery before resolving (it's only for site filtering)
+                pathQueryForResolve = pathQuery
+                if pathQuery.startswith('/site:'):
+                    # Remove /site:SiteName/ and keep the rest
+                    site_path_part = pathQuery[6:]  # Remove '/site:'
+                    if '/' in site_path_part:
+                        # Remove the site name part, keep the folder path
+                        pathQueryForResolve = '/' + site_path_part.split('/', 1)[1]
+                    else:
+                        # Only site name, no path - use root
+                        pathQueryForResolve = '/'
+                
+                # Remove first path segment if it looks like a document library name
+                # In SharePoint Graph API, /drive/root already points to the default document library,
+                # so library names in paths should be removed
+                # Generic approach: if path has multiple segments, store original for fallback
+                path_segments = [s for s in pathQueryForResolve.split('/') if s.strip()]
+                if len(path_segments) > 1:
+                    # Path has multiple segments - first might be a library name
+                    # Store original for potential fallback
+                    original_path = pathQueryForResolve
+                    # Try without first segment (assuming it's a library name)
+                    pathQueryForResolve = '/' + '/'.join(path_segments[1:])
+                    logger.info(f"Removed first path segment (potential library name), path changed from '{original_path}' to '{pathQueryForResolve}'")
+                elif len(path_segments) == 1:
+                    # Only one segment - if it's a common library-like name, use root
+                    first_segment_lower = path_segments[0].lower()
+                    library_indicators = ['document', 'dokument', 'shared', 'freigegeben', 'library', 'bibliothek']
+                    if any(indicator in first_segment_lower for indicator in library_indicators):
+                        pathQueryForResolve = '/'
+                        logger.info(f"First segment '{path_segments[0]}' appears to be a library name, using root")
+                
                 # Resolve path query into folder paths
-                folder_paths = self._resolvePathQuery(pathQuery)
+                folder_paths = self._resolvePathQuery(pathQueryForResolve)
                 logger.info(f"Resolved folder paths: {folder_paths}")
             
             # Process each folder path across all sites
@@ -1673,9 +1725,11 @@ class MethodSharepoint(MethodBase):
                             # Direct folder ID
                             endpoint = f"sites/{site_id}/drive/items/{folderPath}/children"
                         else:
-                            # Specific folder path - remove leading slash if present
+                            # Specific folder path - remove leading slash if present and URL encode
                             folder_path_clean = folderPath.lstrip('/')
-                            endpoint = f"sites/{site_id}/drive/root:/{folder_path_clean}:/children"
+                            # URL encode the path for Graph API (spaces and special characters need encoding)
+                            folder_path_encoded = urllib.parse.quote(folder_path_clean, safe='/')
+                            endpoint = f"sites/{site_id}/drive/root:/{folder_path_encoded}:/children"
                         
                         # Make the API call to list folder contents
                         api_result = await self._makeGraphApiCall(endpoint)

modules/workflows/processing/core/actionExecutor.py

@@ -146,7 +146,7 @@ class ActionExecutor:
                 self.services.chat.storeLog(workflow, {
                     "message": f"❌ **Task {taskNum}**❌ **Action {actionNum}/{totalActions}** failed: {result.error}",
                     "type": "error",
-                    "progress": 100
+                    "progress": 1.0
                 })
 
             # Log action summary

modules/workflows/processing/core/messageCreator.py

@@ -183,8 +183,7 @@ class MessageCreator:
         except Exception as e:
             logger.error(f"Error creating action message: {str(e)}")
     
-    async def createTaskCompletionMessage(self, taskStep: TaskStep, workflow: ChatWorkflow, taskIndex: int, 
+    async def createTaskCompletionMessage(self, taskStep: TaskStep, workflow: ChatWorkflow, taskIndex: int, totalTasks: int, reviewResult: ReviewResult = None):
-                                        totalTasks: int, reviewResult: ReviewResult):
         """Create a task completion message for the user"""
         try:
             # Check workflow status before creating message
@@ -194,14 +193,17 @@ class MessageCreator:
             taskProgress = str(taskIndex)
             
             # Enhanced completion message with criteria details
-            completionMessage = f"🎯 **Task {taskProgress}**\n\n✅ {reviewResult.reason or 'Task completed successfully'}"
+            if reviewResult and hasattr(reviewResult, 'reason'):
+                completionMessage = f"🎯 **Task {taskProgress}**\n\n✅ {reviewResult.reason or 'Task completed successfully'}"
+            else:
+                completionMessage = f"🎯 **Task {taskProgress}**\n\n✅ Task completed successfully"
             
             # Add criteria status if available
-            if hasattr(reviewResult, 'metCriteria') and reviewResult.metCriteria:
+            if reviewResult and hasattr(reviewResult, 'metCriteria') and reviewResult.metCriteria:
                 for criterion in reviewResult.metCriteria:
                     completionMessage += f"\n• {criterion}"
             
-            if hasattr(reviewResult, 'qualityScore'):
+            if reviewResult and hasattr(reviewResult, 'qualityScore'):
                 completionMessage += f"\n📊 Score {reviewResult.qualityScore}/10"
             
             taskCompletionMessage = {

modules/workflows/processing/shared/automationTemplateInitial.json

@@ -1,17 +1,13 @@
 {
-  "overview": "Automated workflow: Web research, SharePoint data extraction, and document generation",
+  "template":
-  "userMessage": "Execute automated workflow: research web, extract SharePoint data, and generate document",
+{
+  "overview": "Automated workflow task",
   "tasks": [
     {
-      "id": "task_1",
+      "id": "Task01",
-      "objective": "Perform web research using provided URL and prompt to gather information",
+      "title": "Main Task",
-      "dependencies": [],
+      "description": "Execute automated workflow",
-      "successCriteria": [
+      "objective": "Execute automated workflow",
-        "Web research completed successfully",
-        "Research results saved as web_research_response"
-      ],
-      "estimatedComplexity": "medium",
-      "userMessage": "Researching web for information",
       "actionList": [
         {
           "execMethod": "ai",
@@ -20,61 +16,51 @@
             "prompt": "{{KEY:webResearchPrompt}}",
             "list(url)": ["{{KEY:webResearchUrl}}"]
           },
-          "execResultLabel": "web_research_response"
+          "execResultLabel": "web_research_results"
-        }
+        },
-      ]
+        {
-    },
+          "execMethod": "sharepoint",
-    {
+          "execAction": "listDocuments",
-      "id": "task_2",
+          "execParameters": {
-      "objective": "Extract data from files in SharePoint directory using provided folder path and prompt",
+            "connectionReference": "{{KEY:connectionName}}",
-      "dependencies": ["task_1"],
+            "pathQuery": "{{KEY:sharepointFolderNameSource}}"
-      "successCriteria": [
+          },
-        "SharePoint files read successfully",
+          "execResultLabel": "sharepoint_source_path"
-        "Data extracted and saved as sharepoint_data"
+        },
-      ],
-      "estimatedComplexity": "medium",
-      "userMessage": "Extracting data from SharePoint files",
-      "actionList": [
         {
           "execMethod": "sharepoint",
           "execAction": "readDocuments",
           "execParameters": {
             "connectionReference": "{{KEY:connectionName}}",
-            "pathQuery": "{{KEY:sharepointFolderNameSource}}",
+            "documentList": ["sharepoint_source_path"],
-            "documentList": [],
+            "pathQuery": "{{KEY:sharepointFolderNameSource}}"
-            "includeMetadata": true
           },
-          "execResultLabel": "sharepoint_data"
+          "execResultLabel": "sharepoint_source_documents"
-        }
+        },
-      ]
-    },
-    {
-      "id": "task_3",
-      "objective": "Generate document using web research results and SharePoint data with provided prompt",
-      "dependencies": ["task_1", "task_2"],
-      "successCriteria": [
-        "Document generated successfully",
-        "Document combines web research and SharePoint data",
-        "Document saved as result_data"
-      ],
-      "estimatedComplexity": "high",
-      "userMessage": "Generating final document",
-      "actionList": [
         {
-          "execMethod": "ai",
+          "execMethod": "sharepoint",
-          "execAction": "process",
+          "execAction": "uploadDocument",
           "execParameters": {
-            "prompt": "{{KEY:documentPrompt}}",
+            "connectionReference": "{{KEY:connectionName}}",
-            "documentList": [
+            "documentList": ["sharepoint_source_documents","web_research_results"],
-              "web_research_response",
+            "pathQuery": "{{KEY:sharepointFolderNameDestination}}",
-              "sharepoint_data"
+            "fileNames": ["report.docx"]
-            ],
-            "outputExtension": ".docx"
           },
-          "execResultLabel": "result_data"
+          "execResultLabel": "sharepoint_upload_documents"
         }
       ]
     }
   ]
 }
-
+,
+"parameters":
+{
+  "connectionName": "connection:msft:p.motsch@valueon.ch",
+  "webResearchUrl": "https://www.valueon.ch",
+  "webResearchPrompt": "Wer arbeitet bei ValueOn AG in der Schweiz und was machen die?",
+  "PromptSharepointSource": "Fasse die Dokumente in einer Liste zusammen",
+  "sharepointFolderNameSource": "/site:Company Share/Freigegebene Dokumente/15. Persoenliche Ordner/Patrick Motsch/input",
+  "sharepointFolderNameDestination": "/site:Company Share/Freigegebene Dokumente/15. Persoenliche Ordner/Patrick Motsch/output",
+  "PromptDeliverable": "Erstelle mir einen Word Bericht der Webanalyse und der Dokumente im Sharepoint"
+}
+}

modules/workflows/workflowManager.py

@@ -59,7 +59,7 @@ class WorkflowManager:
                     "message": "Workflow stopped for new prompt",
                     "type": "info",
                     "status": "stopped",
-                    "progress": 100
+                    "progress": 1.0
                 })
 
                 newRound = workflow.currentRound + 1
@@ -141,7 +141,7 @@ class WorkflowManager:
                 "message": "Workflow stopped",
                 "type": "warning",
                 "status": "stopped",
-                "progress": 100
+                "progress": 1.0
             })
             return workflow
         except Exception as e:
@@ -470,7 +470,7 @@ class WorkflowManager:
                     "message": "Workflow stopped by user",
                     "type": "warning",
                     "status": "stopped",
-                    "progress": 100
+                    "progress": 1.0
                 })
                 return
             elif workflow.status == 'failed':
@@ -509,7 +509,7 @@ class WorkflowManager:
                     "message": "Workflow failed: Unknown error",
                     "type": "error",
                     "status": "failed",
-                    "progress": 100
+                    "progress": 1.0
                 })
                 return
             
@@ -597,7 +597,7 @@ class WorkflowManager:
                 "message": "Workflow completed",
                 "type": "success",
                 "status": "completed",
-                "progress": 100
+                "progress": 1.0
             })
             
         except Exception as e:
@@ -672,7 +672,7 @@ class WorkflowManager:
             "message": "Workflow stopped by user",
             "type": "warning",
             "status": "stopped",
-            "progress": 100
+            "progress": 1.0
         })
 
     def _handleWorkflowError(self, error: Exception) -> None:
@@ -715,7 +715,7 @@ class WorkflowManager:
             "message": f"Workflow failed: {str(error)}",
             "type": "error",
             "status": "failed",
-            "progress": 100
+            "progress": 1.0
         })
         
         raise