""" SharePoint operations method module. Handles SharePoint document operations using the SharePoint service. """ import logging import re import json from typing import Dict, Any, List, Optional from datetime import datetime, UTC import urllib import aiohttp import asyncio from modules.workflows.methods.methodBase import MethodBase, action from modules.datamodels.datamodelChat import ActionResult, ActionDocument logger = logging.getLogger(__name__) class MethodSharepoint(MethodBase): """SharePoint operations methods.""" def __init__(self, services): super().__init__(services) self.name = "sharepoint" self.description = "SharePoint operations methods" def _format_timestamp_for_filename(self) -> str: """Format current timestamp as YYYYMMDD-hhmmss for filenames.""" return datetime.now(UTC).strftime("%Y%m%d-%H%M%S") def _getMicrosoftConnection(self, connectionReference: str) -> Optional[Dict[str, Any]]: """Get Microsoft connection from connection reference and configure SharePoint service""" try: userConnection = self.services.chat.getUserConnectionFromConnectionReference(connectionReference) if not userConnection: logger.warning(f"No user connection found for reference: {connectionReference}") return None if userConnection.authority.value != "msft": logger.warning(f"Connection {userConnection.id} is not Microsoft (authority: {userConnection.authority.value})") return None # Check if connection is active or pending (pending means OAuth in progress) if userConnection.status.value not in ["active", "pending"]: logger.warning(f"Connection {userConnection.id} status is not active/pending: {userConnection.status.value}") return None # Configure SharePoint service with the UserConnection if not self.services.sharepoint.setAccessTokenFromConnection(userConnection): logger.warning(f"Failed to configure SharePoint service with connection {userConnection.id}") return None logger.info(f"Successfully configured SharePoint service with Microsoft connection: {userConnection.id}, status: {userConnection.status.value}, externalId: {userConnection.externalId}") return { "id": userConnection.id, "userConnection": userConnection, "scopes": ["Sites.ReadWrite.All", "Files.ReadWrite.All", "User.Read"] # SharePoint scopes } except Exception as e: logger.error(f"Error getting Microsoft connection: {str(e)}") return None async def _discoverSharePointSites(self, limit: Optional[int] = None) -> List[Dict[str, Any]]: """ Discover SharePoint sites accessible to the user via Microsoft Graph API Parameters: limit (Optional[int]): Limit number of sites to return (for optimization when only hostname is needed) Returns: List[Dict[str, Any]]: List of SharePoint site information """ try: # Query Microsoft Graph to get sites the user has access to endpoint = "sites?search=*" if limit: endpoint += f"&$top={limit}" result = await self._makeGraphApiCall(endpoint) if "error" in result: logger.error(f"Error discovering SharePoint sites: {result['error']}") return [] sites = result.get("value", []) if limit: sites = sites[:limit] logger.info(f"Discovered {len(sites)} SharePoint sites" + (f" (limited to {limit})" if limit else "")) # Process and return site information processedSites = [] for site in sites: siteInfo = { "id": site.get("id"), "displayName": site.get("displayName"), "name": site.get("name"), "webUrl": site.get("webUrl"), "description": site.get("description"), "createdDateTime": site.get("createdDateTime"), "lastModifiedDateTime": site.get("lastModifiedDateTime") } processedSites.append(siteInfo) logger.debug(f"Site: {siteInfo['displayName']} - {siteInfo['webUrl']}") return processedSites except Exception as e: logger.error(f"Error discovering SharePoint sites: {str(e)}") return [] def _extractHostnameFromWebUrl(self, webUrl: str) -> Optional[str]: """Extract hostname from SharePoint webUrl (e.g., https://pcuster.sharepoint.com)""" try: if not webUrl: return None parsed = urllib.parse.urlparse(webUrl) return parsed.hostname except Exception as e: logger.error(f"Error extracting hostname from webUrl '{webUrl}': {str(e)}") return None async def _getSiteByStandardPath(self, sitePath: str) -> Optional[Dict[str, Any]]: """ Get SharePoint site directly by Microsoft-standard path (/sites/SiteName) without loading all sites. Uses hostname from first available site. Parameters: sitePath (str): Site path like 'company-share' (without /sites/ prefix) Returns: Optional[Dict[str, Any]]: Site information if found, None otherwise """ try: # Get hostname from first available site (minimal load - only 1 site) minimalSites = await self._discoverSharePointSites(limit=1) if not minimalSites: logger.warning("No sites available to extract hostname") return None hostname = self._extractHostnameFromWebUrl(minimalSites[0].get("webUrl")) if not hostname: logger.warning("Could not extract hostname from site") return None logger.info(f"Extracted hostname '{hostname}' from first site, now getting site by path: {sitePath}") # Get site directly using hostname + path endpoint = f"sites/{hostname}:/sites/{sitePath}" result = await self._makeGraphApiCall(endpoint) if "error" in result: logger.warning(f"Could not get site directly by path '{sitePath}': {result['error']}") return None siteInfo = { "id": result.get("id"), "displayName": result.get("displayName"), "name": result.get("name"), "webUrl": result.get("webUrl"), "description": result.get("description"), "createdDateTime": result.get("createdDateTime"), "lastModifiedDateTime": result.get("lastModifiedDateTime") } logger.info(f"Successfully got site by standard path: {siteInfo['displayName']} (ID: {siteInfo['id']})") return siteInfo except Exception as e: logger.error(f"Error getting site by standard path '{sitePath}': {str(e)}") return None def _filterSitesByHint(self, sites: List[Dict[str, Any]], siteHint: str) -> List[Dict[str, Any]]: """Filter discovered sites by a human-entered site hint (case-insensitive substring).""" try: if not siteHint: return sites hint = siteHint.strip().lower() filtered: List[Dict[str, Any]] = [] for site in sites: name = (site.get("displayName") or "").lower() webUrl = (site.get("webUrl") or "").lower() if hint in name or hint in webUrl: filtered.append(site) return filtered if filtered else sites except Exception as e: logger.error(f"Error filtering sites by hint '{siteHint}': {str(e)}") return sites def _extractSiteFromStandardPath(self, pathQuery: str) -> Optional[Dict[str, str]]: """ Extract site name from Microsoft-standard server-relative path: /sites/company-share/Freigegebene Dokumente/... Returns dict with keys: siteName, innerPath (no leading slash) on success, else None. """ try: if not pathQuery or not pathQuery.startswith('/sites/'): return None # Remove leading /sites/ prefix remainder = pathQuery[7:] # len('/sites/') = 7 # Split on first '/' to get site name if '/' not in remainder: # Only site name, no inner path return {"siteName": remainder, "innerPath": ""} siteName, inner = remainder.split('/', 1) siteName = siteName.strip() innerPath = inner.strip() if not siteName: return None return {"siteName": siteName, "innerPath": innerPath} except Exception as e: logger.error(f"Error extracting site from standard path '{pathQuery}': {str(e)}") return None def _parseSearchQuery(self, searchQuery: str) -> tuple[str, str, str, dict]: """ Parse searchQuery to extract path, search terms, search type, and search options. CRITICAL: NEVER convert words to paths! Words stay as search terms. - "root document lesson" → fileQuery="root document lesson" (NOT "/root/document/lesson") - "root, gose" → fileQuery="root, gose" (NOT "/root/gose") - "druckersteuerung eskalation logobject" → fileQuery="druckersteuerung eskalation logobject" Parameters: searchQuery (str): Enhanced search query with options: - "budget" -> pathQuery="*", fileQuery="budget", searchType="all", options={} - "root document lesson" -> pathQuery="*", fileQuery="root document lesson", searchType="all", options={} - "root, gose" -> pathQuery="*", fileQuery="root, gose", searchType="all", options={} - "/Documents:budget" -> pathQuery="/Documents", fileQuery="budget", searchType="all", options={} - "files:budget" -> pathQuery="*", fileQuery="budget", searchType="files", options={} - "folders:DELTA" -> pathQuery="*", fileQuery="DELTA", searchType="folders", options={} - "exact:\"Operations 2025\"" -> exact phrase matching - "regex:^Operations.*2025$" -> regex pattern matching - "case:DELTA" -> case-sensitive search - "and:DELTA AND 2025 Mars AND Group" -> all AND terms must be present Returns: tuple[str, str, str, dict]: (pathQuery, fileQuery, searchType, searchOptions) """ try: if not searchQuery or not searchQuery.strip() or searchQuery.strip() == "*": return "*", "*", "all", {} searchQuery = searchQuery.strip() searchOptions = {} # CRITICAL: Do NOT convert space-separated or comma-separated words to paths! # "root document lesson" should stay as "root document lesson", NOT "/root/document/lesson" # "root, gose" should stay as "root, gose", NOT "/root/gose" # Check for search type specification (files:, folders:, all:) FIRST searchType = "all" # Default if searchQuery.startswith(("files:", "folders:", "all:")): typeParts = searchQuery.split(':', 1) searchType = typeParts[0].strip() searchQuery = typeParts[1].strip() # Extract optional site hint tokens: support "site=Name" or leading "site:Name" def _extractSiteHint(q: str) -> tuple[str, Optional[str]]: try: qStrip = q.strip() # Leading form: site:KM LayerFinance ... if qStrip.lower().startswith("site:"): after = qStrip[5:].lstrip() # site name until next space or end if ' ' in after: siteName, rest = after.split(' ', 1) else: siteName, rest = after, '' return rest.strip(), siteName.strip() # Inline key=value form anywhere m = re.search(r"\bsite=([^;\s]+)", qStrip, flags=re.IGNORECASE) if m: siteName = m.group(1).strip() # remove the token from query qNew = re.sub(r"\bsite=[^;\s]+;?", "", qStrip, flags=re.IGNORECASE).strip() return qNew, siteName except Exception: pass return q, None searchQuery, extractedSite = _extractSiteHint(searchQuery) if extractedSite: searchOptions["site_hint"] = extractedSite logger.info(f"Extracted site hint: '{extractedSite}'") # Extract name="..." if present (for quoted multi-word names) nameMatch = re.search(r"name=\"([^\"]+)\"", searchQuery) if nameMatch: searchQuery = nameMatch.group(1) logger.info(f"Extracted name from quotes: '{searchQuery}'") # Check for search mode specification (exact:, regex:, case:, and:) if searchQuery.startswith(("exact:", "regex:", "case:", "and:")): modeParts = searchQuery.split(':', 1) mode = modeParts[0].strip() searchQuery = modeParts[1].strip() if mode == "exact": searchOptions["exact_match"] = True # Remove quotes if present if searchQuery.startswith('"') and searchQuery.endswith('"'): searchQuery = searchQuery[1:-1] elif mode == "regex": searchOptions["regex_match"] = True elif mode == "case": searchOptions["case_sensitive"] = True elif mode == "and": searchOptions["and_terms"] = True # Check if it contains path:search format # Microsoft-standard paths: /sites/SiteName/Path:files:.pdf if ':' in searchQuery: # For Microsoft-standard paths (/sites/...), find the colon that separates path from search if searchQuery.startswith('/sites/'): # Find the colon that separates path from search (after the full path) # Look for pattern: /sites/SiteName/Path/...:files:.pdf # We need to find the colon that's followed by search type or file extension colonPositions = [] for i, char in enumerate(searchQuery): if char == ':': colonPositions.append(i) # If we have colons, find the one that's followed by search type or file extension splitPos = None if colonPositions: for pos in colonPositions: afterColon = searchQuery[pos+1:pos+10].strip().lower() # Check if this colon is followed by search type or looks like a file extension if afterColon.startswith(('files:', 'folders:', 'all:', '.')) or afterColon == '': splitPos = pos break # If no clear split found, use the last colon if splitPos is None and colonPositions: splitPos = colonPositions[-1] if splitPos: pathPart = searchQuery[:splitPos].strip() searchPart = searchQuery[splitPos+1:].strip() else: # Fallback: split on first colon parts = searchQuery.split(':', 1) pathPart = parts[0].strip() searchPart = parts[1].strip() else: # Regular path:search format - split on first colon parts = searchQuery.split(':', 1) pathPart = parts[0].strip() searchPart = parts[1].strip() # Check if searchPart starts with search type (files:, folders:, all:) if searchPart.startswith(("files:", "folders:", "all:")): typeParts = searchPart.split(':', 1) searchType = typeParts[0].strip() # Update searchType searchPart = typeParts[1].strip() if len(typeParts) > 1 else "" # Handle path part if not pathPart or pathPart == "*": pathQuery = "*" elif pathPart.startswith('/'): pathQuery = pathPart else: pathQuery = f"/Documents/{pathPart}" # Handle search part if not searchPart or searchPart == "*": fileQuery = "*" else: fileQuery = searchPart return pathQuery, fileQuery, searchType, searchOptions # No colon - check if it looks like a path elif searchQuery.startswith('/'): # It's a path only return searchQuery, "*", searchType, searchOptions else: # It's a search term only - keep words as-is, do NOT convert to paths # "root document lesson" stays as "root document lesson" # "root, gose" stays as "root, gose" return "*", searchQuery, searchType, searchOptions except Exception as e: logger.error(f"Error parsing searchQuery '{searchQuery}': {str(e)}") raise ValueError(f"Failed to parse searchQuery '{searchQuery}': {str(e)}") def _resolvePathQuery(self, pathQuery: str) -> List[str]: """ Resolve pathQuery into a list of search paths for SharePoint operations. Parameters: pathQuery (str): Query string that can contain: - Direct paths (e.g., "/Documents/Project1") - Wildcards (e.g., "/Documents/*") - Multiple paths separated by semicolons (e.g., "/Docs; /Files") - Single word relative paths (e.g., "Project1" -> resolved to default folder) - Empty string or "*" for global search - Space-separated words are treated as search terms, NOT folder paths Returns: List[str]: List of resolved paths """ try: if not pathQuery or not pathQuery.strip() or pathQuery.strip() == "*": return ["*"] # Global search across all sites # Split by semicolon to handle multiple paths rawPaths = [path.strip() for path in pathQuery.split(';') if path.strip()] resolvedPaths = [] for rawPath in rawPaths: # Handle wildcards - return as-is if '*' in rawPath: resolvedPaths.append(rawPath) # Handle absolute paths elif rawPath.startswith('/'): resolvedPaths.append(rawPath) # Handle single word relative paths - prepend default folder # BUT NOT space-separated words (those are search terms, not paths) elif ' ' not in rawPath: resolvedPaths.append(f"/Documents/{rawPath}") else: # Check if this looks like a path (has path separators) or search terms if '\\' in rawPath or '/' in rawPath: # This looks like a path with spaces in folder names - treat as valid path resolvedPaths.append(rawPath) logger.info(f"Path with spaces '{rawPath}' treated as valid folder path") else: # Space-separated words without path separators are search terms # Return as "*" to search globally logger.info(f"Space-separated words '{rawPath}' treated as search terms, not folder path") resolvedPaths.append("*") # Remove duplicates while preserving order seen = set() uniquePaths = [] for path in resolvedPaths: if path not in seen: seen.add(path) uniquePaths.append(path) logger.info(f"Resolved pathQuery '{pathQuery}' to {len(uniquePaths)} paths: {uniquePaths}") return uniquePaths except Exception as e: logger.error(f"Error resolving pathQuery '{pathQuery}': {str(e)}") raise ValueError(f"Failed to resolve pathQuery '{pathQuery}': {str(e)}") def _parseSiteUrl(self, siteUrl: str) -> Dict[str, str]: """Parse SharePoint site URL to extract hostname and site path""" try: parsed = urllib.parse.urlparse(siteUrl) hostname = parsed.hostname path = parsed.path.strip('/') return { "hostname": hostname, "sitePath": path } except Exception as e: logger.error(f"Error parsing site URL {siteUrl}: {str(e)}") return {"hostname": "", "sitePath": ""} def _cleanSearchQuery(self, query: str) -> str: """ Clean search query to make it compatible with Graph API KQL syntax. Removes path-like syntax and invalid KQL constructs. Parameters: query (str): Raw search query that may contain paths and invalid syntax Returns: str: Cleaned query suitable for Graph API search endpoint """ if not query or not query.strip(): return "" query = query.strip() # Handle patterns like: "Company Share/Freigegebene Dokumente/.../expenses:files:.pdf" # Extract the search term and file extension # First, extract file extension if present (format: :files:.pdf or just .pdf at the end) fileExtension = "" if ':files:' in query.lower() or ':folders:' in query.lower(): # Extract extension after the type filter extMatch = re.search(r':(?:files|folders):(\.\w+)', query, re.IGNORECASE) if extMatch: fileExtension = extMatch.group(1) # Remove the type filter part query = re.sub(r':(?:files|folders):\.?\w*', '', query, flags=re.IGNORECASE) elif query.endswith(('.pdf', '.doc', '.docx', '.xls', '.xlsx', '.txt', '.csv', '.ppt', '.pptx')): # Extract extension from end extMatch = re.search(r'(\.\w+)$', query) if extMatch: fileExtension = extMatch.group(1) query = query[:-len(fileExtension)] # Extract search term: get the last segment after the last slash (filename part) queryNormalized = query.replace('\\', '/') if '/' in queryNormalized: # Extract the last segment (usually the filename/search term) lastSegment = queryNormalized.split('/')[-1] # Remove any remaining colons or type filters if ':' in lastSegment: lastSegment = lastSegment.split(':')[0] searchTerm = lastSegment.strip() else: # No path separators, use the query as-is but remove type filters if ':' in query: searchTerm = query.split(':')[0].strip() else: searchTerm = query.strip() # Remove any remaining type filters or invalid syntax searchTerm = re.sub(r':(?:files|folders|all):?', '', searchTerm, flags=re.IGNORECASE) searchTerm = searchTerm.strip() # If we have a file extension, include it in the search term # Note: Graph API search endpoint may not support filetype: syntax # So we include the extension as part of the search term or filter results after if fileExtension: extWithoutDot = fileExtension.lstrip('.') # Try simple approach: add extension as search term # If this doesn't work, we'll filter results after search if searchTerm: # Include extension in search - Graph API will search in filename searchTerm = f"{searchTerm} {extWithoutDot}" else: searchTerm = extWithoutDot # Final cleanup: remove any remaining invalid characters for KQL # Keep alphanumeric, spaces, hyphens, underscores, dots, and common search operators searchTerm = re.sub(r'[^\w\s\-\.\*]', ' ', searchTerm) searchTerm = ' '.join(searchTerm.split()) # Normalize whitespace return searchTerm if searchTerm else "*" async def _makeGraphApiCall(self, endpoint: str, method: str = "GET", data: bytes = None) -> Dict[str, Any]: """Make a Microsoft Graph API call with timeout and detailed logging""" try: if not hasattr(self.services, 'sharepoint') or not self.services.sharepoint._target.accessToken: return {"error": "SharePoint service not configured with access token"} headers = { "Authorization": f"Bearer {self.services.sharepoint._target.accessToken}", "Content-Type": "application/json" if data and method != "PUT" else "application/octet-stream" if data else "application/json" } url = f"https://graph.microsoft.com/v1.0/{endpoint}" logger.info(f"Making Graph API call: {method} {url}") # Set timeout to 30 seconds timeout = aiohttp.ClientTimeout(total=30) async with aiohttp.ClientSession(timeout=timeout) as session: if method == "GET": logger.debug(f"Starting GET request to {url}") async with session.get(url, headers=headers) as response: logger.info(f"Graph API response: {response.status}") if response.status == 200: result = await response.json() logger.debug(f"Graph API success: {len(str(result))} characters response") return result else: errorText = await response.text() logger.error(f"Graph API call failed: {response.status} - {errorText}") return {"error": f"API call failed: {response.status} - {errorText}"} elif method == "PUT": logger.debug(f"Starting PUT request to {url}") async with session.put(url, headers=headers, data=data) as response: logger.info(f"Graph API response: {response.status}") if response.status in [200, 201]: result = await response.json() logger.debug(f"Graph API success: {len(str(result))} characters response") return result else: errorText = await response.text() logger.error(f"Graph API call failed: {response.status} - {errorText}") return {"error": f"API call failed: {response.status} - {errorText}"} elif method == "POST": logger.debug(f"Starting POST request to {url}") async with session.post(url, headers=headers, data=data) as response: logger.info(f"Graph API response: {response.status}") if response.status in [200, 201]: result = await response.json() logger.debug(f"Graph API success: {len(str(result))} characters response") return result else: errorText = await response.text() logger.error(f"Graph API call failed: {response.status} - {errorText}") return {"error": f"API call failed: {response.status} - {errorText}"} except asyncio.TimeoutError: logger.error(f"Graph API call timed out after 30 seconds: {endpoint}") return {"error": f"API call timed out after 30 seconds: {endpoint}"} except Exception as e: logger.error(f"Error making Graph API call: {str(e)}") return {"error": f"Error making Graph API call: {str(e)}"} async def _getSiteId(self, hostname: str, sitePath: str) -> str: """Get SharePoint site ID from hostname and site path""" try: endpoint = f"sites/{hostname}:/{sitePath}" result = await self._makeGraphApiCall(endpoint) if "error" in result: logger.error(f"Error getting site ID: {result['error']}") return "" return result.get("id", "") except Exception as e: logger.error(f"Error getting site ID: {str(e)}") return "" @action async def findDocumentPath(self, parameters: Dict[str, Any]) -> ActionResult: """ GENERAL: - Purpose: Find documents and folders by name/path across sites. - Input requirements: connectionReference (required); searchQuery (required); optional site, maxResults. - Output format: JSON with found items and paths. Parameters: - connectionReference (str, required): Microsoft connection label. - site (str, optional): Site hint. - searchQuery (str, required): Search terms or path. - maxResults (int, optional): Maximum items to return. Default: 100. """ try: connectionReference = parameters.get("connectionReference") site = parameters.get("site") searchQuery = parameters.get("searchQuery", "*") maxResults = parameters.get("maxResults", 100) if not connectionReference: return ActionResult.isFailure(error="Connection reference is required") # Parse searchQuery to extract path, search terms, search type, and options pathQuery, fileQuery, searchType, searchOptions = self._parseSearchQuery(searchQuery) logger.debug(f"Parsed searchQuery '{searchQuery}' -> pathQuery='{pathQuery}', fileQuery='{fileQuery}', searchType='{searchType}'") connection = self._getMicrosoftConnection(connectionReference) if not connection: return ActionResult.isFailure(error="No valid Microsoft connection found for the provided connection reference") # Extract site name from pathQuery if it contains Microsoft-standard path (/sites/SiteName/...) siteFromPath = None directSite = None if pathQuery and pathQuery.startswith('/sites/'): parsedPath = self._extractSiteFromStandardPath(pathQuery) if parsedPath: siteFromPath = parsedPath.get("siteName") logger.info(f"Extracted site from Microsoft-standard pathQuery '{pathQuery}': '{siteFromPath}'") # Try to get site directly by path (optimization - no need to load all 60 sites) directSite = await self._getSiteByStandardPath(siteFromPath) if directSite: logger.info(f"Got site directly by standard path - no need to discover all sites") sites = [directSite] else: logger.warning(f"Could not get site directly, falling back to site discovery") directSite = None else: logger.warning(f"Failed to parse site from standard pathQuery '{pathQuery}'") # If we didn't get the site directly, use discovery and filtering if not directSite: # Determine which site hint to use (priority: site parameter > site from pathQuery > site_hint from searchOptions) siteHintToUse = site or siteFromPath or searchOptions.get("site_hint") # Discover SharePoint sites - use targeted approach when site hint is available if siteHintToUse: # When site hint is available, discover all sites first, then filter allSites = await self._discoverSharePointSites() if not allSites: return ActionResult.isFailure(error="No SharePoint sites found or accessible") sites = self._filterSitesByHint(allSites, siteHintToUse) logger.info(f"Filtered sites by site hint '{siteHintToUse}' -> {len(sites)} sites") if not sites: return ActionResult.isFailure(error=f"No SharePoint sites found matching '{siteHintToUse}'") else: # No site hint - discover all sites sites = await self._discoverSharePointSites() if not sites: return ActionResult.isFailure(error="No SharePoint sites found or accessible") # Resolve path query into search paths searchPaths = self._resolvePathQuery(pathQuery) try: # Search across all discovered sites foundDocuments = [] allSitesSearched = [] # Handle different search approaches based on search type if searchType == "folders" and fileQuery and fileQuery.strip() != "" and fileQuery.strip() != "*": # Use unified search for folders - this is global and searches all sites try: # Use Microsoft Graph Search API syntax (simple term search only) terms = [t for t in fileQuery.split() if t.strip()] if len(terms) > 1: # Multiple terms: search for ALL terms (AND) - more specific results queryString = " AND ".join(terms) else: # Single term: search for the term queryString = terms[0] if terms else fileQuery logger.info(f"Using unified search for folders: {queryString}") payload = { "requests": [ { "entityTypes": ["driveItem"], "query": {"queryString": queryString}, "from": 0, "size": 50 } ] } logger.info(f"Using unified search API for folders with queryString: {queryString}") # Use global search endpoint (site-specific search not available) unifiedResult = await self._makeGraphApiCall( "search/query", method="POST", data=json.dumps(payload).encode("utf-8") ) if "error" in unifiedResult: logger.warning(f"Unified search failed: {unifiedResult['error']}") items = [] else: # Flatten hits -> driveItem resources items = [] for container in (unifiedResult.get("value", []) or []): for hitsContainer in (container.get("hitsContainers", []) or []): for hit in (hitsContainer.get("hits", []) or []): resource = hit.get("resource") if resource: items.append(resource) logger.info(f"Unified search returned {len(items)} items (pre-filter)") # Apply our improved folder detection logic folderItems = [] for item in items: resource = item # Use the same detection logic as our test isFolder = False if 'folder' in resource: isFolder = True else: # Try to detect by URL pattern or other indicators webUrl = resource.get('webUrl', '') name = resource.get('name', '') # Check if URL has no file extension and looks like a folder path if '.' not in name and ('/' in webUrl or '\\' in webUrl): isFolder = True if isFolder: folderItems.append(item) items = folderItems logger.info(f"Filtered to {len(items)} folders using improved detection logic") # Process unified search results - extract site information from webUrl for item in items: itemName = item.get("name", "") webUrl = item.get("webUrl", "") # Extract site information from webUrl siteName = "Unknown Site" siteId = "unknown" if webUrl and '/sites/' in webUrl: try: # Extract site name from URL: https://pcuster.sharepoint.com/sites/SiteName/... urlParts = webUrl.split('/sites/') if len(urlParts) > 1: sitePath = urlParts[1].split('/')[0] # Find matching site from discovered sites # First try to match by site name (URL path) for site in sites: if site.get("name") == sitePath: siteName = site.get("displayName", sitePath) siteId = site.get("id", "unknown") break else: # If no match by name, try to match by displayName for site in sites: if site.get("displayName") == sitePath: siteName = site.get("displayName", sitePath) siteId = site.get("id", "unknown") break else: # If no exact match, use the site path as site name siteName = sitePath # Try to find a site with similar name for site in sites: if sitePath.lower() in site.get("name", "").lower() or sitePath.lower() in site.get("displayName", "").lower(): siteName = site.get("displayName", sitePath) siteId = site.get("id", "unknown") break except Exception as e: logger.warning(f"Error extracting site info from URL {webUrl}: {e}") # Use improved folder detection logic isFolder = False if 'folder' in item: isFolder = True else: # Try to detect by URL pattern or other indicators name = item.get('name', '') # Check if URL has no file extension and looks like a folder path if '.' not in name and ('/' in webUrl or '\\' in webUrl): isFolder = True itemType = "folder" if isFolder else "file" itemPath = item.get("parentReference", {}).get("path", "") logger.debug(f"Processing {itemType}: '{itemName}' at path: '{itemPath}'") # Simple filtering like test file - just check search type if searchType == "files" and isFolder: continue # Skip folders when searching for files elif searchType == "folders" and not isFolder: continue # Skip files when searching for folders # Simple approach like test file - no complex filtering logger.debug(f"Item '{itemName}' found - adding to results") # Create result with full path information for proper action chaining parentPath = item.get("parentReference", {}).get("path", "") # Extract the full SharePoint path from webUrl or parentReference fullPath = "" if webUrl: # Extract path from webUrl: https://pcuster.sharepoint.com/sites/SSSRESYNachfolge/Freigegebene%20Dokumente/General/Eskalation%20LogObject/Druckersteuerung if '/sites/' in webUrl: pathPart = webUrl.split('/sites/')[1] # Decode URL encoding and convert to backslash format decodedPath = urllib.parse.unquote(pathPart) fullPath = "\\" + decodedPath.replace('/', '\\') elif parentPath: # Use parentReference path if available fullPath = parentPath.replace('/', '\\') docInfo = { "id": item.get("id"), "name": item.get("name"), "type": "folder" if isFolder else "file", "siteName": siteName, "siteId": siteId, "webUrl": webUrl, "fullPath": fullPath, "parentPath": parentPath } foundDocuments.append(docInfo) logger.info(f"Found {len(foundDocuments)} documents from unified search") except Exception as e: logger.error(f"Error performing unified folder search: {str(e)}") # Fallback to site-by-site search pass # If no unified search was performed or it failed, fall back to site-by-site search if not foundDocuments: # Use simple approach like test file - no complex filtering siteScopedSites = sites for site in siteScopedSites: siteId = site["id"] siteName = site["displayName"] siteUrl = site["webUrl"] logger.info(f"Searching in site: {siteName} ({siteUrl})") # Check if pathQuery contains a specific folder path (not just /sites/SiteName) folderPath = None if pathQuery and pathQuery.startswith('/sites/'): parsedPath = self._extractSiteFromStandardPath(pathQuery) if parsedPath: innerPath = parsedPath.get("innerPath", "") if innerPath and innerPath.strip(): # Remove leading slash if present folderPath = innerPath.lstrip('/') # Generic approach: Try to find the folder, if it fails, remove first segment # This works for all languages because we test the actual API response # In SharePoint Graph API, /drive/root already points to the default document library, # so library names in paths should be removed pathSegments = [s for s in folderPath.split('/') if s.strip()] if len(pathSegments) > 1: # Try with first segment removed (first segment is likely the document library) testPath = '/'.join(pathSegments[1:]) # Quick test: try to get folder info (this is fast and doesn't require full search) testEndpoint = f"sites/{siteId}/drive/root:/{urllib.parse.quote(testPath, safe='')}:" testResult = await self._makeGraphApiCall(testEndpoint) if testResult and "error" not in testResult: # Path without first segment works - first segment was likely the document library folderPath = testPath logger.info(f"Removed document library name '{pathSegments[0]}' from folder path (tested via API)") else: # Keep original path - first segment is not a document library logger.info(f"Keeping original folder path '{folderPath}' (first segment is not a document library)") elif len(pathSegments) == 1: # Only one segment - likely the document library itself, use root folderPath = None logger.info(f"Only one segment '{pathSegments[0]}' found, likely document library - using root") if folderPath: logger.info(f"Extracted folder path from pathQuery: '{folderPath}'") else: logger.info(f"Folder path resolved to root (only document library in path)") # Use Microsoft Graph API for this specific site # Handle empty or wildcard queries if not fileQuery or fileQuery.strip() == "" or fileQuery.strip() == "*": # For wildcard/empty queries, list all items if folderPath: # List items in specific folder encodedPath = urllib.parse.quote(folderPath, safe='') endpoint = f"sites/{siteId}/drive/root:/{encodedPath}:/children" logger.info(f"Listing items in folder: '{folderPath}'") else: # List all items in the drive root endpoint = f"sites/{siteId}/drive/root/children" # Make the API call to list items listResult = await self._makeGraphApiCall(endpoint) if "error" in listResult: logger.warning(f"List failed for site {siteName}: {listResult['error']}") continue # Process list results for this site items = listResult.get("value", []) logger.info(f"Retrieved {len(items)} items from site {siteName}") else: # For files, use regular search API # Clean the query: remove path-like syntax and invalid KQL syntax searchQuery = self._cleanSearchQuery(fileQuery) # URL-encode the query parameter encodedQuery = urllib.parse.quote(searchQuery, safe='') if folderPath: # Search in specific folder encodedPath = urllib.parse.quote(folderPath, safe='') endpoint = f"sites/{siteId}/drive/root:/{encodedPath}:/search(q='{encodedQuery}')" logger.info(f"Searching in folder '{folderPath}' with query: '{searchQuery}' (encoded: '{encodedQuery}')") else: # Search in drive root endpoint = f"sites/{siteId}/drive/root/search(q='{encodedQuery}')" logger.info(f"Using search API for files with query: '{searchQuery}' (encoded: '{encodedQuery}')") # Make the search API call (files) searchResult = await self._makeGraphApiCall(endpoint) if "error" in searchResult: logger.warning(f"Search failed for site {siteName}: {searchResult['error']}") continue # Process search results for this site (files) items = searchResult.get("value", []) logger.info(f"Retrieved {len(items)} items from site {siteName}") siteDocuments = [] for item in items: itemName = item.get("name", "") # Use improved folder detection logic isFolder = False if 'folder' in item: isFolder = True else: # Try to detect by URL pattern or other indicators webUrl = item.get('webUrl', '') name = item.get('name', '') # Check if URL has no file extension and looks like a folder path if '.' not in name and ('/' in webUrl or '\\' in webUrl): isFolder = True itemType = "folder" if isFolder else "file" itemPath = item.get("parentReference", {}).get("path", "") logger.debug(f"Processing {itemType}: '{itemName}' at path: '{itemPath}'") # Simple filtering like test file - just check search type if searchType == "files" and isFolder: continue # Skip folders when searching for files elif searchType == "folders" and not isFolder: continue # Skip files when searching for folders # Simple approach like test file - no complex filtering logger.debug(f"Item '{itemName}' found - adding to results") # Create result with full path information for proper action chaining webUrl = item.get("webUrl", "") parentPath = item.get("parentReference", {}).get("path", "") # Extract the full SharePoint path from webUrl or parentReference fullPath = "" if webUrl: # Extract path from webUrl: https://pcuster.sharepoint.com/sites/SSSRESYNachfolge/Freigegebene%20Dokumente/General/Eskalation%20LogObject/Druckersteuerung if '/sites/' in webUrl: pathPart = webUrl.split('/sites/')[1] # Decode URL encoding and convert to backslash format decodedPath = urllib.parse.unquote(pathPart) fullPath = "\\" + decodedPath.replace('/', '\\') elif parentPath: # Use parentReference path if available fullPath = parentPath.replace('/', '\\') docInfo = { "id": item.get("id"), "name": item.get("name"), "type": "folder" if isFolder else "file", "siteName": siteName, "siteId": siteId, "webUrl": webUrl, "fullPath": fullPath, "parentPath": parentPath } siteDocuments.append(docInfo) foundDocuments.extend(siteDocuments) allSitesSearched.append({ "siteName": siteName, "siteUrl": siteUrl, "siteId": siteId, "documentsFound": len(siteDocuments) }) logger.info(f"Found {len(siteDocuments)} documents in site {siteName}") # Limit total results to maxResults if len(foundDocuments) > maxResults: foundDocuments = foundDocuments[:maxResults] logger.info(f"Limited results to {maxResults} items") resultData = { "searchQuery": searchQuery, "totalResults": len(foundDocuments), "maxResults": maxResults, "foundDocuments": foundDocuments, "timestamp": self.services.utils.timestampGetUtc() } except Exception as e: logger.error(f"Error searching SharePoint: {str(e)}") return ActionResult.isFailure(error=str(e)) # Use default JSON format for output outputExtension = ".json" # Default outputMimeType = "application/json" # Default validationMetadata = { "actionType": "sharepoint.findDocumentPath", "searchQuery": searchQuery, "maxResults": maxResults, "totalResults": len(foundDocuments), "hasResults": len(foundDocuments) > 0 } return ActionResult( success=True, documents=[ ActionDocument( documentName=f"sharepoint_find_path_{self._format_timestamp_for_filename()}{outputExtension}", documentData=json.dumps(resultData, indent=2), mimeType=outputMimeType, validationMetadata=validationMetadata ) ] ) except Exception as e: logger.error(f"Error finding document path: {str(e)}") return ActionResult.isFailure(error=str(e)) @action async def readDocuments(self, parameters: Dict[str, Any]) -> ActionResult: """ GENERAL: - Purpose: Read documents from SharePoint and extract content/metadata. - Input requirements: connectionReference (required); optional documentList, pathObject, or pathQuery; includeMetadata. - Output format: Standardized ActionDocument format (documentName, documentData, mimeType). - Binary files (PDFs, etc.) are Base64-encoded in documentData. - Text files are stored as plain text in documentData. - Returns ActionResult with documents list for template processing. Parameters: - connectionReference (str, required): Microsoft connection label. - pathObject (str, optional): Reference to a previous path result (from findDocumentPath). - documentList (list, optional): Document list reference(s) to read (backward compatibility). - pathQuery (str, optional): Path query if no pathObject (backward compatibility). - includeMetadata (bool, optional): Include metadata. Default: True. Returns: - ActionResult with documents: List[ActionDocument] where each ActionDocument contains: - documentName: File name - documentData: Base64-encoded content (binary files) or plain text (text files) - mimeType: MIME type (e.g., application/pdf, text/plain) """ import time operationId = None try: # Init progress logger workflowId = self.services.workflow.id if self.services.workflow else f"no-workflow-{int(time.time())}" operationId = f"sharepoint_read_{workflowId}_{int(time.time())}" # Start progress tracking self.services.chat.progressLogStart( operationId, "Read Documents", "SharePoint Document Reading", f"Path: {parameters.get('pathQuery', parameters.get('pathObject', '*'))}" ) documentList = parameters.get("documentList") if isinstance(documentList, str): documentList = [documentList] connectionReference = parameters.get("connectionReference") pathQuery = parameters.get("pathQuery", "*") pathObject = parameters.get("pathObject") includeMetadata = parameters.get("includeMetadata", True) # Validate connection reference if not connectionReference: if operationId: self.services.chat.progressLogFinish(operationId, False) return ActionResult.isFailure(error="Connection reference is required") # Get connection first - needed for both pathObject and documentList approaches self.services.chat.progressLogUpdate(operationId, 0.2, "Getting Microsoft connection") connection = self._getMicrosoftConnection(connectionReference) if not connection: if operationId: self.services.chat.progressLogFinish(operationId, False) return ActionResult.isFailure(error="No valid Microsoft connection found for the provided connection reference") # If pathObject is provided, extract SharePoint file IDs and read them directly # pathObject contains the result from findDocumentPath with foundDocuments array sharePointFileIds = None sites = None if pathObject: if pathQuery and pathQuery != "*": logger.debug(f"Both pathObject and pathQuery provided - using pathObject (pathQuery '{pathQuery}' will be ignored)") try: # Resolve the reference label to get the actual document list from modules.datamodels.datamodelDocref import DocumentReferenceList pathObjectDocuments = self.services.chat.getChatDocumentsFromDocumentList(DocumentReferenceList.from_string_list([pathObject])) if not pathObjectDocuments or len(pathObjectDocuments) == 0: if operationId: self.services.chat.progressLogFinish(operationId, False) return ActionResult.isFailure(error=f"No document list found for reference: {pathObject}") # Get the first document's content (which should be the JSON from findDocumentPath) firstDocument = pathObjectDocuments[0] fileData = self.services.chat.getFileData(firstDocument.fileId) if not fileData: return ActionResult.isFailure(error=f"No file data found for document: {pathObject}") # Parse the JSON content resultData = json.loads(fileData) foundDocuments = resultData.get("foundDocuments", []) # If no foundDocuments, check if it's a listDocuments result (has listResults) if not foundDocuments and "listResults" in resultData: logger.info(f"pathObject contains listResults from listDocuments, converting to foundDocuments format") listResults = resultData.get("listResults", []) foundDocuments = [] siteIdFromList = None siteNameFromList = None for listResult in listResults: siteResults = listResult.get("siteResults", []) for siteResult in siteResults: items = siteResult.get("items", []) # Extract site info from first item if available if items and not siteIdFromList: # Try to get site info from the siteResult structure # We need to discover sites to get the siteId siteNameFromList = items[0].get("siteName") for item in items: # Convert listDocuments item format to foundDocuments format if item.get("type") == "file": foundDoc = { "id": item.get("id"), "name": item.get("name"), "type": "file", "siteName": item.get("siteName"), "siteId": None, # Will be determined from site discovery "webUrl": item.get("webUrl"), "fullPath": item.get("webUrl", ""), "parentPath": item.get("parentPath", "") } foundDocuments.append(foundDoc) # Discover sites to get siteId if we have siteName if foundDocuments and siteNameFromList and not siteIdFromList: logger.info(f"Discovering sites to find siteId for '{siteNameFromList}'") allSites = await self._discoverSharePointSites() matchingSites = self._filterSitesByHint(allSites, siteNameFromList) if matchingSites: siteIdFromList = matchingSites[0].get("id") # Update all foundDocuments with siteId for doc in foundDocuments: doc["siteId"] = siteIdFromList logger.info(f"Found siteId '{siteIdFromList}' for site '{siteNameFromList}'") logger.info(f"Converted {len(foundDocuments)} files from listResults format") if foundDocuments: # Extract SharePoint file IDs from foundDocuments sharePointFileIds = [doc.get("id") for doc in foundDocuments if doc.get("type") == "file"] if not sharePointFileIds: return ActionResult.isFailure(error=f"No files found in pathObject '{pathObject}'") logger.info(f"Extracted {len(sharePointFileIds)} SharePoint file IDs from pathObject '{pathObject}'") # Extract site information from foundDocuments if foundDocuments: firstDoc = foundDocuments[0] siteName = firstDoc.get("siteName") siteId = firstDoc.get("siteId") # If siteId is missing (from listDocuments conversion), discover sites to find it if siteName and not siteId: logger.info(f"Site ID missing, discovering sites to find siteId for '{siteName}'") allSites = await self._discoverSharePointSites() matchingSites = self._filterSitesByHint(allSites, siteName) if matchingSites: siteId = matchingSites[0].get("id") logger.info(f"Found siteId '{siteId}' for site '{siteName}'") if siteName and siteId: sites = [{ "id": siteId, "displayName": siteName, "webUrl": firstDoc.get("webUrl", "") }] logger.info(f"Using specific site from pathObject: {siteName} (ID: {siteId})") elif siteName: # Try to get site by name allSites = await self._discoverSharePointSites() matchingSites = self._filterSitesByHint(allSites, siteName) if matchingSites: sites = [{ "id": matchingSites[0].get("id"), "displayName": siteName, "webUrl": matchingSites[0].get("webUrl", "") }] logger.info(f"Found site by name: {siteName} (ID: {sites[0]['id']})") else: return ActionResult.isFailure(error=f"Site '{siteName}' not found. Cannot determine target site for read operation.") else: return ActionResult.isFailure(error="Site information missing from pathObject. Cannot determine target site for read operation.") else: return ActionResult.isFailure(error=f"No documents found in pathObject '{pathObject}'") except json.JSONDecodeError as e: return ActionResult.isFailure(error=f"Invalid JSON in pathObject: {str(e)}") except Exception as e: return ActionResult.isFailure(error=f"Error resolving pathObject reference: {str(e)}") # If we have SharePoint file IDs from pathObject, read them directly if sharePointFileIds and sites: # Read SharePoint files directly using their IDs readResults = [] siteId = sites[0]['id'] self.services.chat.progressLogUpdate(operationId, 0.5, f"Reading {len(sharePointFileIds)} file(s) from SharePoint") for idx, fileId in enumerate(sharePointFileIds): try: self.services.chat.progressLogUpdate(operationId, 0.5 + (idx * 0.3 / len(sharePointFileIds)), f"Reading file {idx + 1}/{len(sharePointFileIds)}") # Get file info from SharePoint endpoint = f"sites/{siteId}/drive/items/{fileId}" fileInfo = await self._makeGraphApiCall(endpoint) if "error" in fileInfo: logger.warning(f"Failed to get file info for {fileId}: {fileInfo['error']}") continue # Get file content using SharePoint service (handles binary data correctly) fileName = fileInfo.get("name", f"file_{fileId}") fileContent = await self.services.sharepoint.downloadFile(siteId, fileId) # Create result document resultItem = { "fileId": fileId, "fileName": fileName, "sharepointFileId": fileId, "siteName": sites[0]['displayName'], "siteUrl": sites[0]['webUrl'], "size": fileInfo.get("size", 0), "createdDateTime": fileInfo.get("createdDateTime"), "lastModifiedDateTime": fileInfo.get("lastModifiedDateTime"), "webUrl": fileInfo.get("webUrl") } # Add content if available if fileContent: resultItem["content"] = fileContent # Add metadata if requested if includeMetadata: resultItem["metadata"] = { "mimeType": fileInfo.get("file", {}).get("mimeType"), "downloadUrl": fileInfo.get("@microsoft.graph.downloadUrl"), "createdBy": fileInfo.get("createdBy", {}), "lastModifiedBy": fileInfo.get("lastModifiedBy", {}), "parentReference": fileInfo.get("parentReference", {}) } readResults.append(resultItem) except Exception as e: logger.error(f"Error reading file {fileId}: {str(e)}") continue if not readResults: self.services.chat.progressLogFinish(operationId, False) return ActionResult.isFailure(error="No files could be read from pathObject") # Convert read results to ActionDocument objects # IMPORTANT: For binary files (PDFs), store Base64-encoded content directly in documentData # The system will create FileData and ChatDocument automatically self.services.chat.progressLogUpdate(operationId, 0.8, f"Processing {len(readResults)} document(s)") from modules.datamodels.datamodelChat import ActionDocument import base64 actionDocuments = [] for resultItem in readResults: fileContent = resultItem.get("content") fileName = resultItem.get("fileName", f"file_{resultItem.get('fileId')}") # Determine MIME type from metadata or file extension mimeType = "application/octet-stream" if resultItem.get("metadata", {}).get("mimeType"): mimeType = resultItem["metadata"]["mimeType"] elif fileName: if fileName.endswith('.pdf'): mimeType = "application/pdf" elif fileName.endswith('.txt'): mimeType = "text/plain" elif fileName.endswith('.json'): mimeType = "application/json" # For binary files (PDFs, etc.), store Base64-encoded content directly # The GenerationService will detect PDF mimeType and handle base64 decoding if fileContent and isinstance(fileContent, bytes): # Encode binary content as Base64 string base64Content = base64.b64encode(fileContent).decode('utf-8') validationMetadata = { "actionType": "sharepoint.readDocuments", "fileName": fileName, "sharepointFileId": resultItem.get("sharepointFileId"), "siteName": resultItem.get("siteName"), "mimeType": mimeType, "contentType": "binary", "size": len(fileContent), "includeMetadata": includeMetadata } actionDoc = ActionDocument( documentName=fileName, documentData=base64Content, # Base64 string for binary files mimeType=mimeType, validationMetadata=validationMetadata ) actionDocuments.append(actionDoc) logger.info(f"Stored binary file {fileName} ({len(fileContent)} bytes) as Base64 in ActionDocument") elif fileContent: # Text content - store directly in documentData validationMetadata = { "actionType": "sharepoint.readDocuments", "fileName": fileName, "sharepointFileId": resultItem.get("sharepointFileId"), "siteName": resultItem.get("siteName"), "mimeType": mimeType, "contentType": "text", "includeMetadata": includeMetadata } actionDoc = ActionDocument( documentName=fileName, documentData=fileContent if isinstance(fileContent, str) else str(fileContent), mimeType=mimeType, validationMetadata=validationMetadata ) actionDocuments.append(actionDoc) else: # No content - store metadata only docData = { "fileName": fileName, "sharepointFileId": resultItem.get("sharepointFileId"), "siteName": resultItem.get("siteName"), "siteUrl": resultItem.get("siteUrl"), "size": resultItem.get("size"), "createdDateTime": resultItem.get("createdDateTime"), "lastModifiedDateTime": resultItem.get("lastModifiedDateTime"), "webUrl": resultItem.get("webUrl") } if resultItem.get("metadata"): docData["metadata"] = resultItem["metadata"] validationMetadata = { "actionType": "sharepoint.readDocuments", "fileName": fileName, "sharepointFileId": resultItem.get("sharepointFileId"), "siteName": resultItem.get("siteName"), "mimeType": mimeType, "contentType": "metadata_only", "includeMetadata": includeMetadata } actionDoc = ActionDocument( documentName=fileName, documentData=json.dumps(docData, indent=2), mimeType=mimeType, validationMetadata=validationMetadata ) actionDocuments.append(actionDoc) # Return success with action documents self.services.chat.progressLogUpdate(operationId, 0.9, f"Read {len(actionDocuments)} document(s)") self.services.chat.progressLogFinish(operationId, True) return ActionResult.isSuccess(documents=actionDocuments) # Fallback: Use documentList parameter (for backward compatibility) # Validate documentList if not documentList: return ActionResult.isFailure(error="Document list reference is required. Either provide documentList parameter or use pathObject that contains files.") # Get documents from reference - ensure documentList is a list, not a string # documentList is already normalized above from modules.datamodels.datamodelDocref import DocumentReferenceList # Convert to DocumentReferenceList if needed if isinstance(documentList, DocumentReferenceList): docRefList = documentList elif isinstance(documentList, list): docRefList = DocumentReferenceList.from_string_list(documentList) elif isinstance(documentList, str): docRefList = DocumentReferenceList.from_string_list([documentList]) else: docRefList = DocumentReferenceList(references=[]) chatDocuments = self.services.chat.getChatDocumentsFromDocumentList(docRefList) if not chatDocuments: return ActionResult.isFailure(error="No documents found for the provided reference") # Determine sites to use - strict validation: pathObject → pathQuery → ERROR if not sites: # Step 2: If no pathObject, check pathQuery if pathQuery and pathQuery.strip() != "" and pathQuery.strip() != "*": # Validate pathQuery format if not pathQuery.startswith('/'): return ActionResult.isFailure(error="pathQuery must start with '/' and include site name with Microsoft-standard syntax /sites//... e.g. /sites/company-share/Freigegebene Dokumente/Work") # Check if pathQuery contains search terms (words without proper path structure) validPathPrefixes = ['/sites/', '/Documents', '/documents', '/Shared Documents', '/shared documents'] if not any(pathQuery.startswith(prefix) for prefix in validPathPrefixes): return ActionResult.isFailure(error=f"Invalid pathQuery '{pathQuery}'. This appears to be search terms, not a valid SharePoint path. Use findDocumentPath action first to search for folders, then use the returned folder path as pathQuery.") # If pathQuery starts with Microsoft-standard /sites/, try to get site directly directSite = None if pathQuery.startswith('/sites/'): parsedPath = self._extractSiteFromStandardPath(pathQuery) if parsedPath: siteName = parsedPath.get("siteName") # Try to get site directly by path (optimization - no need to load all 60 sites) directSite = await self._getSiteByStandardPath(siteName) if directSite: logger.info(f"Got site directly by standard path - no need to discover all sites") sites = [directSite] else: logger.warning(f"Could not get site directly, falling back to site discovery") # If we didn't get the site directly, use discovery and filtering if not directSite: # For pathQuery, we need to discover sites to find the specific one allSites = await self._discoverSharePointSites() if not allSites: return ActionResult.isFailure(error="No SharePoint sites found or accessible") # If pathQuery starts with Microsoft-standard /sites/, extract site name and filter if pathQuery.startswith('/sites/'): parsedPath = self._extractSiteFromStandardPath(pathQuery) if parsedPath: siteName = parsedPath.get("siteName") # Filter sites by name (case-insensitive substring match) sites = self._filterSitesByHint(allSites, siteName) if not sites: return ActionResult.isFailure(error=f"No SharePoint site found matching '{siteName}'") logger.info(f"Filtered to site(s) matching '{siteName}': {[s['displayName'] for s in sites]}") else: sites = allSites else: sites = allSites else: # Step 3: Both pathObject and pathQuery failed - ERROR, NO FALLBACK return ActionResult.isFailure(error="No valid read path provided. Either provide pathObject (from findDocumentPath) or a valid pathQuery with specific site information.") if not sites: return ActionResult.isFailure(error="No valid target site determined for read operation") # Resolve path query into search paths searchPaths = self._resolvePathQuery(pathQuery) # Process each chat document across all sites readResults = [] for i, chatDocument in enumerate(chatDocuments): try: fileId = chatDocument.fileId fileName = chatDocument.fileName # Search for this file across all sites fileFound = False for site in sites: siteId = site["id"] siteName = site["displayName"] siteUrl = site["webUrl"] # Try to find the file by name in this site searchQuery = fileName.replace("'", "''") # Escape single quotes for OData endpoint = f"sites/{siteId}/drive/root/search(q='{searchQuery}')" searchResult = await self._makeGraphApiCall(endpoint) if "error" in searchResult: continue items = searchResult.get("value", []) for item in items: if item.get("name") == fileName: # Found the file, get its details fileId = item.get("id") fileEndpoint = f"sites/{siteId}/drive/items/{fileId}" # Get file metadata fileInfoResult = await self._makeGraphApiCall(fileEndpoint) if "error" in fileInfoResult: continue # Build result with metadata resultItem = { "fileId": fileId, "fileName": fileName, "sharepointFileId": fileId, "siteName": siteName, "siteUrl": siteUrl, "size": fileInfoResult.get("size", 0), "createdDateTime": fileInfoResult.get("createdDateTime"), "lastModifiedDateTime": fileInfoResult.get("lastModifiedDateTime"), "webUrl": fileInfoResult.get("webUrl") } # Add metadata if requested if includeMetadata: resultItem["metadata"] = { "mimeType": fileInfoResult.get("file", {}).get("mimeType"), "downloadUrl": fileInfoResult.get("@microsoft.graph.downloadUrl"), "createdBy": fileInfoResult.get("createdBy", {}), "lastModifiedBy": fileInfoResult.get("lastModifiedBy", {}), "parentReference": fileInfoResult.get("parentReference", {}) } # Get file content if it's a readable format mimeType = fileInfoResult.get("file", {}).get("mimeType", "") if mimeType.startswith("text/") or mimeType in [ "application/json", "application/xml", "application/javascript" ]: # Download the file content contentEndpoint = f"sites/{siteId}/drive/items/{fileId}/content" # For content download, we need to handle binary data try: async with aiohttp.ClientSession() as session: headers = {"Authorization": f"Bearer {self.services.sharepoint._target.accessToken}"} async with session.get(f"https://graph.microsoft.com/v1.0/{contentEndpoint}", headers=headers) as response: if response.status == 200: content = await response.text() resultItem["content"] = content else: resultItem["content"] = f"Could not download content: HTTP {response.status}" except Exception as e: resultItem["content"] = f"Error downloading content: {str(e)}" else: resultItem["content"] = f"Binary file type ({mimeType}) - content not retrieved" readResults.append(resultItem) fileFound = True break if fileFound: break if not fileFound: readResults.append({ "fileId": fileId, "fileName": fileName, "error": "File not found in any accessible SharePoint site", "content": None }) except Exception as e: logger.error(f"Error reading document {chatDocument.fileName}: {str(e)}") readResults.append({ "fileId": chatDocument.fileId, "fileName": chatDocument.fileName, "error": str(e), "content": None }) resultData = { "connectionReference": connectionReference, "pathQuery": pathQuery, "documentList": documentList, "includeMetadata": includeMetadata, "sitesSearched": len(sites), "readResults": readResults, "connection": { "id": connection["id"], "authority": "microsoft", "reference": connectionReference }, "timestamp": self.services.utils.timestampGetUtc() } # Use default JSON format for output outputExtension = ".json" # Default outputMimeType = "application/json" # Default validationMetadata = { "actionType": "sharepoint.readDocuments", "connectionReference": connectionReference, "documentCount": len(readResults), "includeMetadata": includeMetadata, "sitesSearched": len(sites) } return ActionResult( success=True, documents=[ ActionDocument( documentName=f"sharepoint_documents_{self._format_timestamp_for_filename()}{outputExtension}", documentData=json.dumps(resultData, indent=2), mimeType=outputMimeType, validationMetadata=validationMetadata ) ] ) except Exception as e: logger.error(f"Error reading SharePoint documents: {str(e)}") if operationId: try: self.services.chat.progressLogFinish(operationId, False) except: pass # Don't fail on progress logging errors return ActionResult( success=False, error=str(e) ) @action async def uploadDocument(self, parameters: Dict[str, Any]) -> ActionResult: """ GENERAL: - Purpose: Upload documents to SharePoint. Only to choose this action with a connectionReference - Input requirements: connectionReference (required); documentList (required); optional pathObject or pathQuery. - Output format: JSON with upload status and file info. Parameters: - connectionReference (str, required): Microsoft connection label. - pathObject (str, optional): Reference to a previous path result. - pathQuery (str, optional): Upload target path if no pathObject. - documentList (list, required): Document reference(s) to upload. File names are taken from the documents. """ try: connectionReference = parameters.get("connectionReference") pathQuery = parameters.get("pathQuery") documentList = parameters.get("documentList") if isinstance(documentList, str): documentList = [documentList] pathObject = parameters.get("pathObject") uploadPath = pathQuery logger.debug(f"Using pathQuery: {pathQuery}") if not connectionReference or not documentList: return ActionResult.isFailure(error="Connection reference and document list are required") # If pathObject is provided, extract folder IDs from it if pathObject: try: # Resolve the reference label to get the actual document list from modules.datamodels.datamodelDocref import DocumentReferenceList documentList = self.services.chat.getChatDocumentsFromDocumentList(DocumentReferenceList.from_string_list([pathObject])) if not documentList or len(documentList) == 0: return ActionResult.isFailure(error=f"No document list found for reference: {pathObject}") # Get the first document's content (which should be the JSON) firstDocument = documentList[0] fileData = self.services.chat.getFileData(firstDocument.fileId) if not fileData: return ActionResult.isFailure(error=f"No file data found for document: {pathObject}") # Parse the JSON content resultData = json.loads(fileData) # Debug: Log the structure of the result document logger.info(f"Result document keys: {list(resultData.keys())}") # Handle different result document formats foundDocuments = [] # Check if it's a direct SharePoint result (has foundDocuments) if "foundDocuments" in resultData: foundDocuments = resultData.get("foundDocuments", []) logger.info(f"Found {len(foundDocuments)} documents in foundDocuments array") # Check if it's an AI validation result (has result string with validationReport) elif "result" in resultData and "validationReport" in resultData["result"]: try: # Parse the nested JSON in the result field nestedResult = json.loads(resultData["result"]) validationReport = nestedResult.get("validationReport", {}) documentDetails = validationReport.get("documentDetails", {}) if documentDetails: # Convert the single document details to the expected format doc = { "id": documentDetails.get("id"), "name": documentDetails.get("name"), "type": documentDetails.get("type", "").lower(), # Convert "Folder" to "folder" "siteName": documentDetails.get("siteName"), "siteId": documentDetails.get("siteId"), "fullPath": documentDetails.get("fullPath"), "webUrl": documentDetails.get("webUrl", ""), "parentPath": documentDetails.get("parentPath", "") } foundDocuments = [doc] logger.info(f"Extracted 1 document from validation report") except json.JSONDecodeError as e: logger.error(f"Failed to parse nested JSON in result field: {e}") return ActionResult.isFailure(error=f"Invalid nested JSON in pathObject: {str(e)}") # Debug: Log what we found in the result document logger.info(f"Result document contains {len(foundDocuments)} documents") for i, doc in enumerate(foundDocuments): logger.info(f" Document {i+1}: name='{doc.get('name')}', type='{doc.get('type')}', id='{doc.get('id')}'") # Extract folder information from the result folders = [] for doc in foundDocuments: if doc.get("type") == "folder": folders.append(doc) logger.info(f"Found {len(folders)} folders in result document") if folders: # Use the first folder found - prefer folder ID for direct API calls firstFolder = folders[0] if firstFolder.get("id"): # Use folder ID directly for most reliable API calls uploadPath = firstFolder.get("id") logger.info(f"Using folder ID from pathObject: {uploadPath}") elif firstFolder.get("fullPath"): # Extract the correct path portion from fullPath by removing site name fullPath = firstFolder.get("fullPath") # fullPath format: \\SiteName\\Library\\Folder\\SubFolder # We need to remove the first two parts (\\SiteName\\) to get the actual folder path pathParts = fullPath.lstrip('\\').split('\\') if len(pathParts) > 1: # Remove the first part (site name) and reconstruct the path actualPath = '\\'.join(pathParts[1:]) uploadPath = actualPath logger.info(f"Extracted path from fullPath: {uploadPath}") else: uploadPath = fullPath logger.info(f"Using full path from pathObject (no site name to remove): {uploadPath}") else: return ActionResult.isFailure(error="No valid folder information found in pathObject") else: return ActionResult.isFailure(error="No folders found in pathObject") except json.JSONDecodeError as e: return ActionResult.isFailure(error=f"Invalid JSON in pathObject: {str(e)}") except Exception as e: return ActionResult.isFailure(error=f"Error resolving pathObject reference: {str(e)}") # Get Microsoft connection connection = self._getMicrosoftConnection(connectionReference) if not connection: return ActionResult.isFailure(error="No valid Microsoft connection found for the provided connection reference") # Get documents from reference - ensure documentList is a list, not a string if isinstance(documentList, str): documentList = [documentList] # Convert string to list from modules.datamodels.datamodelDocref import DocumentReferenceList # Convert to DocumentReferenceList if needed if isinstance(documentList, DocumentReferenceList): docRefList = documentList elif isinstance(documentList, list): docRefList = DocumentReferenceList.from_string_list(documentList) elif isinstance(documentList, str): docRefList = DocumentReferenceList.from_string_list([documentList]) else: docRefList = DocumentReferenceList(references=[]) chatDocuments = self.services.chat.getChatDocumentsFromDocumentList(docRefList) if not chatDocuments: return ActionResult.isFailure(error="No documents found for the provided reference") # Determine sites to use based on whether pathObject was provided sites = None if pathObject: # When pathObject is provided, we should have specific site information # Extract site information from the pathObject result try: # Get the site information from the first folder in pathObject if 'foundDocuments' in locals() and foundDocuments: firstFolder = foundDocuments[0] siteName = firstFolder.get("siteName") siteId = firstFolder.get("siteId") if siteName and siteId: # Use the specific site from pathObject instead of discovering all sites sites = [{ "id": siteId, "displayName": siteName, "webUrl": firstFolder.get("webUrl", "") }] logger.info(f"Using specific site from pathObject: {siteName} (ID: {siteId})") else: # Site info missing from pathObject - this is an error, not a fallback return ActionResult.isFailure(error="Site information missing from pathObject. Cannot determine target site for upload.") else: # No documents found in pathObject - this is an error return ActionResult.isFailure(error="No valid folder information found in pathObject. Cannot determine target site for upload.") except Exception as e: # Error processing pathObject - this is an error, not a fallback return ActionResult.isFailure(error=f"Error processing pathObject: {str(e)}. Cannot determine target site for upload.") else: # No pathObject provided - check if pathQuery is valid if not uploadPath or uploadPath.strip() == "" or uploadPath.strip() == "*": return ActionResult.isFailure(error="No valid upload path provided. Either provide pathObject (from findDocumentPath) or a valid pathQuery with specific site information.") # Validate pathQuery format if not uploadPath.startswith('/'): return ActionResult.isFailure(error="pathQuery must start with '/' and include site name with Microsoft-standard syntax /sites//... e.g. /sites/company-share/Freigegebene Dokumente/Work") # Check if uploadPath contains search terms (words without proper path structure) validPathPrefixes = ['/sites/', '/Documents', '/documents', '/Shared Documents', '/shared documents'] if not any(uploadPath.startswith(prefix) for prefix in validPathPrefixes): return ActionResult.isFailure(error=f"Invalid pathQuery '{uploadPath}'. This appears to be search terms, not a valid SharePoint path. Use findDocumentPath action first to search for folders, then use the returned folder path as pathQuery.") # If uploadPath starts with Microsoft-standard /sites/, try to get site directly directSite = None if uploadPath.startswith('/sites/'): parsedPath = self._extractSiteFromStandardPath(uploadPath) if parsedPath: siteName = parsedPath.get("siteName") # Try to get site directly by path (optimization - no need to load all 60 sites) directSite = await self._getSiteByStandardPath(siteName) if directSite: logger.info(f"Got site directly by standard path - no need to discover all sites") sites = [directSite] else: logger.warning(f"Could not get site directly, falling back to site discovery") # If we didn't get the site directly, use discovery and filtering if not directSite: # For pathQuery, we need to discover sites to find the specific one allSites = await self._discoverSharePointSites() if not allSites: return ActionResult.isFailure(error="No SharePoint sites found or accessible") # If uploadPath starts with Microsoft-standard /sites/, extract site name and filter if uploadPath.startswith('/sites/'): parsedPath = self._extractSiteFromStandardPath(uploadPath) if parsedPath: siteName = parsedPath.get("siteName") # Filter sites by name (case-insensitive substring match) sites = self._filterSitesByHint(allSites, siteName) if not sites: return ActionResult.isFailure(error=f"No SharePoint site found matching '{siteName}'") logger.info(f"Filtered to site(s) matching '{siteName}': {[s['displayName'] for s in sites]}") else: sites = allSites else: sites = allSites if not sites: return ActionResult.isFailure(error="No valid target site determined for upload") # Process upload paths based on whether pathObject was provided uploadSiteScope = None if not pathObject: # Parse the validated pathQuery to extract site and path information parsed = self._extractSiteFromStandardPath(uploadPath) if not parsed: return ActionResult.isFailure(error="Invalid uploadPath. Use Microsoft-standard /sites//") # Find matching site (already filtered above, but ensure we have the right one) candidateSites = self._filterSitesByHint(sites, parsed["siteName"]) # substring match # Choose exact displayName match if available exact = [s for s in candidateSites if (s.get("displayName") or "").strip().lower() == parsed["siteName"].strip().lower()] selectedSite = exact[0] if exact else (candidateSites[0] if candidateSites else None) if not selectedSite: return ActionResult.isFailure(error=f"SharePoint site '{parsed['siteName']}' not found or not accessible") uploadSiteScope = selectedSite # Use the inner path portion as the actual upload target path # Remove document library name from path (same logic as listDocuments) innerPath = parsed.get('innerPath', '').lstrip('/') pathSegments = [s for s in innerPath.split('/') if s.strip()] if len(pathSegments) > 1: # Path has multiple segments - first might be a library name # Try without first segment (assuming it's a library name) innerPath = '/'.join(pathSegments[1:]) logger.info(f"Removed first path segment (potential library name), path changed from '{parsed['innerPath']}' to '{innerPath}'") elif len(pathSegments) == 1: # Only one segment - if it's a common library-like name, use empty path (root) firstSegmentLower = pathSegments[0].lower() libraryIndicators = ['document', 'dokument', 'shared', 'freigegeben', 'library', 'bibliothek'] if any(indicator in firstSegmentLower for indicator in libraryIndicators): innerPath = '' logger.info(f"First segment '{pathSegments[0]}' appears to be a library name, using root") uploadPaths = [f"/{innerPath}" if innerPath else "/"] sites = [selectedSite] else: # When using pathObject, check if uploadPath is a folder ID or a path if uploadPath.startswith('01PPXICCB') or uploadPath.startswith('01'): # It's a folder ID - use it directly uploadPaths = [uploadPath] logger.info(f"Using folder ID directly for upload: {uploadPath}") else: # It's a path - resolve it normally uploadPaths = self._resolvePathQuery(uploadPath) # Process each document upload uploadResults = [] # Extract file names from documents fileNames = [doc.fileName for doc in chatDocuments] logger.info(f"Using file names from documentList: {fileNames}") for i, (chatDocument, fileName) in enumerate(zip(chatDocuments, fileNames)): try: fileId = chatDocument.fileId fileData = self.services.chat.getFileData(fileId) if not fileData: logger.warning(f"File data not found for fileId: {fileId}") uploadResults.append({ "fileName": fileName, "fileId": fileId, "error": "File data not found", "uploadStatus": "failed" }) continue # Upload to the first available site (or could be made configurable) uploadSuccessful = False for site in sites: siteId = site["id"] siteName = site["displayName"] siteUrl = site["webUrl"] # Use the first upload path or default to Documents uploadPath = uploadPaths[0] if uploadPaths else "/Documents" # Handle wildcard paths - replace with default Documents folder if uploadPath == "*": uploadPath = "/Documents" logger.warning(f"Wildcard path '*' detected, using default '/Documents' folder for upload") # Check if uploadPath is a folder ID or a regular path if uploadPath.startswith('01PPXICCB') or uploadPath.startswith('01'): # It's a folder ID - use the folder-specific upload endpoint uploadEndpoint = f"sites/{siteId}/drive/items/{uploadPath}:/{fileName}:/content" logger.info(f"Using folder ID upload endpoint: {uploadEndpoint}") else: # It's a regular path - use the root-based upload endpoint uploadPath = uploadPath.rstrip('/') + '/' + fileName uploadPathClean = uploadPath.lstrip('/') uploadEndpoint = f"sites/{siteId}/drive/root:/{uploadPathClean}:/content" logger.info(f"Using path-based upload endpoint: {uploadEndpoint}") # Upload endpoint for small files (< 4MB) if len(fileData) < 4 * 1024 * 1024: # 4MB # Upload the file uploadResult = await self._makeGraphApiCall( uploadEndpoint, method="PUT", data=fileData ) if "error" not in uploadResult: uploadResults.append({ "fileName": fileName, "fileId": fileId, "uploadStatus": "success", "siteName": siteName, "siteUrl": siteUrl, "uploadPath": uploadPath, "uploadEndpoint": uploadEndpoint, "sharepointFileId": uploadResult.get("id"), "webUrl": uploadResult.get("webUrl"), "size": uploadResult.get("size"), "createdDateTime": uploadResult.get("createdDateTime") }) uploadSuccessful = True break else: logger.warning(f"Upload failed to site {siteName}: {uploadResult['error']}") else: # For large files, we would need to implement resumable upload logger.warning(f"File too large ({len(fileData)} bytes) for site {siteName}") continue if not uploadSuccessful: uploadResults.append({ "fileName": fileName, "fileId": fileId, "error": f"File too large ({len(fileData)} bytes) or upload failed to all sites. Files larger than 4MB require resumable upload (not implemented).", "uploadStatus": "failed" }) except Exception as e: logger.error(f"Error uploading document {fileName}: {str(e)}") uploadResults.append({ "fileName": fileName, "fileId": fileId, "error": str(e), "uploadStatus": "failed" }) # Create result data resultData = { "connectionReference": connectionReference, "pathQuery": uploadPath, "documentList": documentList, "fileNames": fileNames, "sitesAvailable": len(sites), "uploadResults": uploadResults, "connection": { "id": connection["id"], "authority": "microsoft", "reference": connectionReference }, "timestamp": self.services.utils.timestampGetUtc() } # Use default JSON format for output outputExtension = ".json" # Default outputMimeType = "application/json" # Default validationMetadata = { "actionType": "sharepoint.uploadDocument", "connectionReference": connectionReference, "uploadPath": uploadPath, "fileNames": fileNames, "uploadCount": len(uploadResults), "successfulUploads": len([r for r in uploadResults if r.get("uploadStatus") == "success"]), "failedUploads": len([r for r in uploadResults if r.get("uploadStatus") == "failed"]) } return ActionResult( success=True, documents=[ ActionDocument( documentName=f"sharepoint_upload_{self._format_timestamp_for_filename()}{outputExtension}", documentData=json.dumps(resultData, indent=2), mimeType=outputMimeType, validationMetadata=validationMetadata ) ] ) except Exception as e: logger.error(f"Error uploading to SharePoint: {str(e)}") return ActionResult( success=False, error=str(e) ) @action async def listDocuments(self, parameters: Dict[str, Any]) -> ActionResult: """ GENERAL: - Purpose: List documents and folders in SharePoint paths across sites. - Input requirements: connectionReference (required); optional pathObject or pathQuery; includeSubfolders. - Output format: JSON with folder items and metadata. Parameters: - connectionReference (str, required): Microsoft connection label. - pathObject (str, optional): Reference to a previous path result. - pathQuery (str, optional): Path query if no pathObject. - includeSubfolders (bool, optional): Include one level of subfolders. Default: False. """ try: connectionReference = parameters.get("connectionReference") pathObject = parameters.get("pathObject") pathQuery = parameters.get("pathQuery") includeSubfolders = parameters.get("includeSubfolders", False) # Default to False for better UX listQuery = pathQuery logger.info(f"Using pathQuery: {pathQuery}") if not connectionReference: return ActionResult.isFailure(error="Connection reference is required") # If pathObject is provided, resolve the reference and extract folder IDs from it # Note: pathObject takes precedence over pathQuery when both are provided if pathObject: if pathQuery and pathQuery != "*": logger.debug(f"Both pathObject and pathQuery provided - using pathObject (pathQuery '{pathQuery}' will be ignored)") try: # Resolve the reference label to get the actual document list from modules.datamodels.datamodelDocref import DocumentReferenceList documentList = self.services.chat.getChatDocumentsFromDocumentList(DocumentReferenceList.from_string_list([pathObject])) if not documentList or len(documentList) == 0: return ActionResult.isFailure(error=f"No document list found for reference: {pathObject}") # Get the first document's content (which should be the JSON) firstDocument = documentList[0] logger.info(f"Document fileId: {firstDocument.fileId}, fileName: {firstDocument.fileName}") fileData = self.services.chat.getFileData(firstDocument.fileId) if not fileData: return ActionResult.isFailure(error=f"No file data found for document: {pathObject} (fileId: {firstDocument.fileId})") logger.info(f"File data length: {len(fileData) if fileData else 0}") # Parse the JSON content resultData = json.loads(fileData) # Debug: Log the structure of the result document logger.info(f"Result document keys: {list(resultData.keys())}") # Handle different result document formats foundDocuments = [] # Check if it's a direct SharePoint result (has foundDocuments) if "foundDocuments" in resultData: foundDocuments = resultData.get("foundDocuments", []) logger.info(f"Found {len(foundDocuments)} documents in foundDocuments array") # Check if it's an AI validation result (has result string with validationReport) elif "result" in resultData and "validationReport" in resultData["result"]: try: # Parse the nested JSON in the result field nestedResult = json.loads(resultData["result"]) validationReport = nestedResult.get("validationReport", {}) documentDetails = validationReport.get("documentDetails", {}) if documentDetails: # Convert the single document details to the expected format doc = { "id": documentDetails.get("id"), "name": documentDetails.get("name"), "type": documentDetails.get("type", "").lower(), # Convert "Folder" to "folder" "siteName": documentDetails.get("siteName"), "siteId": documentDetails.get("siteId"), "fullPath": documentDetails.get("fullPath"), "webUrl": documentDetails.get("webUrl", ""), "parentPath": documentDetails.get("parentPath", "") } foundDocuments = [doc] logger.info(f"Extracted 1 document from validation report") except ValueError as e: logger.error(f"Failed to parse nested JSON in result field: {e}") return ActionResult.isFailure(error=f"Invalid nested JSON in pathObject: {str(e)}") # Debug: Log what we found in the result document logger.info(f"Result document contains {len(foundDocuments)} documents") for i, doc in enumerate(foundDocuments): logger.info(f" Document {i+1}: name='{doc.get('name')}', type='{doc.get('type')}', id='{doc.get('id')}'") # Extract folder information from the result folders = [] for doc in foundDocuments: if doc.get("type") == "folder": folders.append(doc) logger.info(f"Found {len(folders)} folders in result document") if folders: # Use the first folder found - prefer folder ID for direct API calls firstFolder = folders[0] if firstFolder.get("id"): # Use folder ID directly for most reliable API calls listQuery = firstFolder.get("id") logger.info(f"Using folder ID from pathObject: {listQuery}") elif firstFolder.get("fullPath"): # Extract the correct path portion from fullPath by removing site name fullPath = firstFolder.get("fullPath") # fullPath format: \\SiteName\\Library\\Folder\\SubFolder # We need to remove the first two parts (\\SiteName\\) to get the actual folder path pathParts = fullPath.lstrip('\\').split('\\') if len(pathParts) > 1: # Remove the first part (site name) and reconstruct the path actualPath = '\\'.join(pathParts[1:]) listQuery = actualPath logger.info(f"Extracted path from fullPath: {listQuery}") else: listQuery = fullPath logger.info(f"Using full path from pathObject (no site name to remove): {listQuery}") else: return ActionResult.isFailure(error="No valid folder information found in pathObject") else: return ActionResult.isFailure(error="No folders found in pathObject") except ValueError as e: return ActionResult.isFailure(error=f"Invalid JSON in pathObject: {str(e)}") except Exception as e: return ActionResult.isFailure(error=f"Error resolving pathObject reference: {str(e)}") # Get Microsoft connection connection = self._getMicrosoftConnection(connectionReference) if not connection: return ActionResult.isFailure(error="No valid Microsoft connection found for the provided connection reference") logger.info(f"Starting SharePoint listDocuments for listQuery: {listQuery}") logger.debug(f"Connection ID: {connection['id']}") # Parse listQuery to extract path, search terms, search type, and options pathQuery, fileQuery, searchType, searchOptions = self._parseSearchQuery(listQuery) # Determine sites to use - strict validation: pathObject → pathQuery → ERROR sites = None # Step 1: Check pathObject first if pathObject: # When pathObject is provided, we should have specific site information # Extract site information from the pathObject result try: # Get the site information from the first folder in pathObject if 'foundDocuments' in locals() and foundDocuments: firstFolder = foundDocuments[0] siteName = firstFolder.get("siteName") siteId = firstFolder.get("siteId") if siteName and siteId: # Use the specific site from pathObject instead of discovering all sites sites = [{ "id": siteId, "displayName": siteName, "webUrl": firstFolder.get("webUrl", "") }] logger.info(f"Using specific site from pathObject: {siteName} (ID: {siteId})") else: # Site info missing from pathObject - this is an error return ActionResult.isFailure(error="Site information missing from pathObject. Cannot determine target site for list operation.") else: # No documents found in pathObject - this is an error return ActionResult.isFailure(error="No valid folder information found in pathObject. Cannot determine target site for list operation.") except Exception as e: # Error processing pathObject - this is an error return ActionResult.isFailure(error=f"Error processing pathObject: {str(e)}. Cannot determine target site for list operation.") # Step 2: If no pathObject, check pathQuery elif pathQuery and pathQuery.strip() != "" and pathQuery.strip() != "*": # Validate pathQuery format if not pathQuery.startswith('/'): return ActionResult.isFailure(error="pathQuery must start with '/' and include site name with Microsoft-standard syntax /sites//... e.g. /sites/company-share/Freigegebene Dokumente/Work") # Check if pathQuery contains search terms (words without proper path structure) validPathPrefixes = ['/sites/', '/Documents', '/documents', '/Shared Documents', '/shared documents'] if not any(pathQuery.startswith(prefix) for prefix in validPathPrefixes): return ActionResult.isFailure(error=f"Invalid pathQuery '{pathQuery}'. This appears to be search terms, not a valid SharePoint path. Use findDocumentPath action first to search for folders, then use the returned folder path as pathQuery.") # If pathQuery starts with Microsoft-standard /sites/, try to get site directly directSite = None if pathQuery.startswith('/sites/'): parsedPath = self._extractSiteFromStandardPath(pathQuery) if parsedPath: siteName = parsedPath.get("siteName") # Try to get site directly by path (optimization - no need to load all 60 sites) directSite = await self._getSiteByStandardPath(siteName) if directSite: logger.info(f"Got site directly by standard path - no need to discover all sites") sites = [directSite] else: logger.warning(f"Could not get site directly, falling back to site discovery") # If we didn't get the site directly, use discovery and filtering if not directSite: # For pathQuery, we need to discover sites to find the specific one allSites = await self._discoverSharePointSites() if not allSites: return ActionResult.isFailure(error="No SharePoint sites found or accessible") # If pathQuery starts with Microsoft-standard /sites/, extract site name and filter if pathQuery.startswith('/sites/'): parsedPath = self._extractSiteFromStandardPath(pathQuery) if parsedPath: siteName = parsedPath.get("siteName") # Filter sites by name (case-insensitive substring match) sites = self._filterSitesByHint(allSites, siteName) if not sites: return ActionResult.isFailure(error=f"No SharePoint site found matching '{siteName}'") logger.info(f"Filtered to site(s) matching '{siteName}': {[s['displayName'] for s in sites]}") else: sites = allSites else: sites = allSites else: # Step 3: Both pathObject and pathQuery failed - ERROR, NO FALLBACK return ActionResult.isFailure(error="No valid list path provided. Either provide pathObject (from findDocumentPath) or a valid pathQuery with specific site information.") if not sites: return ActionResult.isFailure(error="No valid target site determined for list operation") # Check if listQuery is a folder ID (starts with 01PPXICCB...) if listQuery.startswith('01PPXICCB') or listQuery.startswith('01'): # Direct folder ID - use it directly folderPaths = [listQuery] logger.info(f"Using direct folder ID: {listQuery}") else: # Remove site prefix from pathQuery before resolving (it's only for site filtering) pathQueryForResolve = pathQuery # Microsoft-standard path: /sites/SiteName/Path -> /Path if pathQuery.startswith('/sites/'): parsedPath = self._extractSiteFromStandardPath(pathQuery) if parsedPath: innerPath = parsedPath.get("innerPath", "") pathQueryForResolve = '/' + innerPath if innerPath else '/' else: pathQueryForResolve = '/' # Remove first path segment if it looks like a document library name # In SharePoint Graph API, /drive/root already points to the default document library, # so library names in paths should be removed # Generic approach: if path has multiple segments, store original for fallback pathSegments = [s for s in pathQueryForResolve.split('/') if s.strip()] if len(pathSegments) > 1: # Path has multiple segments - first might be a library name # Store original for potential fallback originalPath = pathQueryForResolve # Try without first segment (assuming it's a library name) pathQueryForResolve = '/' + '/'.join(pathSegments[1:]) logger.info(f"Removed first path segment (potential library name), path changed from '{originalPath}' to '{pathQueryForResolve}'") elif len(pathSegments) == 1: # Only one segment - if it's a common library-like name, use root firstSegmentLower = pathSegments[0].lower() libraryIndicators = ['document', 'dokument', 'shared', 'freigegeben', 'library', 'bibliothek'] if any(indicator in firstSegmentLower for indicator in libraryIndicators): pathQueryForResolve = '/' logger.info(f"First segment '{pathSegments[0]}' appears to be a library name, using root") # Resolve path query into folder paths folderPaths = self._resolvePathQuery(pathQueryForResolve) logger.info(f"Resolved folder paths: {folderPaths}") # Process each folder path across all sites listResults = [] for folderPath in folderPaths: try: folderResults = [] for site in sites: siteId = site["id"] siteName = site["displayName"] siteUrl = site["webUrl"] logger.info(f"Listing folder {folderPath} in site: {siteName}") # Determine the endpoint based on folder path if folderPath in ["/", ""] or folderPath == "*": # Root folder endpoint = f"sites/{siteId}/drive/root/children" elif folderPath.startswith('01PPXICCB') or folderPath.startswith('01'): # Direct folder ID endpoint = f"sites/{siteId}/drive/items/{folderPath}/children" else: # Specific folder path - remove leading slash if present and URL encode folderPathClean = folderPath.lstrip('/') # URL encode the path for Graph API (spaces and special characters need encoding) folderPathEncoded = urllib.parse.quote(folderPathClean, safe='/') endpoint = f"sites/{siteId}/drive/root:/{folderPathEncoded}:/children" # Make the API call to list folder contents apiResult = await self._makeGraphApiCall(endpoint) if "error" in apiResult: logger.warning(f"Failed to list folder {folderPath} in site {siteName}: {apiResult['error']}") continue # Process the results items = apiResult.get("value", []) processedItems = [] for item in items: # Use improved folder detection logic isFolder = False if 'folder' in item: isFolder = True else: # Try to detect by URL pattern or other indicators webUrl = item.get('webUrl', '') name = item.get('name', '') # Check if URL has no file extension and looks like a folder path if '.' not in name and ('/' in webUrl or '\\' in webUrl): isFolder = True itemInfo = { "id": item.get("id"), "name": item.get("name"), "size": item.get("size", 0), "createdDateTime": item.get("createdDateTime"), "lastModifiedDateTime": item.get("lastModifiedDateTime"), "webUrl": item.get("webUrl"), "type": "folder" if isFolder else "file", "siteName": siteName, "siteUrl": siteUrl } # Add file-specific information if "file" in item: itemInfo.update({ "mimeType": item["file"].get("mimeType"), "downloadUrl": item.get("@microsoft.graph.downloadUrl") }) # Add folder-specific information if "folder" in item: itemInfo.update({ "childCount": item["folder"].get("childCount", 0) }) processedItems.append(itemInfo) # If include subfolders is enabled, get ONLY direct subfolder contents (1 level deep only) if includeSubfolders: folderItems = [item for item in processedItems if item['type'] == 'folder'] logger.info(f"Including subfolders - processing {len(folderItems)} folders") subfolderCount = 0 maxSubfolders = 10 # Limit to prevent infinite loops for item in processedItems[:]: # Use slice to avoid modifying list during iteration if item["type"] == "folder" and subfolderCount < maxSubfolders: subfolderCount += 1 subfolderPath = f"{folderPath.rstrip('/')}/{item['name']}" subfolderEndpoint = f"sites/{siteId}/drive/items/{item['id']}/children" logger.debug(f"Getting contents of subfolder: {item['name']}") subfolderResult = await self._makeGraphApiCall(subfolderEndpoint) if "error" not in subfolderResult: subfolderItems = subfolderResult.get("value", []) logger.debug(f"Found {len(subfolderItems)} items in subfolder {item['name']}") for subfolderItem in subfolderItems: # Use improved folder detection logic for subfolder items subfolderIsFolder = False if 'folder' in subfolderItem: subfolderIsFolder = True else: # Try to detect by URL pattern or other indicators subfolderWebUrl = subfolderItem.get('webUrl', '') subfolderName = subfolderItem.get('name', '') # Check if URL has no file extension and looks like a folder path if '.' not in subfolderName and ('/' in subfolderWebUrl or '\\' in subfolderWebUrl): subfolderIsFolder = True # Only add files and direct subfolders, NO RECURSION subfolderItemInfo = { "id": subfolderItem.get("id"), "name": subfolderItem.get("name"), "size": subfolderItem.get("size", 0), "createdDateTime": subfolderItem.get("createdDateTime"), "lastModifiedDateTime": subfolderItem.get("lastModifiedDateTime"), "webUrl": subfolderItem.get("webUrl"), "type": "folder" if subfolderIsFolder else "file", "parentPath": subfolderPath, "siteName": siteName, "siteUrl": siteUrl } if "file" in subfolderItem: subfolderItemInfo.update({ "mimeType": subfolderItem["file"].get("mimeType"), "downloadUrl": subfolderItem.get("@microsoft.graph.downloadUrl") }) processedItems.append(subfolderItemInfo) else: logger.warning(f"Failed to get contents of subfolder {item['name']}: {subfolderResult.get('error')}") elif subfolderCount >= maxSubfolders: logger.warning(f"Reached maximum subfolder limit ({maxSubfolders}), skipping remaining folders") break logger.info(f"Processed {subfolderCount} subfolders, total items: {len(processedItems)}") folderResults.append({ "siteName": siteName, "siteUrl": siteUrl, "itemCount": len(processedItems), "items": processedItems }) listResults.append({ "folderPath": folderPath, "sitesProcessed": len(folderResults), "siteResults": folderResults }) except Exception as e: logger.error(f"Error listing folder {folderPath}: {str(e)}") listResults.append({ "folderPath": folderPath, "error": str(e), "siteResults": [] }) # Create result data resultData = { "pathQuery": listQuery, "includeSubfolders": includeSubfolders, "sitesSearched": len(sites), "listResults": listResults, "timestamp": self.services.utils.timestampGetUtc() } # Use default JSON format for output outputExtension = ".json" # Default outputMimeType = "application/json" # Default validationMetadata = { "actionType": "sharepoint.listDocuments", "pathQuery": listQuery, "includeSubfolders": includeSubfolders, "sitesSearched": len(sites), "folderCount": len(listResults), "totalItems": sum(len(result.get("siteResults", [])) for result in listResults) } return ActionResult( success=True, documents=[ ActionDocument( documentName=f"sharepoint_document_list_{self._format_timestamp_for_filename()}{outputExtension}", documentData=json.dumps(resultData, indent=2), mimeType=outputMimeType, validationMetadata=validationMetadata ) ] ) except Exception as e: logger.error(f"Error listing SharePoint documents: {str(e)}") return ActionResult( success=False, error=str(e) )