From fc662fbe59d00209d5e88f8df117dca80a829f20 Mon Sep 17 00:00:00 2001 From: ValueOn AG Date: Fri, 30 May 2025 03:41:24 +0200 Subject: [PATCH] root, files, prompts running --- modules/interfaces/serviceChatClass.py | 21 --- modules/interfaces/serviceManagementClass.py | 167 +++++++++++++++---- modules/interfaces/serviceManagementModel.py | 45 ++++- modules/routes/routeDataFiles.py | 94 ++++++++++- modules/routes/routeWorkflows.py | 69 -------- 5 files changed, 265 insertions(+), 131 deletions(-) diff --git a/modules/interfaces/serviceChatClass.py b/modules/interfaces/serviceChatClass.py index 562d61d8..dc7c823c 100644 --- a/modules/interfaces/serviceChatClass.py +++ b/modules/interfaces/serviceChatClass.py @@ -31,27 +31,6 @@ logger = logging.getLogger(__name__) # Singleton factory for Chat instances with AI service per context _chatInterfaces = {} -# Custom exceptions for file handling -class FileError(Exception): - """Base class for file handling exceptions.""" - pass - -class FileNotFoundError(FileError): - """Exception raised when a file is not found.""" - pass - -class FileStorageError(FileError): - """Exception raised when there's an error storing a file.""" - pass - -class FilePermissionError(FileError): - """Exception raised when there's a permission issue with a file.""" - pass - -class FileDeletionError(FileError): - """Exception raised when there's an error deleting a file.""" - pass - class ChatInterface: """ Interface to Chat database and AI Connectors. diff --git a/modules/interfaces/serviceManagementClass.py b/modules/interfaces/serviceManagementClass.py index 27663604..4d398eeb 100644 --- a/modules/interfaces/serviceManagementClass.py +++ b/modules/interfaces/serviceManagementClass.py @@ -323,7 +323,7 @@ class ServiceManagement: raise PermissionError("No permission to create prompts") # Create prompt record - createdRecord = self.db.recordCreate("prompts", promptData.to_dict()) + createdRecord = self.db.recordCreate("prompts", promptData) if not createdRecord or not createdRecord.get("id"): raise ValueError("Failed to create prompt record") @@ -369,15 +369,23 @@ class ServiceManagement: """Calculates a SHA-256 hash for the file content""" return hashlib.sha256(fileContent).hexdigest() - def checkForDuplicateFile(self, fileHash: str) -> Optional[Dict[str, Any]]: + def checkForDuplicateFile(self, fileHash: str) -> Optional[FileItem]: """Checks if a file with the same hash already exists for the current user and mandate.""" files = self.db.getRecordset("files", recordFilter={ "fileHash": fileHash, - "mandateId": self.currentUser.get("mandateId"), - "_createdBy": self.currentUser.get("id") + "mandateId": self.currentUser.mandateId, + "_createdBy": self.currentUser.id }) if files: - return files[0] + return FileItem( + id=files[0]["id"], + mandateId=files[0]["mandateId"], + filename=files[0]["filename"], + mimeType=files[0]["mimeType"], + workflowId=files[0]["workflowId"], + fileHash=files[0]["fileHash"], + fileSize=files[0]["fileSize"] + ) return None def getMimeType(self, filename: str) -> str: @@ -412,34 +420,85 @@ class ServiceManagement: # File methods - metadata-based operations - def getAllFiles(self) -> List[Dict[str, Any]]: + def getAllFiles(self) -> List[FileItem]: """Returns files based on user access level.""" allFiles = self.db.getRecordset("files") - return self._uam("files", allFiles) + filteredFiles = self._uam("files", allFiles) + + # Convert database records to FileItem instances + fileItems = [] + for file in filteredFiles: + try: + # Get creation date from record or use current time + creationDate = file.get("creationDate") + if not creationDate: + creationDate = datetime.now().isoformat() + + fileItem = FileItem( + id=file.get("id"), + mandateId=file.get("mandateId"), + filename=file.get("filename"), + mimeType=file.get("mimeType"), + workflowId=file.get("workflowId"), + fileHash=file.get("fileHash"), + fileSize=file.get("fileSize"), + creationDate=creationDate + ) + fileItems.append(fileItem) + except Exception as e: + logger.warning(f"Skipping invalid file record: {str(e)}") + continue + + return fileItems - def getFile(self, fileId: str) -> Optional[Dict[str, Any]]: + def getFile(self, fileId: str) -> Optional[FileItem]: """Returns a file by ID if user has access.""" files = self.db.getRecordset("files", recordFilter={"id": fileId}) if not files: return None filteredFiles = self._uam("files", files) - return filteredFiles[0] if filteredFiles else None + if not filteredFiles: + return None + + file = filteredFiles[0] + try: + # Get creation date from record or use current time + creationDate = file.get("creationDate") + if not creationDate: + creationDate = datetime.now().isoformat() + + return FileItem( + id=file.get("id"), + mandateId=file.get("mandateId"), + filename=file.get("filename"), + mimeType=file.get("mimeType"), + workflowId=file.get("workflowId"), + fileHash=file.get("fileHash"), + fileSize=file.get("fileSize"), + creationDate=creationDate + ) + except Exception as e: + logger.error(f"Error converting file record: {str(e)}") + return None - def createFile(self, name: str, mimeType: str, size: int = None, fileHash: str = None) -> Dict[str, Any]: + def createFile(self, name: str, mimeType: str, size: int = None, fileHash: str = None) -> FileItem: """Creates a new file entry if user has permission.""" if not self._canModify("files"): raise PermissionError("No permission to create files") - fileData = { - "mandateId": self.currentUser.get("mandateId"), - "name": name, - "mimeType": mimeType, - "size": size, - "fileHash": fileHash, - "creationDate": self._getCurrentTimestamp() - } - return self.db.recordCreate("files", fileData) + # Create FileItem instance + fileItem = FileItem( + mandateId=self.currentUser.mandateId, + filename=name, + mimeType=mimeType, + fileSize=size, + fileHash=fileHash + ) + + # Store in database + self.db.recordCreate("files", fileItem.to_dict()) + return fileItem def updateFile(self, fileId: str, updateData: Dict[str, Any]) -> Dict[str, Any]: """Updates file metadata if user has access.""" @@ -467,10 +526,10 @@ class ServiceManagement: raise PermissionError(f"No permission to delete file {fileId}") # Check for other references to this file (by hash) - fileHash = file.get("fileHash") + fileHash = file.fileHash if fileHash: otherReferences = [f for f in self.db.getRecordset("files", recordFilter={"fileHash": fileHash}) - if f.get("id") != fileId] + if f["id"] != fileId] # Only delete associated fileData if no other references exist if not otherReferences: @@ -507,7 +566,7 @@ class ServiceManagement: return False # Determine if this is a text-based format - mimeType = file.get("mimeType", "application/octet-stream") + mimeType = file.mimeType isTextFormat = isTextMimeType(mimeType) base64Encoded = False @@ -581,6 +640,52 @@ class ServiceManagement: logger.error(f"Error processing file data for {fileId}: {str(e)}") return None + def getFilePreview(self, fileId: str) -> Optional[Dict[str, Any]]: + """Returns a preview of the file content if user has access.""" + try: + # Get file metadata + file = self.getFile(fileId) + if not file: + logger.warning(f"No access to file ID {fileId}") + return None + + # Get file content + fileContent = self.getFileData(fileId) + if not fileContent: + logger.warning(f"No content found for file ID {fileId}") + return None + + # Determine if content is text based on MIME type + isText = file.mimeType.startswith(('text/', 'application/json', 'application/xml', 'application/javascript')) + + # For text content, decode to string + if isText: + try: + content = fileContent.decode('utf-8') + encoding = 'utf-8' + except UnicodeDecodeError: + try: + content = fileContent.decode('latin-1') + encoding = 'latin-1' + except: + content = fileContent + encoding = None + else: + content = fileContent + encoding = None + + return { + "content": content, + "mimeType": file.mimeType, + "filename": file.filename, + "isText": isText, + "encoding": encoding, + "size": len(fileContent) + } + except Exception as e: + logger.error(f"Error getting file preview for {fileId}: {str(e)}") + return None + def updateFileData(self, fileId: str, data: Union[bytes, str]) -> bool: """Updates file data if user has access.""" # Check file access @@ -597,7 +702,7 @@ class ServiceManagement: import base64 # Determine if this is a text-based format - mimeType = file.get("mimeType", "application/octet-stream") + mimeType = file.mimeType isTextFormat = isTextMimeType(mimeType) base64Encoded = False @@ -667,7 +772,7 @@ class ServiceManagement: logger.error(f"Error updating data for file {fileId}: {str(e)}") return False - def saveUploadedFile(self, fileContent: bytes, fileName: str) -> Dict[str, Any]: + def saveUploadedFile(self, fileContent: bytes, fileName: str) -> FileItem: """Saves an uploaded file if user has permission.""" try: # Check file creation permission @@ -687,7 +792,7 @@ class ServiceManagement: # Check for duplicate within same user/mandate existingFile = self.checkForDuplicateFile(fileHash) if existingFile: - logger.debug(f"Duplicate found for {fileName}: {existingFile['id']}") + logger.debug(f"Duplicate found for {fileName}: {existingFile.id}") return existingFile # Determine MIME type and size @@ -696,7 +801,7 @@ class ServiceManagement: # Save metadata logger.debug(f"Saving file metadata to database for file: {fileName}") - dbFile = self.createFile( + fileItem = self.createFile( name=fileName, mimeType=mimeType, size=fileSize, @@ -705,10 +810,10 @@ class ServiceManagement: # Save binary data logger.debug(f"Saving file content to database for file: {fileName}") - self.createFileData(dbFile["id"], fileContent) + self.createFileData(fileItem.id, fileContent) logger.debug(f"File upload process completed for: {fileName}") - return dbFile + return fileItem except Exception as e: logger.error(f"Error in saveUploadedFile for {fileName}: {str(e)}", exc_info=True) @@ -731,9 +836,9 @@ class ServiceManagement: return { "id": fileId, - "name": file.get("name", f"file_{fileId}"), - "contentType": file.get("mimeType", "application/octet-stream"), - "size": file.get("size", len(fileContent)), + "name": file.filename, + "contentType": file.mimeType, + "size": file.fileSize, "content": fileContent } except FileNotFoundError as e: diff --git a/modules/interfaces/serviceManagementModel.py b/modules/interfaces/serviceManagementModel.py index 1a7195a8..d8756542 100644 --- a/modules/interfaces/serviceManagementModel.py +++ b/modules/interfaces/serviceManagementModel.py @@ -4,7 +4,7 @@ Updated to match the Entity Relation Diagram structure. """ from pydantic import BaseModel, Field -from typing import List, Dict, Any, Optional +from typing import List, Dict, Any, Optional, Union from datetime import datetime import uuid @@ -22,6 +22,14 @@ class FileItem(BaseModel, ModelMixin): workflowId: Optional[str] = Field(None, description="Foreign key to workflow") fileHash: str = Field(description="Hash of the file") fileSize: int = Field(description="Size of the file in bytes") + creationDate: str = Field(default_factory=lambda: datetime.now().isoformat(), description="Date when the file was created") + + def to_dict(self) -> Dict[str, Any]: + """Convert model to dictionary with proper datetime handling""" + data = super().to_dict() + if isinstance(data.get("creationDate"), datetime): + data["creationDate"] = data["creationDate"].isoformat() + return data # Register labels for FileItem register_model_labels( @@ -34,7 +42,40 @@ register_model_labels( "mimeType": {"en": "MIME Type", "fr": "Type MIME"}, "workflowId": {"en": "Workflow ID", "fr": "ID du flux de travail"}, "fileHash": {"en": "File Hash", "fr": "Hash du fichier"}, - "fileSize": {"en": "File Size", "fr": "Taille du fichier"} + "fileSize": {"en": "File Size", "fr": "Taille du fichier"}, + "creationDate": {"en": "Creation Date", "fr": "Date de création"} + } +) + +class FilePreview(BaseModel, ModelMixin): + """Data model for file preview""" + content: Union[str, bytes] = Field(description="File content (text or binary)") + mimeType: str = Field(description="MIME type of the file") + filename: str = Field(description="Original filename") + isText: bool = Field(description="Whether the content is text (True) or binary (False)") + encoding: Optional[str] = Field(None, description="Text encoding if content is text") + size: int = Field(description="Size of the content in bytes") + + def to_dict(self) -> Dict[str, Any]: + """Convert model to dictionary with proper content handling""" + data = super().to_dict() + # Convert bytes to base64 string if content is binary + if isinstance(data.get("content"), bytes): + import base64 + data["content"] = base64.b64encode(data["content"]).decode('utf-8') + return data + +# Register labels for FilePreview +register_model_labels( + "FilePreview", + {"en": "File Preview", "fr": "Aperçu du fichier"}, + { + "content": {"en": "Content", "fr": "Contenu"}, + "mimeType": {"en": "MIME Type", "fr": "Type MIME"}, + "filename": {"en": "Filename", "fr": "Nom de fichier"}, + "isText": {"en": "Is Text", "fr": "Est du texte"}, + "encoding": {"en": "Encoding", "fr": "Encodage"}, + "size": {"en": "Size", "fr": "Taille"} } ) diff --git a/modules/routes/routeDataFiles.py b/modules/routes/routeDataFiles.py index 43bc7108..7860fec3 100644 --- a/modules/routes/routeDataFiles.py +++ b/modules/routes/routeDataFiles.py @@ -1,6 +1,6 @@ from fastapi import APIRouter, HTTPException, Depends, File, UploadFile, Form, Path, Request, status, Query, Response, Body from fastapi.responses import JSONResponse, FileResponse -from typing import List, Dict, Any, Optional +from typing import List, Dict, Any, Optional, Union import logging from datetime import datetime, timezone from dataclasses import dataclass @@ -15,7 +15,7 @@ from modules.security.auth import limiter, getCurrentUser # Import interfaces import modules.interfaces.serviceManagementClass as serviceManagementClass -from modules.interfaces.serviceManagementModel import FileItem +from modules.interfaces.serviceManagementModel import FileItem, FilePreview from modules.shared.attributeUtils import getModelAttributeDefinitions, AttributeResponse, AttributeDefinition from modules.interfaces.serviceAppModel import User @@ -50,7 +50,9 @@ async def get_files( # Get all files generically - only metadata, no binary data files = managementInterface.getAllFiles() - return [FileItem(**file) for file in files] + + # Return files directly since they are already FileItem objects + return files except Exception as e: logger.error(f"Error getting files: {str(e)}") raise HTTPException( @@ -82,13 +84,16 @@ async def upload_file( ) # Save file via LucyDOM interface in the database - fileMeta = managementInterface.saveUploadedFile(fileContent, file.filename) + fileItem = managementInterface.saveUploadedFile(fileContent, file.filename) # If workflowId is provided, update the file information if workflowId: updateData = {"workflowId": workflowId} - managementInterface.updateFile(fileMeta["id"], updateData) - fileMeta["workflowId"] = workflowId + managementInterface.updateFile(fileItem.id, updateData) + fileItem.workflowId = workflowId + + # Convert FileItem to dictionary for JSON response + fileMeta = fileItem.to_dict() # Successful response return JSONResponse({ @@ -245,12 +250,12 @@ async def get_file_stats( # Calculate statistics totalFiles = len(allFiles) - totalSize = sum(file.get("size", 0) for file in allFiles) + totalSize = sum(file.fileSize for file in allFiles) # Group by file type fileTypes = {} for file in allFiles: - fileType = file.get("mimeType", "unknown").split("/")[0] + fileType = file.mimeType.split("/")[0] if fileType not in fileTypes: fileTypes[fileType] = 0 fileTypes[fileType] += 1 @@ -268,3 +273,76 @@ async def get_file_stats( detail=f"Error retrieving file statistics: {str(e)}" ) +@router.get("/{fileId}/download") +@limiter.limit("30/minute") +async def download_file( + request: Request, + fileId: str = Path(..., description="ID of the file to download"), + currentUser: User = Depends(getCurrentUser) +) -> Response: + """Download a file""" + try: + managementInterface = serviceManagementClass.getInterface(currentUser) + + # Get file data + fileData = managementInterface.getFile(fileId) + if not fileData: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"File with ID {fileId} not found" + ) + + # Get file content + fileContent = managementInterface.getFileData(fileId) + if not fileContent: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"File content not found for ID {fileId}" + ) + + # Return file as response + return Response( + content=fileContent, + media_type=fileData.mimeType, + headers={ + "Content-Disposition": f"attachment; filename={fileData.filename}" + } + ) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error downloading file: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Error downloading file: {str(e)}" + ) + +@router.get("/{fileId}/preview", response_model=FilePreview) +@limiter.limit("30/minute") +async def preview_file( + request: Request, + fileId: str = Path(..., description="ID of the file to preview"), + currentUser: User = Depends(getCurrentUser) +) -> FilePreview: + """Preview a file's content""" + try: + managementInterface = serviceManagementClass.getInterface(currentUser) + + # Get file preview + preview = managementInterface.getFilePreview(fileId) + if not preview: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"File with ID {fileId} not found or no content available" + ) + + return FilePreview(**preview) + except HTTPException: + raise + except Exception as e: + logger.error(f"Error previewing file: {str(e)}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Error previewing file: {str(e)}" + ) + diff --git a/modules/routes/routeWorkflows.py b/modules/routes/routeWorkflows.py index 47797b68..bd97f855 100644 --- a/modules/routes/routeWorkflows.py +++ b/modules/routes/routeWorkflows.py @@ -402,75 +402,6 @@ async def delete_file_from_message( detail=f"Error deleting file reference: {str(e)}" ) -# File preview and download routes - -@router.get("/files/{fileId}/preview", response_model=ChatDocument) -@limiter.limit("30/minute") -async def preview_file( - request: Request, - fileId: str = Path(..., description="ID of the file to preview"), - currentUser: User = Depends(getCurrentUser) -) -> ChatDocument: - """Preview a file's content.""" - try: - # Get service container - service = createServiceContainer(currentUser) - - # Get file document - document = service.base.getFileDocument(fileId) - if not document: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"File with ID {fileId} not found" - ) - - return ChatDocument(**document) - except HTTPException: - raise - except Exception as e: - logger.error(f"Error previewing file: {str(e)}", exc_info=True) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Error previewing file: {str(e)}" - ) - -@router.get("/files/{fileId}/download") -@limiter.limit("30/minute") -async def download_file( - request: Request, - fileId: str = Path(..., description="ID of the file to download"), - currentUser: User = Depends(getCurrentUser) -) -> Response: - """Download a file.""" - try: - # Get service container - service = createServiceContainer(currentUser) - - # Get file data - fileInfo = service.base.downloadFile(fileId) - if not fileInfo: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"File with ID {fileId} not found" - ) - - # Return file as response - return Response( - content=fileInfo["content"], - media_type=fileInfo["contentType"], - headers={ - "Content-Disposition": f"attachment; filename={fileInfo['name']}" - } - ) - except HTTPException: - raise - except Exception as e: - logger.error(f"Error downloading file: {str(e)}", exc_info=True) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Error downloading file: {str(e)}" - ) - @router.get("/workflows", response_model=List[ChatWorkflow]) @limiter.limit("30/minute") async def get_workflows(