gateway/modules/routes/routeDataFiles.py

# Copyright (c) 2025 Patrick Motsch
# All rights reserved.
from fastapi import APIRouter, HTTPException, Depends, File, UploadFile, Form, Path, Request, status, Query, Response, Body
from fastapi.responses import JSONResponse
from typing import List, Dict, Any, Optional
import logging
import json

# Import auth module
from modules.auth import limiter, getCurrentUser

# Import interfaces
import modules.interfaces.interfaceDbManagement as interfaceDbManagement
from modules.datamodels.datamodelFiles import FileItem, FilePreview
from modules.shared.attributeUtils import getModelAttributeDefinitions
from modules.datamodels.datamodelUam import User
from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict

# Configure logger
logger = logging.getLogger(__name__)

# Model attributes for FileItem
fileAttributes = getModelAttributeDefinitions(FileItem)

# Create router for file endpoints
router = APIRouter(
    prefix="/api/files",
    tags=["Manage Files"],
    responses={
        404: {"description": "Not found"}, 
        400: {"description": "Bad request"},
        401: {"description": "Unauthorized"},
        403: {"description": "Forbidden"},
        500: {"description": "Internal server error"}
    }
)

@router.get("/list", response_model=PaginatedResponse[FileItem])
@limiter.limit("30/minute")
def get_files(
    request: Request,
    pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"),
    currentUser: User = Depends(getCurrentUser)
) -> PaginatedResponse[FileItem]:
    """
    Get files with optional pagination, sorting, and filtering.
    
    Query Parameters:
    - pagination: JSON-encoded PaginationParams object, or None for no pagination
    
    Examples:
    - GET /api/files/list (no pagination - returns all items)
    - GET /api/files/list?pagination={"page":1,"pageSize":10,"sort":[]}
    - GET /api/files/list?pagination={"page":2,"pageSize":20,"sort":[{"field":"fileName","direction":"asc"}]}
    """
    try:
        # Parse pagination parameter
        paginationParams = None
        if pagination:
            try:
                paginationDict = json.loads(pagination)
                if paginationDict:
                    # Normalize pagination dict (handles top-level "search" field)
                    paginationDict = normalize_pagination_dict(paginationDict)
                    paginationParams = PaginationParams(**paginationDict)
            except (json.JSONDecodeError, ValueError) as e:
                raise HTTPException(
                    status_code=400,
                    detail=f"Invalid pagination parameter: {str(e)}"
                )
        
        managementInterface = interfaceDbManagement.getInterface(currentUser)
        result = managementInterface.getAllFiles(pagination=paginationParams)
        
        # If pagination was requested, result is PaginatedResult
        # If no pagination, result is List[FileItem]
        if paginationParams:
            return PaginatedResponse(
                items=result.items,
                pagination=PaginationMetadata(
                    currentPage=paginationParams.page,
                    pageSize=paginationParams.pageSize,
                    totalItems=result.totalItems,
                    totalPages=result.totalPages,
                    sort=paginationParams.sort,
                    filters=paginationParams.filters
                )
            )
        else:
            return PaginatedResponse(
                items=result,
                pagination=None
            )
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error getting files: {str(e)}")
        raise HTTPException(
            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
            detail=f"Failed to get files: {str(e)}"
        )

@router.post("/upload", status_code=status.HTTP_201_CREATED)
@limiter.limit("10/minute")
async def upload_file(
    request: Request,
    file: UploadFile = File(...),
    workflowId: Optional[str] = Form(None),
    currentUser: User = Depends(getCurrentUser)
) -> JSONResponse:
    # Add fileName property to UploadFile for consistency with backend model
    file.fileName = file.filename
    """Upload a file"""
    try:
        managementInterface = interfaceDbManagement.getInterface(currentUser)
        
        # Read file
        fileContent = await file.read()
        
        # Check size limits
        maxSize = int(interfaceDbManagement.APP_CONFIG.get("File_Management_MAX_UPLOAD_SIZE_MB")) * 1024 * 1024  # in bytes
        if len(fileContent) > maxSize:
            raise HTTPException(
                status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
                detail=f"File too large. Maximum size: {interfaceDbManagement.APP_CONFIG.get('File_Management_MAX_UPLOAD_SIZE_MB')}MB"
            )
        
        # Save file via LucyDOM interface in the database
        fileItem, duplicateType = managementInterface.saveUploadedFile(fileContent, file.filename)
        
        # Determine response message based on duplicate type
        if duplicateType == "exact_duplicate":
            message = f"File '{file.filename}' already exists with identical content. Reusing existing file."
        elif duplicateType == "name_conflict":
            message = f"File '{file.filename}' already exists with different content. Uploaded as '{fileItem.fileName}'."
        else:  # new_file
            message = "File uploaded successfully"
        
        # Convert FileItem to dictionary for JSON response
        fileMeta = fileItem.model_dump()
        
        # If workflowId is provided, include it in the response (not stored in FileItem model)
        if workflowId:
            fileMeta["workflowId"] = workflowId
        
        # Response with duplicate information
        return JSONResponse({
            "message": message,
            "file": fileMeta,
            "duplicateType": duplicateType,
            "originalFileName": file.filename,
            "storedFileName": fileItem.fileName,
            "isDuplicate": duplicateType != "new_file"
        })
        
    except interfaceDbManagement.FileStorageError as e:
        logger.error(f"Error during file upload (storage): {str(e)}")
        raise HTTPException(
            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, 
            detail=str(e)
        )
    except Exception as e:
        logger.error(f"Error during file upload: {str(e)}")
        raise HTTPException(
            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
            detail=f"Error during file upload: {str(e)}"
        )

@router.get("/{fileId}", response_model=FileItem)
@limiter.limit("30/minute")
def get_file(
    request: Request,
    fileId: str = Path(..., description="ID of the file"),
    currentUser: User = Depends(getCurrentUser)
) -> FileItem:
    """Get a file"""
    try:
        managementInterface = interfaceDbManagement.getInterface(currentUser)
        
        # Get file via LucyDOM interface from the database
        fileData = managementInterface.getFile(fileId)
        if not fileData:
            raise HTTPException(
                status_code=status.HTTP_404_NOT_FOUND,
                detail=f"File with ID {fileId} not found"
            )
            
        return fileData
            
    except interfaceDbManagement.FileNotFoundError as e:
        logger.warning(f"File not found: {str(e)}")
        raise HTTPException(
            status_code=status.HTTP_404_NOT_FOUND,
            detail=str(e)
        )
    except interfaceDbManagement.FilePermissionError as e:
        logger.warning(f"No permission for file: {str(e)}")
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail=str(e)
        )
    except interfaceDbManagement.FileError as e:
        logger.error(f"Error retrieving file: {str(e)}")
        raise HTTPException(
            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
            detail=str(e)
        )
    except Exception as e:
        logger.error(f"Unexpected error retrieving file: {str(e)}")
        raise HTTPException(
            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
            detail=f"Error retrieving file: {str(e)}"
        )

@router.put("/{fileId}", response_model=FileItem)
@limiter.limit("10/minute")
def update_file(
    request: Request,
    fileId: str = Path(..., description="ID of the file to update"),
    file_info: Dict[str, Any] = Body(...),
    currentUser: User = Depends(getCurrentUser)
) -> FileItem:
    """Update file info"""
    try:
        managementInterface = interfaceDbManagement.getInterface(currentUser)
        
        # Get the file from the database
        file = managementInterface.getFile(fileId)
        if not file:
            raise HTTPException(
                status_code=status.HTTP_404_NOT_FOUND,
                detail=f"File with ID {fileId} not found"
            )
            
        # Check if user has access to the file using RBAC
        if not managementInterface.checkRbacPermission(FileItem, "update", fileId):
            raise HTTPException(
                status_code=status.HTTP_403_FORBIDDEN,
                detail="Not authorized to update this file"
            )
            
        # Update the file
        result = managementInterface.updateFile(fileId, file_info)
        if not result:
            raise HTTPException(
                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
                detail="Failed to update file"
            )
            
        # Get updated file
        updatedFile = managementInterface.getFile(fileId)
        return updatedFile
        
    except HTTPException as he:
        raise he
    except Exception as e:
        logger.error(f"Error updating file: {str(e)}")
        raise HTTPException(
            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
            detail=str(e)
        )

@router.delete("/{fileId}", response_model=Dict[str, Any])
@limiter.limit("10/minute")
def delete_file(
    request: Request,
    fileId: str = Path(..., description="ID of the file to delete"),
    currentUser: User = Depends(getCurrentUser)
) -> Dict[str, Any]:
    """Delete a file"""
    managementInterface = interfaceDbManagement.getInterface(currentUser)
    
    # Check if the file exists
    existingFile = managementInterface.getFile(fileId)
    if not existingFile:
        raise HTTPException(
            status_code=status.HTTP_404_NOT_FOUND,
            detail=f"File with ID {fileId} not found"
        )
    
    success = managementInterface.deleteFile(fileId)
    if not success:
        raise HTTPException(
            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
            detail="Error deleting the file"
        )
    
    return {"message": f"File with ID {fileId} successfully deleted"}

@router.get("/stats", response_model=Dict[str, Any])
@limiter.limit("30/minute")
def get_file_stats(
    request: Request,
    currentUser: User = Depends(getCurrentUser)
) -> Dict[str, Any]:
    """Returns statistics about the stored files"""
    try:
        managementInterface = interfaceDbManagement.getInterface(currentUser)
        
        # Get all files - metadata only
        allFiles = managementInterface.getAllFiles()
        
        # Calculate statistics
        totalFiles = len(allFiles)
        totalSize = sum(file.fileSize for file in allFiles)
        
        # Group by file type
        fileTypes = {}
        for file in allFiles:
            fileType = file.mimeType.split("/")[0]
            if fileType not in fileTypes:
                fileTypes[fileType] = 0
            fileTypes[fileType] += 1
        
        return {
            "totalFiles": totalFiles,
            "totalSizeBytes": totalSize,
            "fileTypes": fileTypes
        }
        
    except Exception as e:
        logger.error(f"Error retrieving file statistics: {str(e)}")
        raise HTTPException(
            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
            detail=f"Error retrieving file statistics: {str(e)}"
        )

@router.get("/{fileId}/download")
@limiter.limit("30/minute")
def download_file(
    request: Request,
    fileId: str = Path(..., description="ID of the file to download"),
    currentUser: User = Depends(getCurrentUser)
) -> Response:
    """Download a file"""
    try:
        managementInterface = interfaceDbManagement.getInterface(currentUser)
        
        # Get file data
        fileData = managementInterface.getFile(fileId)
        if not fileData:
            raise HTTPException(
                status_code=status.HTTP_404_NOT_FOUND,
                detail=f"File with ID {fileId} not found"
            )
            
        # Get file content
        fileContent = managementInterface.getFileData(fileId)
        if not fileContent:
            raise HTTPException(
                status_code=status.HTTP_404_NOT_FOUND,
                detail=f"File content not found for ID {fileId}"
            )
        
        # Return file as response
        # Properly encode filename for Content-Disposition header to handle Unicode characters
        import urllib.parse
        encoded_filename = urllib.parse.quote(fileData.fileName)
        
        return Response(
            content=fileContent,
            media_type=fileData.mimeType,
            headers={
                "Content-Disposition": f"attachment; filename*=UTF-8''{encoded_filename}"
            }
        )
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error downloading file: {str(e)}")
        raise HTTPException(
            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
            detail=f"Error downloading file: {str(e)}"
        )

@router.get("/{fileId}/preview", response_model=FilePreview)
@limiter.limit("30/minute")
def preview_file(
    request: Request,
    fileId: str = Path(..., description="ID of the file to preview"),
    currentUser: User = Depends(getCurrentUser)
) -> FilePreview:
    """Preview a file's content"""
    try:
        managementInterface = interfaceDbManagement.getInterface(currentUser)
        
        # Get file preview using the correct method
        preview = managementInterface.getFileContent(fileId)
        if not preview:
            raise HTTPException(
                status_code=status.HTTP_404_NOT_FOUND,
                detail=f"File with ID {fileId} not found or no content available"
            )
        
        return preview
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error previewing file: {str(e)}")
        raise HTTPException(
            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
            detail=f"Error previewing file: {str(e)}"
        )