# Copyright (c) 2025 Patrick Motsch
# All rights reserved.
"""
Automation routes for the backend API.
Implements the endpoints for automation definition management.
"""

from fastapi import APIRouter, HTTPException, Depends, Body, Path, Request, Response, Query
from typing import List, Dict, Any, Optional
from fastapi import status
from fastapi.responses import JSONResponse
import logging
import json

# Import interfaces and models
from modules.features.automation.interfaceFeatureAutomation import getInterface as getAutomationInterface
from modules.features.automation.mainAutomation import getAutomationServices
from modules.auth import limiter, getRequestContext, RequestContext
from modules.features.automation.datamodelFeatureAutomation import AutomationDefinition, AutomationTemplate
from modules.datamodels.datamodelChat import ChatWorkflow, ChatMessage, ChatLog, UserInputRequest, WorkflowModeEnum
from modules.datamodels.datamodelPagination import PaginationParams, PaginatedResponse, PaginationMetadata, normalize_pagination_dict
from modules.shared.attributeUtils import getModelAttributeDefinitions
from modules.interfaces import interfaceDbChat
from modules.interfaces.interfaceDbBilling import getInterface as _getBillingInterface
# Configure logger
logger = logging.getLogger(__name__)

# Model attributes for AutomationDefinition and ChatWorkflow
automationAttributes = getModelAttributeDefinitions(AutomationDefinition)
workflowAttributes = getModelAttributeDefinitions(ChatWorkflow)

# Create router for automation endpoints
router = APIRouter(
    prefix="/api/automations",
    tags=["Manage Automations"],
    responses={
        404: {"description": "Not found"},
        400: {"description": "Bad request"},
        401: {"description": "Unauthorized"},
        403: {"description": "Forbidden"},
        500: {"description": "Internal server error"}
    }
)

@router.get("", response_model=PaginatedResponse[AutomationDefinition])
@limiter.limit("30/minute")
def get_automations(
    request: Request,
    pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"),
    context: RequestContext = Depends(getRequestContext)
) -> PaginatedResponse[AutomationDefinition]:
    """
    Get automation definitions with optional pagination, sorting, and filtering.
    
    Query Parameters:
    - pagination: JSON-encoded PaginationParams object, or None for no pagination
    """
    try:
        # Parse pagination parameter
        paginationParams = None
        if pagination:
            try:
                paginationDict = json.loads(pagination)
                if paginationDict:
                    paginationDict = normalize_pagination_dict(paginationDict)
                    paginationParams = PaginationParams(**paginationDict)
            except (json.JSONDecodeError, ValueError) as e:
                raise HTTPException(
                    status_code=400,
                    detail=f"Invalid pagination parameter: {str(e)}"
                )
        
        # AutomationDefinitions can belong to ANY feature instance within a mandate.
        # The list endpoint must show all definitions for the user's mandate, not filter by a specific featureInstanceId.
        chatInterface = getAutomationInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None)
        result = chatInterface.getAllAutomationDefinitions(pagination=paginationParams)
        
        # If pagination was requested, result is PaginatedResult
        # If no pagination, result is List[Dict]
        # Note: Using JSONResponse to bypass Pydantic validation which would filter out sysCreatedBy
        # The enriched fields (sysCreatedByUserName, mandateName) are not in the Pydantic model
        from fastapi.responses import JSONResponse
        
        if paginationParams:
            response_data = {
                "items": result.items,
                "pagination": {
                    "currentPage": paginationParams.page,
                    "pageSize": paginationParams.pageSize,
                    "totalItems": result.totalItems,
                    "totalPages": result.totalPages,
                    "sort": paginationParams.sort,
                    "filters": paginationParams.filters
                }
            }
        else:
            response_data = {
                "items": result,
                "pagination": None
            }
        
        return JSONResponse(content=response_data)
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error getting automations: {str(e)}")
        raise HTTPException(
            status_code=500,
            detail=f"Error getting automations: {str(e)}"
        )

@router.get("/filter-values")
@limiter.limit("60/minute")
def get_automation_filter_values(
    request: Request,
    column: str = Query(..., description="Column key"),
    pagination: Optional[str] = Query(None, description="JSON-encoded current filters"),
    context: RequestContext = Depends(getRequestContext)
) -> list:
    """Return distinct filter values for a column in automations."""
    try:
        from modules.routes.routeDataUsers import _handleFilterValuesRequest
        chatInterface = getAutomationInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None)
        result = chatInterface.getAllAutomationDefinitions(pagination=None)
        items = result if isinstance(result, list) else [r if isinstance(r, dict) else r.model_dump() if hasattr(r, 'model_dump') else r for r in result]
        return _handleFilterValuesRequest(items, column, pagination)
    except Exception as e:
        logger.error(f"Error getting filter values for automations: {str(e)}")
        raise HTTPException(status_code=500, detail=str(e))


@router.post("", response_model=AutomationDefinition)
@limiter.limit("10/minute")
def create_automation(
    request: Request,
    automation: AutomationDefinition,
    context: RequestContext = Depends(getRequestContext)
) -> AutomationDefinition:
    """Create a new automation definition"""
    try:
        chatInterface = getAutomationInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None, featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None)
        automationData = automation.model_dump()
        created = chatInterface.createAutomationDefinition(automationData)
        return created
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error creating automation: {str(e)}")
        raise HTTPException(
            status_code=500,
            detail=f"Error creating automation: {str(e)}"
        )

@router.get("/attributes", response_model=Dict[str, Any])
def get_automation_attributes(
    request: Request
) -> Dict[str, Any]:
    """Get attribute definitions for AutomationDefinition model"""
    return {"attributes": automationAttributes}


@router.get("/actions")
@limiter.limit("30/minute")
def get_available_actions(
    request: Request,
    context: RequestContext = Depends(getRequestContext)
) -> JSONResponse:
    """
    Get available workflow actions for template editor.
    Returns action definitions with parameters and example JSON snippets.
    """
    try:
        from modules.workflows.processing.shared.methodDiscovery import methods, discoverMethods
        
        # Ensure methods are discovered (need a service hub for discovery)
        if not methods:
            services = getAutomationServices(
                context.user,
                mandateId=context.mandateId,
                featureInstanceId=context.featureInstanceId,
            )
            discoverMethods(services)
        
        actionsList = []
        processedMethods = set()
        
        for methodName, methodInfo in methods.items():
            # Skip short name aliases - only process full class names (MethodXxx)
            if not methodName.startswith('Method'):
                continue
            
            shortName = methodName.replace('Method', '').lower()
            
            # Skip if already processed
            if shortName in processedMethods:
                continue
            processedMethods.add(shortName)
            
            methodInstance = methodInfo.get('instance')
            if not methodInstance:
                continue
            
            # Get actions from method instance
            for actionName, actionDef in methodInstance._actions.items():
                # Build action info
                actionInfo = {
                    "method": shortName,
                    "action": actionName,
                    "actionId": actionDef.actionId if hasattr(actionDef, 'actionId') else f"{shortName}.{actionName}",
                    "description": actionDef.description if hasattr(actionDef, 'description') else "",
                    "category": actionDef.category if hasattr(actionDef, 'category') else "general",
                    "parameters": []
                }
                
                # Add parameters from WorkflowActionParameter
                parametersDef = actionDef.parameters if hasattr(actionDef, 'parameters') else {}
                for paramName, paramDef in parametersDef.items():
                    paramInfo = {
                        "name": paramName,
                        "type": paramDef.type if hasattr(paramDef, 'type') else "Any",
                        "frontendType": paramDef.frontendType.value if hasattr(paramDef, 'frontendType') and paramDef.frontendType else "text",
                        "required": paramDef.required if hasattr(paramDef, 'required') else False,
                        "default": paramDef.default if hasattr(paramDef, 'default') else None,
                        "description": paramDef.description if hasattr(paramDef, 'description') else "",
                    }
                    if hasattr(paramDef, 'frontendOptions') and paramDef.frontendOptions:
                        paramInfo["frontendOptions"] = paramDef.frontendOptions
                    actionInfo["parameters"].append(paramInfo)
                
                # Build example JSON snippet for copy/paste
                exampleParams = {}
                for paramName, paramDef in parametersDef.items():
                    if hasattr(paramDef, 'required') and paramDef.required:
                        exampleParams[paramName] = f"{{{{KEY:{paramName}}}}}"
                    else:
                        default = paramDef.default if hasattr(paramDef, 'default') else None
                        exampleParams[paramName] = default or f"{{{{KEY:{paramName}}}}}"
                
                actionInfo["exampleJson"] = {
                    "execMethod": shortName,
                    "execAction": actionName,
                    "execParameters": exampleParams,
                    "execResultLabel": f"{shortName}_{actionName}_result"
                }
                
                actionsList.append(actionInfo)
        
        return JSONResponse(content={"actions": actionsList})
    except Exception as e:
        logger.error(f"Error getting available actions: {str(e)}")
        raise HTTPException(
            status_code=500,
            detail=f"Error getting available actions: {str(e)}"
        )


# -----------------------------------------------------------------------------
# Workflow routes under /{instanceId}/workflows/ (instance-scoped)
# -----------------------------------------------------------------------------

def _validateAutomationInstanceAccess(instanceId: str, context: RequestContext) -> Optional[str]:
    """Validate user has access to the automation feature instance. Returns mandateId."""
    from modules.interfaces.interfaceDbApp import getRootInterface
    rootInterface = getRootInterface()
    instance = rootInterface.getFeatureInstance(instanceId)
    if not instance:
        raise HTTPException(status_code=404, detail=f"Feature instance {instanceId} not found")
    featureAccess = rootInterface.getFeatureAccess(str(context.user.id), instanceId)
    if not featureAccess or not featureAccess.enabled:
        raise HTTPException(status_code=403, detail="Access denied to this feature instance")
    return str(instance.mandateId) if instance.mandateId else None


def _getAutomationServiceChat(context: RequestContext, featureInstanceId: str = None, mandateId: str = None):
    """Get chat interface with feature instance context for workflows."""
    return interfaceDbChat.getInterface(
        context.user,
        mandateId=mandateId or (str(context.mandateId) if context.mandateId else None),
        featureInstanceId=featureInstanceId
    )


@router.get("/{instanceId}/workflows", response_model=PaginatedResponse[ChatWorkflow])
@limiter.limit("120/minute")
def get_automation_workflows(
    request: Request,
    instanceId: str = Path(..., description="Feature instance ID"),
    pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"),
    page: int = Query(1, ge=1, description="Page number (legacy)"),
    pageSize: int = Query(20, ge=1, le=100, description="Items per page (legacy)"),
    context: RequestContext = Depends(getRequestContext)
) -> PaginatedResponse[ChatWorkflow]:
    """Get all workflows for this automation feature instance."""
    try:
        mandateId = _validateAutomationInstanceAccess(instanceId, context)
        chatInterface = _getAutomationServiceChat(context, featureInstanceId=instanceId, mandateId=mandateId)
        paginationParams = None
        if pagination:
            try:
                paginationDict = json.loads(pagination)
                if paginationDict:
                    paginationDict = normalize_pagination_dict(paginationDict)
                    paginationParams = PaginationParams(**paginationDict)
            except (json.JSONDecodeError, ValueError) as e:
                raise HTTPException(status_code=400, detail=f"Invalid pagination parameter: {str(e)}")
        else:
            paginationParams = PaginationParams(page=page, pageSize=pageSize)
        result = chatInterface.getWorkflows(pagination=paginationParams)
        if paginationParams:
            return PaginatedResponse(
                items=result.items,
                pagination=PaginationMetadata(
                    currentPage=paginationParams.page,
                    pageSize=paginationParams.pageSize,
                    totalItems=result.totalItems,
                    totalPages=result.totalPages,
                    sort=paginationParams.sort,
                    filters=paginationParams.filters
                )
            )
        return PaginatedResponse(items=result, pagination=None)
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error getting automation workflows: {str(e)}", exc_info=True)
        raise HTTPException(status_code=500, detail=f"Error getting workflows: {str(e)}")


# Workflow attributes (ChatWorkflow model)
@router.get("/{instanceId}/workflows/attributes", response_model=Dict[str, Any])
@limiter.limit("120/minute")
def get_automation_workflow_attributes(
    request: Request,
    instanceId: str = Path(..., description="Feature instance ID"),
    context: RequestContext = Depends(getRequestContext)
) -> Dict[str, Any]:
    """Get attribute definitions for ChatWorkflow model."""
    _validateAutomationInstanceAccess(instanceId, context)
    return {"attributes": workflowAttributes}


# Actions (must be before /{workflowId} to avoid path conflict)
@router.get("/{instanceId}/workflows/actions", response_model=Dict[str, Any])
@limiter.limit("120/minute")
def get_automation_workflow_actions(
    request: Request,
    instanceId: str = Path(..., description="Feature instance ID"),
    context: RequestContext = Depends(getRequestContext)
) -> Dict[str, Any]:
    """Get all available workflow actions."""
    try:
        mandateId = _validateAutomationInstanceAccess(instanceId, context)
        services = getAutomationServices(context.user, mandateId=mandateId, featureInstanceId=instanceId)
        from modules.workflows.processing.shared.methodDiscovery import discoverMethods, methods
        discoverMethods(services)
        allActions = []
        for methodName, methodInfo in methods.items():
            if methodName.startswith('Method'):
                continue
            methodInstance = methodInfo['instance']
            for actionName, actionInfo in methodInstance.actions.items():
                allActions.append({
                    "module": methodInstance.name,
                    "actionId": f"{methodInstance.name}.{actionName}",
                    "name": actionName,
                    "description": actionInfo.get('description', ''),
                    "parameters": actionInfo.get('parameters', {})
                })
        return {"actions": allActions}
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error getting actions: {str(e)}", exc_info=True)
        raise HTTPException(status_code=500, detail=f"Failed to get actions: {str(e)}")


@router.get("/{instanceId}/workflows/actions/{method}", response_model=Dict[str, Any])
@limiter.limit("120/minute")
def get_automation_method_actions(
    request: Request,
    instanceId: str = Path(..., description="Feature instance ID"),
    method: str = Path(..., description="Method name"),
    context: RequestContext = Depends(getRequestContext)
) -> Dict[str, Any]:
    """Get actions for a specific method."""
    try:
        _validateAutomationInstanceAccess(instanceId, context)
        services = getAutomationServices(context.user, mandateId=str(context.mandateId) if context.mandateId else None, featureInstanceId=instanceId)
        from modules.workflows.processing.shared.methodDiscovery import discoverMethods, methods
        discoverMethods(services)
        methodInstance = None
        for mn, mi in methods.items():
            if mi['instance'].name == method:
                methodInstance = mi['instance']
                break
        if not methodInstance:
            raise HTTPException(status_code=404, detail=f"Method '{method}' not found")
        actions = [{"actionId": f"{methodInstance.name}.{an}", "name": an, "description": ai.get('description', ''), "parameters": ai.get('parameters', {})}
                  for an, ai in methodInstance.actions.items()]
        return {"module": methodInstance.name, "description": methodInstance.description, "actions": actions}
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error getting actions for {method}: {str(e)}", exc_info=True)
        raise HTTPException(status_code=500, detail=f"Failed to get actions: {str(e)}")


@router.get("/{instanceId}/workflows/actions/{method}/{action}", response_model=Dict[str, Any])
@limiter.limit("120/minute")
def get_automation_action_schema(
    request: Request,
    instanceId: str = Path(..., description="Feature instance ID"),
    method: str = Path(..., description="Method name"),
    action: str = Path(..., description="Action name"),
    context: RequestContext = Depends(getRequestContext)
) -> Dict[str, Any]:
    """Get action schema for a specific action."""
    try:
        _validateAutomationInstanceAccess(instanceId, context)
        services = getAutomationServices(context.user, mandateId=str(context.mandateId) if context.mandateId else None, featureInstanceId=instanceId)
        from modules.workflows.processing.shared.methodDiscovery import discoverMethods, methods
        discoverMethods(services)
        methodInstance = None
        for mn, mi in methods.items():
            if mi['instance'].name == method:
                methodInstance = mi['instance']
                break
        if not methodInstance:
            raise HTTPException(status_code=404, detail=f"Method '{method}' not found")
        if action not in methodInstance.actions:
            raise HTTPException(status_code=404, detail=f"Action '{action}' not found in method '{method}'")
        ai = methodInstance.actions[action]
        return {"method": methodInstance.name, "action": action, "actionId": f"{methodInstance.name}.{action}",
                "description": ai.get('description', ''), "parameters": ai.get('parameters', {})}
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error getting action schema: {str(e)}", exc_info=True)
        raise HTTPException(status_code=500, detail=f"Failed to get action schema: {str(e)}")


@router.get("/{instanceId}/workflows/{workflowId}", response_model=ChatWorkflow)
@limiter.limit("120/minute")
def get_automation_workflow(
    request: Request,
    instanceId: str = Path(..., description="Feature instance ID"),
    workflowId: str = Path(..., description="Workflow ID"),
    context: RequestContext = Depends(getRequestContext)
) -> ChatWorkflow:
    """Get workflow by ID."""
    try:
        _validateAutomationInstanceAccess(instanceId, context)
        chatInterface = _getAutomationServiceChat(context, featureInstanceId=instanceId)
        workflow = chatInterface.getWorkflow(workflowId)
        if not workflow:
            raise HTTPException(status_code=404, detail="Workflow not found")
        return workflow
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error getting workflow: {str(e)}")
        raise HTTPException(status_code=500, detail=f"Failed to get workflow: {str(e)}")


@router.put("/{instanceId}/workflows/{workflowId}", response_model=ChatWorkflow)
@limiter.limit("120/minute")
def update_automation_workflow(
    request: Request,
    instanceId: str = Path(..., description="Feature instance ID"),
    workflowId: str = Path(..., description="Workflow ID"),
    workflowData: Dict[str, Any] = Body(...),
    context: RequestContext = Depends(getRequestContext)
) -> ChatWorkflow:
    """Update workflow by ID."""
    try:
        _validateAutomationInstanceAccess(instanceId, context)
        chatInterface = _getAutomationServiceChat(context, featureInstanceId=instanceId)
        workflow = chatInterface.getWorkflow(workflowId)
        if not workflow:
            raise HTTPException(status_code=404, detail="Workflow not found")
        if not chatInterface.checkRbacPermission(ChatWorkflow, "update", workflowId):
            raise HTTPException(status_code=403, detail="You don't have permission to update this workflow")
        updated = chatInterface.updateWorkflow(workflowId, workflowData)
        if not updated:
            raise HTTPException(status_code=500, detail="Failed to update workflow")
        return updated
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error updating workflow: {str(e)}")
        raise HTTPException(status_code=500, detail=f"Failed to update workflow: {str(e)}")


@router.delete("/{instanceId}/workflows/{workflowId}")
@limiter.limit("120/minute")
def delete_automation_workflow(
    request: Request,
    instanceId: str = Path(..., description="Feature instance ID"),
    workflowId: str = Path(..., description="Workflow ID"),
    context: RequestContext = Depends(getRequestContext)
) -> Dict[str, Any]:
    """Delete workflow and associated data."""
    try:
        _validateAutomationInstanceAccess(instanceId, context)
        chatInterface = _getAutomationServiceChat(context, featureInstanceId=instanceId)
        workflow = chatInterface.getWorkflow(workflowId)
        if not workflow:
            raise HTTPException(status_code=404, detail=f"Workflow {workflowId} not found")
        if not chatInterface.checkRbacPermission(ChatWorkflow, "delete", workflowId):
            raise HTTPException(status_code=403, detail="You don't have permission to delete this workflow")
        success = chatInterface.deleteWorkflow(workflowId)
        if not success:
            raise HTTPException(status_code=500, detail="Failed to delete workflow")
        return {"id": workflowId, "message": "Workflow and associated data deleted successfully"}
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error deleting workflow: {str(e)}", exc_info=True)
        raise HTTPException(status_code=500, detail=f"Error deleting workflow: {str(e)}")


@router.get("/{instanceId}/workflows/{workflowId}/status", response_model=ChatWorkflow)
@limiter.limit("120/minute")
def get_automation_workflow_status(
    request: Request,
    instanceId: str = Path(..., description="Feature instance ID"),
    workflowId: str = Path(..., description="Workflow ID"),
    context: RequestContext = Depends(getRequestContext)
) -> ChatWorkflow:
    """Get workflow status."""
    try:
        _validateAutomationInstanceAccess(instanceId, context)
        chatInterface = _getAutomationServiceChat(context, featureInstanceId=instanceId)
        workflow = chatInterface.getWorkflow(workflowId)
        if not workflow:
            raise HTTPException(status_code=404, detail=f"Workflow {workflowId} not found")
        return workflow
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error getting workflow status: {str(e)}", exc_info=True)
        raise HTTPException(status_code=500, detail=f"Error getting workflow status: {str(e)}")


@router.get("/{instanceId}/workflows/{workflowId}/logs", response_model=PaginatedResponse[ChatLog])
@limiter.limit("120/minute")
def get_automation_workflow_logs(
    request: Request,
    instanceId: str = Path(..., description="Feature instance ID"),
    workflowId: str = Path(..., description="Workflow ID"),
    logId: Optional[str] = Query(None),
    pagination: Optional[str] = Query(None),
    context: RequestContext = Depends(getRequestContext)
) -> PaginatedResponse[ChatLog]:
    """Get workflow logs."""
    try:
        _validateAutomationInstanceAccess(instanceId, context)
        chatInterface = _getAutomationServiceChat(context, featureInstanceId=instanceId)
        workflow = chatInterface.getWorkflow(workflowId)
        if not workflow:
            raise HTTPException(status_code=404, detail=f"Workflow {workflowId} not found")
        paginationParams = None
        if pagination:
            try:
                pd = json.loads(pagination)
                if pd:
                    pd = normalize_pagination_dict(pd)
                    paginationParams = PaginationParams(**pd)
            except (json.JSONDecodeError, ValueError) as e:
                raise HTTPException(status_code=400, detail=f"Invalid pagination: {str(e)}")
        result = chatInterface.getLogs(workflowId, pagination=paginationParams)
        if logId:
            allLogs = result.items if paginationParams else result
            idx = next((i for i, log in enumerate(allLogs) if log.id == logId), -1)
            if idx >= 0:
                return PaginatedResponse(items=allLogs[idx + 1:], pagination=None)
        if paginationParams:
            return PaginatedResponse(items=result.items, pagination=PaginationMetadata(
                currentPage=paginationParams.page, pageSize=paginationParams.pageSize,
                totalItems=result.totalItems, totalPages=result.totalPages,
                sort=paginationParams.sort, filters=paginationParams.filters))
        return PaginatedResponse(items=result, pagination=None)
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error getting workflow logs: {str(e)}", exc_info=True)
        raise HTTPException(status_code=500, detail=f"Error getting workflow logs: {str(e)}")


@router.get("/{instanceId}/workflows/{workflowId}/messages", response_model=PaginatedResponse[ChatMessage])
@limiter.limit("120/minute")
def get_automation_workflow_messages(
    request: Request,
    instanceId: str = Path(..., description="Feature instance ID"),
    workflowId: str = Path(..., description="Workflow ID"),
    messageId: Optional[str] = Query(None),
    pagination: Optional[str] = Query(None),
    context: RequestContext = Depends(getRequestContext)
) -> PaginatedResponse[ChatMessage]:
    """Get workflow messages."""
    try:
        _validateAutomationInstanceAccess(instanceId, context)
        chatInterface = _getAutomationServiceChat(context, featureInstanceId=instanceId)
        workflow = chatInterface.getWorkflow(workflowId)
        if not workflow:
            raise HTTPException(status_code=404, detail=f"Workflow {workflowId} not found")
        paginationParams = None
        if pagination:
            try:
                pd = json.loads(pagination)
                if pd:
                    pd = normalize_pagination_dict(pd)
                    paginationParams = PaginationParams(**pd)
            except (json.JSONDecodeError, ValueError) as e:
                raise HTTPException(status_code=400, detail=f"Invalid pagination: {str(e)}")
        result = chatInterface.getMessages(workflowId, pagination=paginationParams)
        if messageId:
            allMsgs = result.items if paginationParams else result
            idx = next((i for i, m in enumerate(allMsgs) if m.id == messageId), -1)
            if idx >= 0:
                return PaginatedResponse(items=allMsgs[idx + 1:], pagination=None)
        if paginationParams:
            return PaginatedResponse(items=result.items, pagination=PaginationMetadata(
                currentPage=paginationParams.page, pageSize=paginationParams.pageSize,
                totalItems=result.totalItems, totalPages=result.totalPages,
                sort=paginationParams.sort, filters=paginationParams.filters))
        return PaginatedResponse(items=result, pagination=None)
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error getting workflow messages: {str(e)}", exc_info=True)
        raise HTTPException(status_code=500, detail=f"Error getting workflow messages: {str(e)}")


@router.delete("/{instanceId}/workflows/{workflowId}/messages/{messageId}")
@limiter.limit("120/minute")
def delete_automation_workflow_message(
    request: Request,
    instanceId: str = Path(..., description="Feature instance ID"),
    workflowId: str = Path(..., description="Workflow ID"),
    messageId: str = Path(..., description="Message ID"),
    context: RequestContext = Depends(getRequestContext)
) -> Dict[str, Any]:
    """Delete message from workflow."""
    try:
        _validateAutomationInstanceAccess(instanceId, context)
        chatInterface = _getAutomationServiceChat(context, featureInstanceId=instanceId)
        workflow = chatInterface.getWorkflow(workflowId)
        if not workflow:
            raise HTTPException(status_code=404, detail=f"Workflow {workflowId} not found")
        success = chatInterface.deleteMessage(workflowId, messageId)
        if not success:
            raise HTTPException(status_code=404, detail=f"Message {messageId} not found")
        return {"workflowId": workflowId, "messageId": messageId, "message": "Message deleted successfully"}
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error deleting message: {str(e)}", exc_info=True)
        raise HTTPException(status_code=500, detail=f"Error deleting message: {str(e)}")


@router.delete("/{instanceId}/workflows/{workflowId}/messages/{messageId}/files/{fileId}")
@limiter.limit("120/minute")
def delete_automation_file_from_message(
    request: Request,
    instanceId: str = Path(..., description="Feature instance ID"),
    workflowId: str = Path(..., description="Workflow ID"),
    messageId: str = Path(..., description="Message ID"),
    fileId: str = Path(..., description="File ID"),
    context: RequestContext = Depends(getRequestContext)
) -> Dict[str, Any]:
    """Delete file from message."""
    try:
        _validateAutomationInstanceAccess(instanceId, context)
        chatInterface = _getAutomationServiceChat(context, featureInstanceId=instanceId)
        workflow = chatInterface.getWorkflow(workflowId)
        if not workflow:
            raise HTTPException(status_code=404, detail=f"Workflow {workflowId} not found")
        success = chatInterface.deleteFileFromMessage(workflowId, messageId, fileId)
        if not success:
            raise HTTPException(status_code=404, detail=f"File {fileId} not found")
        return {"workflowId": workflowId, "messageId": messageId, "fileId": fileId, "message": "File reference deleted successfully"}
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error deleting file: {str(e)}", exc_info=True)
        raise HTTPException(status_code=500, detail=f"Error deleting file: {str(e)}")


@router.get("/{instanceId}/workflows/{workflowId}/chatData")
@limiter.limit("120/minute")
def get_automation_workflow_chat_data(
    request: Request,
    instanceId: str = Path(..., description="Feature instance ID"),
    workflowId: str = Path(..., description="Workflow ID"),
    afterTimestamp: Optional[float] = Query(None),
    context: RequestContext = Depends(getRequestContext)
) -> Dict[str, Any]:
    """Get unified chat data for workflow."""
    try:
        _validateAutomationInstanceAccess(instanceId, context)
        chatInterface = _getAutomationServiceChat(context, featureInstanceId=instanceId)
        workflow = chatInterface.getWorkflow(workflowId)
        if not workflow:
            raise HTTPException(status_code=404, detail=f"Workflow {workflowId} not found")
        billingInterface = _getBillingInterface(context.user, context.mandateId)
        workflowCost = billingInterface.getWorkflowCost(workflowId)
        return chatInterface.getUnifiedChatData(workflowId, afterTimestamp, workflowCost=workflowCost)
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error getting chat data: {str(e)}", exc_info=True)
        raise HTTPException(status_code=500, detail=f"Error getting chat data: {str(e)}")


@router.post("/{instanceId}/workflows/{workflowId}/stop", response_model=ChatWorkflow)
@limiter.limit("30/minute")
async def stop_automation_workflow(
    request: Request,
    instanceId: str = Path(..., description="Feature instance ID"),
    workflowId: str = Path(..., description="Workflow ID"),
    context: RequestContext = Depends(getRequestContext)
) -> ChatWorkflow:
    """Stop a running automation workflow. Uses instance-scoped services."""
    try:
        from modules.workflows.automation import chatStop
        mandateId = _validateAutomationInstanceAccess(instanceId, context)
        services = getAutomationServices(
            context.user,
            mandateId=mandateId,
            featureInstanceId=instanceId,
        )
        services.featureCode = "automation"
        return await chatStop(
            context.user,
            workflowId,
            mandateId=mandateId,
            featureInstanceId=instanceId,
            featureCode="automation",
            services=services,
        )
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error stopping automation workflow: {str(e)}")
        raise HTTPException(status_code=500, detail=str(e))


@router.get("/{automationId}", response_model=AutomationDefinition)
@limiter.limit("30/minute")
def get_automation(
    request: Request,
    automationId: str = Path(..., description="Automation ID"),
    context: RequestContext = Depends(getRequestContext)
) -> AutomationDefinition:
    """Get a single automation definition by ID"""
    try:
        chatInterface = getAutomationInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None)
        automation = chatInterface.getAutomationDefinition(automationId)
        if not automation:
            raise HTTPException(
                status_code=404,
                detail=f"Automation {automationId} not found"
            )
        
        return automation
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error getting automation: {str(e)}")
        raise HTTPException(
            status_code=500,
            detail=f"Error getting automation: {str(e)}"
        )

@router.put("/{automationId}", response_model=AutomationDefinition)
@limiter.limit("10/minute")
def update_automation(
    request: Request,
    automationId: str = Path(..., description="Automation ID"),
    automation: AutomationDefinition = Body(...),
    context: RequestContext = Depends(getRequestContext)
) -> AutomationDefinition:
    """Update an automation definition"""
    try:
        chatInterface = getAutomationInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None)
        automationData = automation.model_dump()
        automationData.pop("executionLogs", None)
        updated = chatInterface.updateAutomationDefinition(automationId, automationData)
        return updated
    except HTTPException:
        raise
    except PermissionError as e:
        raise HTTPException(
            status_code=403,
            detail=str(e)
        )
    except Exception as e:
        logger.error(f"Error updating automation: {str(e)}")
        raise HTTPException(
            status_code=500,
            detail=f"Error updating automation: {str(e)}"
        )

@router.patch("/{automationId}/status")
@limiter.limit("30/minute")
def update_automation_status(
    request: Request,
    automationId: str = Path(..., description="Automation ID"),
    active: bool = Body(..., embed=True),
    context: RequestContext = Depends(getRequestContext)
) -> AutomationDefinition:
    """Update only the active status of an automation definition"""
    try:
        chatInterface = getAutomationInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None)
        
        # Get existing automation
        automation = chatInterface.getAutomationDefinition(automationId)
        if not automation:
            raise HTTPException(
                status_code=404,
                detail=f"Automation {automationId} not found"
            )
        
        # Update only the active field
        automationData = automation if isinstance(automation, dict) else automation.model_dump()
        automationData['active'] = active
        
        updated = chatInterface.updateAutomationDefinition(automationId, automationData)
        return updated
    except HTTPException:
        raise
    except PermissionError as e:
        raise HTTPException(
            status_code=403,
            detail=str(e)
        )
    except Exception as e:
        logger.error(f"Error updating automation status: {str(e)}")
        raise HTTPException(
            status_code=500,
            detail=f"Error updating automation status: {str(e)}"
        )


@router.delete("/{automationId}")
@limiter.limit("10/minute")
def delete_automation(
    request: Request,
    automationId: str = Path(..., description="Automation ID"),
    context: RequestContext = Depends(getRequestContext)
) -> Response:
    """Delete an automation definition"""
    try:
        chatInterface = getAutomationInterface(context.user, mandateId=str(context.mandateId) if context.mandateId else None)
        success = chatInterface.deleteAutomationDefinition(automationId)
        if success:
            return Response(status_code=204)
        else:
            raise HTTPException(
                status_code=500,
                detail="Failed to delete automation"
            )
    except HTTPException:
        raise
    except PermissionError as e:
        raise HTTPException(
            status_code=403,
            detail=str(e)
        )
    except Exception as e:
        logger.error(f"Error deleting automation: {str(e)}")
        raise HTTPException(
            status_code=500,
            detail=f"Error deleting automation: {str(e)}"
        )

@router.post("/{instanceId}/start", response_model=ChatWorkflow)
@limiter.limit("120/minute")
async def start_automation_workflow(
    request: Request,
    instanceId: str = Path(..., description="Feature instance ID"),
    workflowId: Optional[str] = Query(None, description="Optional ID of the workflow to continue"),
    workflowMode: WorkflowModeEnum = Query(..., description="Workflow mode: 'Dynamic' or 'Automation' (mandatory)"),
    userInput: UserInputRequest = Body(...),
    context: RequestContext = Depends(getRequestContext)
) -> ChatWorkflow:
    """Start a new workflow or continue an existing one."""
    try:
        from modules.workflows.automation import chatStart
        mandateId = _validateAutomationInstanceAccess(instanceId, context)
        services = getAutomationServices(
            context.user,
            mandateId=mandateId,
            featureInstanceId=instanceId,
        )
        services.featureCode = "automation"
        if hasattr(userInput, 'allowedProviders') and userInput.allowedProviders:
            services.allowedProviders = userInput.allowedProviders
        workflow = await chatStart(
            context.user,
            userInput,
            workflowMode,
            workflowId,
            mandateId=mandateId,
            featureInstanceId=instanceId,
            featureCode="automation",
            services=services,
        )
        return workflow
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error in start_automation_workflow: {str(e)}")
        raise HTTPException(status_code=500, detail=str(e))


@router.post("/{automationId}/execute", response_model=ChatWorkflow)
@limiter.limit("5/minute")
async def execute_automation_route(
    request: Request,
    automationId: str = Path(..., description="Automation ID"),
    context: RequestContext = Depends(getRequestContext)
) -> ChatWorkflow:
    """Execute an automation immediately (test mode)"""
    try:
        services = getAutomationServices(
            context.user,
            mandateId=context.mandateId,
            featureInstanceId=context.featureInstanceId,
        )
        
        # Load automation with current user's context (user has RBAC permissions via UI)
        automation = services.interfaceDbAutomation.getAutomationDefinition(automationId, includeSystemFields=True)
        if not automation:
            raise ValueError(f"Automation {automationId} not found")
        
        from modules.workflows.automation import executeAutomation, chatStop
        workflow = await executeAutomation(automationId, automation, context.user, services)
        return workflow
    except HTTPException:
        raise
    except ValueError as e:
        raise HTTPException(
            status_code=404,
            detail=str(e)
        )
    except Exception as e:
        logger.error(f"Error executing automation: {str(e)}")
        raise HTTPException(
            status_code=500,
            detail=f"Error executing automation: {str(e)}"
        )


# =============================================================================
# AutomationTemplate Routes (DB-persistiert)
# =============================================================================
# Separater Router für /api/automation-templates

templateRouter = APIRouter(
    prefix="/api/automation-templates",
    tags=["Manage Automation Templates"],
    responses={
        404: {"description": "Not found"},
        400: {"description": "Bad request"},
        401: {"description": "Unauthorized"},
        403: {"description": "Forbidden"},
        500: {"description": "Internal server error"}
    }
)

# Model attributes for AutomationTemplate
templateAttributes = getModelAttributeDefinitions(AutomationTemplate)


@templateRouter.get("", response_model=PaginatedResponse[AutomationTemplate])
@limiter.limit("30/minute")
def get_db_templates(
    request: Request,
    pagination: Optional[str] = Query(None, description="JSON-encoded PaginationParams object"),
    context: RequestContext = Depends(getRequestContext)
) -> JSONResponse:
    """
    Get automation templates from database (RBAC-filtered, MY = own templates).
    
    Query Parameters:
    - pagination: JSON-encoded PaginationParams object, or None for no pagination
    """
    try:
        # Parse pagination parameter
        paginationParams = None
        if pagination:
            try:
                paginationDict = json.loads(pagination)
                if paginationDict:
                    paginationDict = normalize_pagination_dict(paginationDict)
                    paginationParams = PaginationParams(**paginationDict)
            except (json.JSONDecodeError, ValueError) as e:
                raise HTTPException(
                    status_code=400,
                    detail=f"Invalid pagination parameter: {str(e)}"
                )
        
        chatInterface = getAutomationInterface(
            context.user,
            mandateId=str(context.mandateId) if context.mandateId else None,
            featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None
        )
        result = chatInterface.getAllAutomationTemplates(pagination=paginationParams)
        
        if paginationParams:
            response_data = {
                "items": result.items,
                "pagination": {
                    "currentPage": paginationParams.page,
                    "pageSize": paginationParams.pageSize,
                    "totalItems": result.totalItems,
                    "totalPages": result.totalPages,
                    "sort": paginationParams.sort,
                    "filters": paginationParams.filters
                }
            }
        else:
            response_data = {
                "items": result,
                "pagination": None
            }
        
        return JSONResponse(content=response_data)
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error getting templates: {str(e)}")
        raise HTTPException(
            status_code=500,
            detail=f"Error getting templates: {str(e)}"
        )


@templateRouter.get("/filter-values")
@limiter.limit("60/minute")
def get_template_filter_values(
    request: Request,
    column: str = Query(..., description="Column key"),
    pagination: Optional[str] = Query(None, description="JSON-encoded current filters"),
    context: RequestContext = Depends(getRequestContext)
) -> list:
    """Return distinct filter values for a column in automation templates."""
    try:
        from modules.routes.routeDataUsers import _handleFilterValuesRequest
        chatInterface = getAutomationInterface(
            context.user,
            mandateId=str(context.mandateId) if context.mandateId else None,
            featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None
        )
        result = chatInterface.getAllAutomationTemplates(pagination=None)
        items = [r if isinstance(r, dict) else r.model_dump() if hasattr(r, 'model_dump') else r for r in result]
        return _handleFilterValuesRequest(items, column, pagination)
    except Exception as e:
        logger.error(f"Error getting filter values for automation templates: {str(e)}")
        raise HTTPException(status_code=500, detail=str(e))


@templateRouter.get("/attributes", response_model=Dict[str, Any])
def get_template_attributes(
    request: Request
) -> Dict[str, Any]:
    """Get attribute definitions for AutomationTemplate model"""
    return {"attributes": templateAttributes}


@templateRouter.get("/{templateId}")
@limiter.limit("30/minute")
def get_db_template(
    request: Request,
    templateId: str = Path(..., description="Template ID"),
    context: RequestContext = Depends(getRequestContext)
) -> JSONResponse:
    """Get a single automation template by ID"""
    try:
        chatInterface = getAutomationInterface(
            context.user,
            mandateId=str(context.mandateId) if context.mandateId else None,
            featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None
        )
        template = chatInterface.getAutomationTemplate(templateId)
        if not template:
            raise HTTPException(
                status_code=404,
                detail=f"Template {templateId} not found"
            )
        
        return JSONResponse(content=template)
    except HTTPException:
        raise
    except Exception as e:
        logger.error(f"Error getting template: {str(e)}")
        raise HTTPException(
            status_code=500,
            detail=f"Error getting template: {str(e)}"
        )


@templateRouter.post("")
@limiter.limit("10/minute")
def create_db_template(
    request: Request,
    templateData: Dict[str, Any] = Body(...),
    context: RequestContext = Depends(getRequestContext)
) -> JSONResponse:
    """Create a new automation template"""
    try:
        chatInterface = getAutomationInterface(
            context.user,
            mandateId=str(context.mandateId) if context.mandateId else None,
            featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None
        )
        created = chatInterface.createAutomationTemplate(templateData, isSysAdmin=context.hasSysAdminRole)
        return JSONResponse(content=created)
    except HTTPException:
        raise
    except PermissionError as e:
        raise HTTPException(
            status_code=403,
            detail=str(e)
        )
    except Exception as e:
        logger.error(f"Error creating template: {str(e)}")
        raise HTTPException(
            status_code=500,
            detail=f"Error creating template: {str(e)}"
        )


@templateRouter.put("/{templateId}")
@limiter.limit("10/minute")
def update_db_template(
    request: Request,
    templateId: str = Path(..., description="Template ID"),
    templateData: Dict[str, Any] = Body(...),
    context: RequestContext = Depends(getRequestContext)
) -> JSONResponse:
    """Update an automation template"""
    try:
        chatInterface = getAutomationInterface(
            context.user,
            mandateId=str(context.mandateId) if context.mandateId else None,
            featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None
        )
        updated = chatInterface.updateAutomationTemplate(templateId, templateData, isSysAdmin=context.hasSysAdminRole)
        return JSONResponse(content=updated)
    except HTTPException:
        raise
    except PermissionError as e:
        raise HTTPException(
            status_code=403,
            detail=str(e)
        )
    except Exception as e:
        logger.error(f"Error updating template: {str(e)}")
        raise HTTPException(
            status_code=500,
            detail=f"Error updating template: {str(e)}"
        )


@templateRouter.delete("/{templateId}")
@limiter.limit("10/minute")
def delete_db_template(
    request: Request,
    templateId: str = Path(..., description="Template ID"),
    context: RequestContext = Depends(getRequestContext)
) -> Response:
    """Delete an automation template"""
    try:
        chatInterface = getAutomationInterface(
            context.user,
            mandateId=str(context.mandateId) if context.mandateId else None,
            featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None
        )
        success = chatInterface.deleteAutomationTemplate(templateId, isSysAdmin=context.hasSysAdminRole)
        if success:
            return Response(status_code=204)
        else:
            raise HTTPException(
                status_code=404,
                detail="Template not found or no permission"
            )
    except HTTPException:
        raise
    except PermissionError as e:
        raise HTTPException(
            status_code=403,
            detail=str(e)
        )
    except Exception as e:
        logger.error(f"Error deleting template: {str(e)}")
        raise HTTPException(
            status_code=500,
            detail=f"Error deleting template: {str(e)}"
        )


@templateRouter.post("/{templateId}/duplicate")
@limiter.limit("10/minute")
def duplicate_db_template(
    request: Request,
    templateId: str = Path(..., description="Template ID to duplicate"),
    context: RequestContext = Depends(getRequestContext)
) -> JSONResponse:
    """Duplicate a template into the current feature instance (system or instance template)."""
    try:
        chatInterface = getAutomationInterface(
            context.user,
            mandateId=str(context.mandateId) if context.mandateId else None,
            featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None
        )
        duplicated = chatInterface.duplicateAutomationTemplate(templateId)
        return JSONResponse(content=duplicated)
    except HTTPException:
        raise
    except PermissionError as e:
        raise HTTPException(
            status_code=403,
            detail=str(e)
        )
    except Exception as e:
        logger.error(f"Error duplicating template: {str(e)}")
        raise HTTPException(
            status_code=500,
            detail=f"Error duplicating template: {str(e)}"
        )


@router.post("/{automationId}/duplicate")
@limiter.limit("10/minute")
def duplicate_automation(
    request: Request,
    automationId: str = Path(..., description="Automation definition ID to duplicate"),
    context: RequestContext = Depends(getRequestContext)
) -> JSONResponse:
    """Duplicate an automation definition within the same feature instance."""
    try:
        chatInterface = getAutomationInterface(
            context.user,
            mandateId=str(context.mandateId) if context.mandateId else None,
            featureInstanceId=str(context.featureInstanceId) if context.featureInstanceId else None
        )
        duplicated = chatInterface.duplicateAutomationDefinition(automationId)
        return JSONResponse(content=duplicated)
    except HTTPException:
        raise
    except PermissionError as e:
        raise HTTPException(
            status_code=403,
            detail=str(e)
        )
    except Exception as e:
        logger.error(f"Error duplicating automation: {str(e)}")
        raise HTTPException(
            status_code=500,
            detail=f"Error duplicating automation: {str(e)}"
        )
